Best JavaScript code snippet using best
tokenMatcher.ts
Source:tokenMatcher.ts
1import { PartialToken } from '@tokens/token';2import { TokenType, tt } from '@tokens/tokenTypes';3export interface MatchSuccess {4 matched: true;5 length: number;6 token: PartialToken;7}8export interface MatchFailure {9 matched: false;10}11export type TokenMatchResult = MatchSuccess | MatchFailure;12export type TokenMatcher = (input: string) => TokenMatchResult;13/**14 * Returns whether a char code corresponds to a valid identifier character.15 * @param charCode The char code.16 * @returns Whether.17 */18const isIdentifierCharCode = (charCode: number): boolean => {19 return (charCode >= 48 && charCode <= 56) // 0-920 || (charCode >= 65 && charCode <= 90) // A-Z21 || (charCode >= 97 && charCode <= 122) // a-z22 || charCode == 36 // $23 || charCode == 95; // _24}25/**26 * Returns a matcher for a single character in the input.27 * @param type The type of the token.28 * @param char The character to be matched.29 * @returns The token matcher.30 */31const characterMatcher = (type: TokenType, char: string): TokenMatcher => {32 return (input: string) => {33 return input.charAt(0) == char34 ? {35 matched: true,36 length: 1,37 token: new PartialToken(type, char)38 }39 : {40 matched: false41 };42 };43};44/**45 * Returns a matcher for a string.46 * @param type The type of the token.47 * @param str The string to be matched.48 * @returns The token matcher.49 */50 const stringMatcher = (type: TokenType, str: string): TokenMatcher => {51 return (input: string) => {52 return input.startsWith(str)53 ? {54 matched: true,55 length: str.length,56 token: new PartialToken(type, str)57 }58 : {59 matched: false60 };61 };62};63/**64 * Returns a matcher for a keyword. This differs from a string matcher65 * as the next character cannot be a valid identifier character (as then66 * the token must be interpreted as an identifier).67 * @param type The type of the token.68 * @param str The keyword to be matched.69 * @returns The token matcher.70 */71const keywordMatcher = (type: TokenType, str: string): TokenMatcher => {72 return (input: string) => {73 return input.startsWith(str) && !isIdentifierCharCode(input.charCodeAt(str.length))74 ? {75 matched: true,76 length: str.length,77 token: new PartialToken(type, str)78 }79 : {80 matched: false81 };82 };83};84/**85 * Returns a matcher for a regular expression.86 * @param type The type of the token.87 * @param regex The regex to be matched.88 * @returns The token matcher.89 */90const regexMatcher = (type: TokenType, regex: RegExp): TokenMatcher => {91 return (input: string) => {92 const result = input.match(regex);93 if (result) {94 const match = result[0];95 const token = new PartialToken(type, match);96 return {97 matched: true,98 length: match.length,99 token100 };101 } else {102 return {103 matched: false104 };105 }106 }107};108/**109 * Matches all tokens that start with '+'.110 * @param input The input string.111 * @returns The match result.112 */113const matchPlusTokens = (input: string): TokenMatchResult => {114 let nextChar = input.charAt(0);115 if (nextChar == '+') {116 nextChar = input.charAt(1);117 if (nextChar == '+') {118 return {119 matched: true,120 length: 2,121 token: new PartialToken(tt.Increment, '++')122 };123 } else if (nextChar == '=') {124 return {125 matched: true,126 length: 2,127 token: new PartialToken(tt.AddAssignment, '+=')128 };129 } else {130 return {131 matched: true,132 length: 1,133 token: new PartialToken(tt.Add, '+')134 };135 }136 } else {137 return {138 matched: false139 };140 }141};142/**143 * Matches all tokens that start with '-'.144 * @param input The input string.145 * @returns The match result.146 */147const matchMinusTokens = (input: string): TokenMatchResult => {148 let nextChar = input.charAt(0);149 if (nextChar == '-') {150 nextChar = input.charAt(1);151 if (nextChar == '-') {152 return {153 matched: true,154 length: 2,155 token: new PartialToken(tt.Decrement, '--')156 };157 } else if (nextChar == '=') {158 return {159 matched: true,160 length: 2,161 token: new PartialToken(tt.SubtractAssignment, '-=')162 };163 } else {164 return {165 matched: true,166 length: 1,167 token: new PartialToken(tt.Subtract, '-')168 };169 }170 } else {171 return {172 matched: false173 };174 }175};176/**177 * Matches all tokens that start with '*'.178 * @param input The input string.179 * @returns The match result.180 */181const matchStarTokens = (input: string): TokenMatchResult => {182 let nextChar = input.charAt(0);183 if (nextChar == '*') {184 nextChar = input.charAt(1);185 if (nextChar == '*') {186 nextChar = input.charAt(2);187 if (nextChar == '=') {188 return {189 matched: true,190 length: 3,191 token: new PartialToken(tt.ExponentialAssignment, '**=')192 };193 } else {194 return {195 matched: true,196 length: 2,197 token: new PartialToken(tt.Exponential, '**')198 };199 }200 } else if (nextChar == '=') {201 return {202 matched: true,203 length: 2,204 token: new PartialToken(tt.MultiplyAssignment, '*=')205 };206 } else {207 return {208 matched: true,209 length: 1,210 token: new PartialToken(tt.Multiply, '*')211 };212 }213 } else {214 return {215 matched: false216 };217 }218};219/**220 * Matches all tokens that start with '/'.221 * @param input The input string.222 * @returns The match result.223 */224const matchDivideTokens = (input: string): TokenMatchResult => {225 let nextChar = input.charAt(0);226 if (nextChar == '/') {227 nextChar = input.charAt(1);228 if (nextChar == '=') {229 return {230 matched: true,231 length: 2,232 token: new PartialToken(tt.DivideAssignment, '/=')233 };234 } else {235 return {236 matched: true,237 length: 1,238 token: new PartialToken(tt.Divide, '/')239 };240 }241 } else {242 return {243 matched: false244 };245 }246};247/**248 * Matches all tokens that start with '%'.249 * @param input The input string.250 * @returns The match result.251 */252const matchModulusTokens = (input: string): TokenMatchResult => {253 let nextChar = input.charAt(0);254 if (nextChar == '%') {255 nextChar = input.charAt(1);256 if (nextChar == '=') {257 return {258 matched: true,259 length: 2,260 token: new PartialToken(tt.ModulusAssignment, '%=')261 };262 } else {263 return {264 matched: true,265 length: 1,266 token: new PartialToken(tt.Modulus, '%')267 };268 }269 } else {270 return {271 matched: false272 };273 }274};275/**276 * Matches all tokens that start with '>'.277 * @param input The input string.278 * @returns The match result.279 */280const matchLeftArrowTokens = (input: string): TokenMatchResult => {281 let nextChar = input.charAt(0);282 if (nextChar == '<') {283 nextChar = input.charAt(1);284 if (nextChar == '<') {285 nextChar = input.charAt(2);286 if (nextChar == '=') {287 return {288 matched: true,289 length: 3,290 token: new PartialToken(tt.LeftShiftAssignment, '<<=')291 };292 } else {293 return {294 matched: true,295 length: 2,296 token: new PartialToken(tt.LeftShift, '<<')297 };298 }299 } else if (nextChar == '=') {300 return {301 matched: true,302 length: 2,303 token: new PartialToken(tt.LessThanEqual, '<=')304 };305 } else {306 return {307 matched: true,308 length: 1,309 token: new PartialToken(tt.LessThan, '<')310 };311 }312 } else {313 return {314 matched: false315 };316 }317};318/**319 * Matches all tokens that start with '>'.320 * @param input The input string.321 * @returns The match result.322 */323const matchRightArrowTokens = (input: string): TokenMatchResult => {324 let nextChar = input.charAt(0);325 if (nextChar == '>') {326 nextChar = input.charAt(1);327 if (nextChar == '>') {328 nextChar = input.charAt(2);329 if (nextChar == '>') {330 nextChar = input.charAt(3);331 if (nextChar == '=') {332 return {333 matched: true,334 length: 4,335 token: new PartialToken(tt.UnsignedRightShiftAssignment, '>>>=')336 };337 } else {338 return {339 matched: true,340 length: 3,341 token: new PartialToken(tt.UnsignedRightShift, '>>>')342 };343 }344 } else if (nextChar == '=') {345 return {346 matched: true,347 length: 3,348 token: new PartialToken(tt.RightShiftAssignment, '>>=')349 };350 } else {351 return {352 matched: true,353 length: 2,354 token: new PartialToken(tt.RightShift, '>>')355 };356 }357 } else if (nextChar == '=') {358 return {359 matched: true,360 length: 2,361 token: new PartialToken(tt.GreaterThanEqual, '>=')362 };363 } else {364 return {365 matched: true,366 length: 1,367 token: new PartialToken(tt.GreaterThan, '>')368 };369 }370 } else {371 return {372 matched: false373 };374 }375};376/**377 * Matches all tokens that start with '='.378 * @param input The input string.379 * @returns The match result.380 */381const matchEqualTokens = (input: string): TokenMatchResult => {382 let nextChar = input.charAt(0);383 if (nextChar == '=') {384 nextChar = input.charAt(1);385 if (nextChar == '=') {386 nextChar = input.charAt(2);387 if (nextChar == '=') {388 return {389 matched: true,390 length: 3,391 token: new PartialToken(tt.StrictEquality, '===')392 };393 } else {394 return {395 matched: true,396 length: 2,397 token: new PartialToken(tt.Equality, '==')398 };399 }400 } else if (nextChar == '>') {401 return {402 matched: true,403 length: 2,404 token: new PartialToken(tt.Arrow, '=>')405 };406 } else {407 return {408 matched: true,409 length: 1,410 token: new PartialToken(tt.Assignment, '=')411 };412 }413 } else {414 return {415 matched: false416 };417 }418}419/**420 * Matches all tokens that start with '!'.421 * @param input The input string.422 * @returns The match result.423 */424 const matchExclamationTokens = (input: string): TokenMatchResult => {425 let nextChar = input.charAt(0);426 if (nextChar == '!') {427 nextChar = input.charAt(1);428 if (nextChar == '=') {429 nextChar = input.charAt(2);430 if (nextChar == '=') {431 return {432 matched: true,433 length: 3,434 token: new PartialToken(tt.StrictInequality, '!==')435 };436 } else {437 return {438 matched: true,439 length: 2,440 token: new PartialToken(tt.Inequality, '!=')441 };442 }443 } else {444 return {445 matched: true,446 length: 1,447 token: new PartialToken(tt.Not, '!')448 };449 }450 } else {451 return {452 matched: false453 };454 }455}456/**457 * Matches all tokens that start with '|'.458 * @param input The input string.459 * @returns The match result.460 */461const matchBarTokens = (input: string): TokenMatchResult => {462 let nextChar = input.charAt(0);463 if (nextChar == '|') {464 nextChar = input.charAt(1);465 if (nextChar == '|') {466 nextChar = input.charAt(2);467 if (nextChar == '=') {468 return {469 matched: true,470 length: 3,471 token: new PartialToken(tt.OrAssignment, '||=')472 };473 } else {474 return {475 matched: true,476 length: 2,477 token: new PartialToken(tt.Or, '||')478 };479 }480 } else if (nextChar == '=') {481 return {482 matched: true,483 length: 2,484 token: new PartialToken(tt.BitwiseOrAssignment, '|=')485 };486 } else {487 return {488 matched: true,489 length: 1,490 token: new PartialToken(tt.BitwiseOr, '|')491 };492 }493 } else {494 return {495 matched: false496 };497 }498};499/**500 * Matches all tokens that start with '^'.501 * @param input The input string.502 * @returns The match result.503 */504const matchCaretTokens = (input: string): TokenMatchResult => {505 let nextChar = input.charAt(0);506 if (nextChar == '^') {507 nextChar = input.charAt(1);508 if (nextChar == '=') {509 return {510 matched: true,511 length: 2,512 token: new PartialToken(tt.BitwiseXorAssignment, '^=')513 };514 } else {515 return {516 matched: true,517 length: 1,518 token: new PartialToken(tt.BitwiseXor, '^')519 };520 }521 } else {522 return {523 matched: false524 };525 }526};527/**528 * Matches all tokens that start with '&'.529 * @param input The input string.530 * @returns The match result.531 */532const matchAmpersandTokens = (input: string): TokenMatchResult => {533 let nextChar = input.charAt(0);534 if (nextChar == '&') {535 nextChar = input.charAt(1);536 if (nextChar == '&') {537 nextChar = input.charAt(2);538 if (nextChar == '=') {539 return {540 matched: true,541 length: 3,542 token: new PartialToken(tt.AndAssignment, '&&=')543 };544 } else {545 return {546 matched: true,547 length: 2,548 token: new PartialToken(tt.And, '&&')549 };550 }551 } else if (nextChar == '=') {552 return {553 matched: true,554 length: 2,555 token: new PartialToken(tt.BitwiseAndAssignment, '&=')556 };557 } else {558 return {559 matched: true,560 length: 1,561 token: new PartialToken(tt.BitwiseAnd, '&')562 };563 }564 } else {565 return {566 matched: false567 };568 }569};570/**571 * Matches all tokens that start with '?'.572 * @param input The input string.573 * @returns The match result.574 */575const matchQuestionTokens = (input: string): TokenMatchResult => {576 let nextChar = input.charAt(0);577 if (nextChar == '?') {578 nextChar = input.charAt(1);579 if (nextChar == '?') {580 nextChar = input.charAt(2);581 if (nextChar == '=') {582 return {583 matched: true,584 length: 3,585 token: new PartialToken(tt.NullCoalescingAssignment, '??=')586 };587 } else {588 return {589 matched: true,590 length: 2,591 token: new PartialToken(tt.NullCoalescing, '??')592 };593 }594 } else {595 return {596 matched: true,597 length: 1,598 token: new PartialToken(tt.Question, '?')599 };600 }601 } else {602 return {603 matched: false604 };605 }606};607/**608 * Matches single line strings (i.e. strings that start with ' or ").609 * @param input The input string.610 * @returns The match result.611 */612export const matchSingleLineString = (input: string): TokenMatchResult => {613 const firstChar = input.charAt(0);614 if (firstChar == "'" || firstChar == '"') {615 let pos = 1;616 let value = '';617 let lastChar = '';618 while (true) {619 if (pos >= input.length) {620 return {621 matched: false622 };623 }624 const char = input.charAt(pos++);625 if ((char == firstChar && lastChar != '\\')) {626 break;627 } else if (char == '\n') {628 return { 629 matched: false630 };631 }632 lastChar = char;633 value += char;634 }635 return {636 matched: true,637 length: value.length + 2,638 token: new PartialToken(tt.String, value)639 };640 } else {641 return {642 matched: false643 }644 }645};646/**647 * Matches a template string (i.e. strings that start with `).648 * @param input The input string.649 * @returns The match result.650 */651export const matchTemplateString = (input: string): TokenMatchResult => {652 const firstChar = input.charAt(0);653 if (firstChar == "`") {654 let pos = 1;655 let value = '';656 let lastChar = '';657 while (true) {658 if (pos >= input.length) {659 return {660 matched: false661 };662 }663 const char = input.charAt(pos++);664 if ((char == firstChar && lastChar != '\\')) {665 break;666 }667 lastChar = char;668 value += char;669 }670 return {671 matched: true,672 length: value.length + 2,673 token: new PartialToken(tt.TemplateString, value)674 };675 } else {676 return {677 matched: false678 }679 }680};681// used as key for all other characters in matcher map682export const OTHER_CHARS_KEY = {};683/**684 * Map of token matchers grouped by the first char they match.685 * Allows for more efficient token matching.686 */687export const matcherMap: Map<string | {}, TokenMatcher[]> = new Map([688 ['a', [keywordMatcher(tt.Async, 'async'), keywordMatcher(tt.Await, 'await')]],689 ['b', [keywordMatcher(tt.Break, 'break')]],690 ['c', [keywordMatcher(tt.Case, 'case'), keywordMatcher(tt.Catch, 'catch'), keywordMatcher(tt.Class, 'class'), keywordMatcher(tt.Const, 'const'), keywordMatcher(tt.Continue, 'continue')]],691 ['d', [keywordMatcher(tt.Debugger, 'debugger'), keywordMatcher(tt.Default, 'default'), keywordMatcher(tt.Delete, 'delete'), keywordMatcher(tt.Do, 'do')]],692 ['e', [keywordMatcher(tt.Else, 'else'), keywordMatcher(tt.Extends, 'extends')]],693 ['f', [keywordMatcher(tt.False, 'false'), keywordMatcher(tt.Finally, 'finally'), keywordMatcher(tt.For, 'for'), keywordMatcher(tt.Function, 'function')]],694 ['i', [keywordMatcher(tt.If, 'if'), keywordMatcher(tt.InstanceOf, 'instanceof'), keywordMatcher(tt.In, 'in')]],695 ['l', [keywordMatcher(tt.Let, 'let')]],696 ['n', [keywordMatcher(tt.New, 'new'), keywordMatcher(tt.Null, 'null')]],697 ['o', [keywordMatcher(tt.Of, 'of')]],698 ['r', [keywordMatcher(tt.Return, 'return')]],699 ['s', [keywordMatcher(tt.Static, 'static'), keywordMatcher(tt.Super, 'super'), keywordMatcher(tt.Switch, 'switch')]],700 ['t', [keywordMatcher(tt.This, 'this'), keywordMatcher(tt.Throw, 'throw'), keywordMatcher(tt.True, 'true'), keywordMatcher(tt.Try, 'try'), keywordMatcher(tt.Typeof, 'typeof')]],701 ['v', [keywordMatcher(tt.Var, 'var'), keywordMatcher(tt.Void, 'void')]],702 ['w', [keywordMatcher(tt.While, 'while'), keywordMatcher(tt.With, 'with')]],703 ['y', [keywordMatcher(tt.Yield, 'yield')]],704 [',', [characterMatcher(tt.Comma, ',')]],705 ['+', [matchPlusTokens]],706 ['-', [matchMinusTokens]],707 ['*', [matchStarTokens]],708 ['/', [matchDivideTokens]],709 ['%', [matchModulusTokens]],710 ['<', [matchLeftArrowTokens]],711 ['>', [matchRightArrowTokens]],712 ['=', [matchEqualTokens]],713 ['!', [matchExclamationTokens]],714 ['|', [matchBarTokens]],715 ['^', [matchCaretTokens]],716 ['&', [matchAmpersandTokens]],717 ['?', [matchQuestionTokens]],718 ['~', [characterMatcher(tt.BitwiseNot, '~')]],719 ['[', [characterMatcher(tt.LeftBracket, '[')]],720 [']', [characterMatcher(tt.RightBracket, ']')]],721 ['(', [characterMatcher(tt.LeftParenthesis, '(')]],722 [')', [characterMatcher(tt.RightParenthesis, ')')]],723 ['{', [characterMatcher(tt.LeftBrace, '{')]],724 ['}', [characterMatcher(tt.RightBrace, '}')]],725 [':', [characterMatcher(tt.Colon, ':')]],726 [';', [characterMatcher(tt.SemiColon, ';')]],727 ['#', [characterMatcher(tt.Hash, '#')]],728 ['.', [stringMatcher(tt.Ellipsis, '...'), characterMatcher(tt.Dot, '.')]],729 ["'", [matchSingleLineString]],730 ['"', [matchSingleLineString]],731 ['`', [matchTemplateString]],732 // matches all other characters733 [OTHER_CHARS_KEY, [734 regexMatcher(tt.Identifier, /^[a-zA-Z_$][a-zA-Z0-9_$]*/),735 regexMatcher(tt.Number, /^[0-9]+/)736 ]],...
form_combobox.spec.js
Source:form_combobox.spec.js
1import { mount } from '@vue/test-utils';2import GlDropdownItem from '../../dropdown/dropdown_item.vue';3import GlFormInput from '../form_input/form_input.vue';4import { tokenList, labelText } from './constants';5import GlFormCombobox from './form_combobox.vue';6const partialToken = 'do';7const partialTokenMatch = ['dog', 'dodo', 'komodo dragon'];8const unlistedToken = 'elephant';9const doTimes = (num, fn) => {10 for (let i = 0; i < num; i += 1) {11 fn();12 }13};14describe('GlFormCombobox', () => {15 let wrapper;16 const createComponent = () => {17 wrapper = mount({18 data() {19 return {20 inputVal: '',21 tokens: tokenList,22 labelText,23 };24 },25 components: { GlFormCombobox },26 template: `27 <div>28 <gl-form-combobox29 v-model="inputVal"30 :token-list="tokens"31 :labelText="labelText"32 />33 </div>34 `,35 });36 };37 // needs new selector now38 const findDropdown = () => wrapper.find('[data-testid="combobox-dropdown"]');39 const findDropdownOptions = () =>40 wrapper.findAllComponents(GlDropdownItem).wrappers.map((item) => item.text());41 const findInput = () => wrapper.findComponent(GlFormInput);42 const findInputValue = () => findInput().element.value;43 const setInput = (val) => findInput().setValue(val);44 const arrowDown = () => findInput().trigger('keydown.down');45 afterEach(() => {46 wrapper.destroy();47 });48 describe('match and filter functionality', () => {49 beforeEach(() => {50 createComponent();51 });52 it('is closed when the input is empty', () => {53 expect(findInput().isVisible()).toBe(true);54 expect(findInputValue()).toBe('');55 expect(findDropdown().isVisible()).toBe(false);56 });57 it('is open when the input text matches a token', async () => {58 await setInput(partialToken);59 expect(findDropdown().isVisible()).toBe(true);60 });61 it('shows partial matches at string start and mid-string', async () => {62 await setInput(partialToken);63 expect(findDropdown().isVisible()).toBe(true);64 expect(findDropdownOptions()).toEqual(partialTokenMatch);65 });66 it('is closed when the text does not match', async () => {67 await setInput(unlistedToken);68 expect(findDropdown().isVisible()).toBe(false);69 });70 });71 describe('keyboard navigation in dropdown', () => {72 beforeEach(() => {73 createComponent();74 });75 describe('on down arrow + enter', () => {76 it('selects the next item in the list and closes the dropdown', async () => {77 await setInput(partialToken);78 findInput().trigger('keydown.down');79 await findInput().trigger('keydown.enter');80 expect(findInputValue()).toBe(partialTokenMatch[0]);81 });82 it('loops to the top when it reaches the bottom', async () => {83 await setInput(partialToken);84 doTimes(findDropdownOptions().length + 1, arrowDown);85 await findInput().trigger('keydown.enter');86 expect(findInputValue()).toBe(partialTokenMatch[0]);87 });88 });89 describe('on up arrow + enter', () => {90 it('selects the previous item in the list and closes the dropdown', async () => {91 setInput(partialToken);92 await wrapper.vm.$nextTick();93 doTimes(3, arrowDown);94 findInput().trigger('keydown.up');95 findInput().trigger('keydown.enter');96 await wrapper.vm.$nextTick();97 expect(findInputValue()).toBe(partialTokenMatch[1]);98 expect(findDropdown().isVisible()).toBe(false);99 });100 it('loops to the bottom when it reaches the top', async () => {101 await setInput(partialToken);102 findInput().trigger('keydown.down');103 findInput().trigger('keydown.up');104 await findInput().trigger('keydown.enter');105 expect(findInputValue()).toBe(partialTokenMatch[partialTokenMatch.length - 1]);106 });107 });108 describe('on enter with no item highlighted', () => {109 it('does not select any item and closes the dropdown', async () => {110 await setInput(partialToken);111 await findInput().trigger('keydown.enter');112 expect(findInputValue()).toBe(partialToken);113 expect(findDropdown().isVisible()).toBe(false);114 });115 });116 describe('on click', () => {117 it('selects the clicked item regardless of arrow highlight', async () => {118 await setInput(partialToken);119 await wrapper.find('[data-testid="combobox-dropdown"] button').trigger('click');120 expect(findInputValue()).toBe(partialTokenMatch[0]);121 });122 });123 describe('on tab', () => {124 it('selects entered text, closes dropdown', async () => {125 await setInput(partialToken);126 findInput().trigger('keydown.tab');127 doTimes(2, arrowDown);128 await wrapper.vm.$nextTick();129 expect(findInputValue()).toBe(partialToken);130 expect(findDropdown().isVisible()).toBe(false);131 });132 });133 describe('on esc', () => {134 describe('when dropdown is open', () => {135 it('closes dropdown and does not select anything', async () => {136 await setInput(partialToken);137 await findInput().trigger('keydown.esc');138 expect(findInputValue()).toBe(partialToken);139 expect(findDropdown().isVisible()).toBe(false);140 });141 });142 describe('when dropdown is closed', () => {143 it('clears the input field', async () => {144 await setInput(unlistedToken);145 expect(findDropdown().isVisible()).toBe(false);146 await findInput().trigger('keydown.esc');147 expect(findInputValue()).toBe('');148 });149 });150 });151 });...
tokenizer.ts
Source:tokenizer.ts
1import * as _ from 'lodash';2export type TokenType = 'comma' | 'identifier' | 'lparen' | 'number' | 'rparen' | 'space' | 'string';3export interface Token {4 type: TokenType;5 value: string;6 startIndex: number;7 endIndex: number;8}9export class Tokenizer {10 public static parse(codeFragment: string): Tokenizer {11 const lexer = new this(codeFragment);12 lexer.parse();13 return lexer;14 }15 private readonly regExps: Record<TokenType, RegExp> = {16 comma: /^,/s,17 lparen: /^\(/s,18 rparen: /^\)/s,19 number: /^\d+(\.\d+)?/s,20 identifier: /^[\w.-]+/s,21 space: /^\s+/s,22 string: /^(['"])((?:\\\1|(?!\1).)*)\1/s,23 };24 private tokens: Token[] = [];25 private constructor(26 private readonly source: string,27 ) {}28 public getTokens(): Token[] {29 return [...this.tokens];30 }31 private parse(): void {32 const tokens: Token[] = []33 let sourcePart = this.source;34 let nextStartIndex = 0;35 const ITERATIONS_LIMIT = 50000; // iterations limit to protect an infinite loop36 let i: number;37 for (i = 0; i < ITERATIONS_LIMIT; i++) {38 const partialToken = this.findToken(sourcePart);39 if (!partialToken) break;40 tokens.push({41 ...partialToken,42 startIndex: nextStartIndex,43 endIndex: nextStartIndex + partialToken.value.length - 1,44 });45 nextStartIndex += partialToken.value.length46 sourcePart = sourcePart.slice(partialToken.value.length);47 }48 if (i >= ITERATIONS_LIMIT) {49 throw new Error(`Lexer.parse() Iterations limit has been reached. Usually it means the loop is infinite.`);50 }51 if (sourcePart.length) {52 throw new Error(`Lexer.parse() Unexpected token at ${ nextStartIndex }. Invalid fragment: ${ sourcePart.slice(0, 10) } ...`);53 }54 this.tokens = tokens;55 }56 private findToken(sourcePart: string): Pick<Token, 'type' | 'value'> | undefined {57 let res: Pick<Token, 'type' | 'value'> | undefined;58 _.some<any>(this.regExps, (regExp: RegExp, type: TokenType) => {59 // console.log('TYPE: ', type);60 const match = sourcePart.match(regExp);61 if (match?.[0]) {62 res = { type, value: match[0] }63 return true;64 }65 });66 return res;67 }...
Using AI Code Generation
1var bestbuy = require('bestbuy')('your-api-key-here');2var prompt = require('prompt');3prompt.start();4prompt.get(['search'], function(err, result) {5 bestbuy.products('(search=' + result.search + ')', {show: 'sku,name,salePrice', page: 1, pageSize: 5}, function(err, data) {6 console.log(data.products);7 });8});
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Get 100 minutes of automation test minutes FREE!!