1namespace ts { 2 export type ErrorCallback = (message: DiagnosticMessage, length: number) => void; 3 4 /* @internal */ 5 export function tokenIsIdentifierOrKeyword(token: SyntaxKind): boolean { 6 return token >= SyntaxKind.Identifier; 7 } 8 9 /* @internal */ 10 export function tokenIsIdentifierOrKeywordOrGreaterThan(token: SyntaxKind): boolean { 11 return token === SyntaxKind.GreaterThanToken || tokenIsIdentifierOrKeyword(token); 12 } 13 14 export interface Scanner { 15 getStartPos(): number; 16 getToken(): SyntaxKind; 17 getTextPos(): number; 18 getTokenPos(): number; 19 getTokenText(): string; 20 getTokenValue(): string; 21 hasUnicodeEscape(): boolean; 22 hasExtendedUnicodeEscape(): boolean; 23 hasPrecedingLineBreak(): boolean; 24 /* @internal */ 25 hasPrecedingJSDocComment(): boolean; 26 isIdentifier(): boolean; 27 isReservedWord(): boolean; 28 isUnterminated(): boolean; 29 /* @internal */ 30 getNumericLiteralFlags(): TokenFlags; 31 /* @internal */ 32 getCommentDirectives(): CommentDirective[] | undefined; 33 /* @internal */ 34 getTokenFlags(): TokenFlags; 35 reScanGreaterToken(): SyntaxKind; 36 reScanSlashToken(): SyntaxKind; 37 reScanAsteriskEqualsToken(): SyntaxKind; 38 reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind; 39 reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind; 40 scanJsxIdentifier(): SyntaxKind; 41 scanJsxAttributeValue(): SyntaxKind; 42 reScanJsxAttributeValue(): SyntaxKind; 43 reScanJsxToken(allowMultilineJsxText?: boolean): JsxTokenSyntaxKind; 44 reScanLessThanToken(): SyntaxKind; 45 reScanHashToken(): SyntaxKind; 46 reScanQuestionToken(): SyntaxKind; 47 reScanInvalidIdentifier(): SyntaxKind; 48 scanJsxToken(): JsxTokenSyntaxKind; 49 scanJsDocToken(): JSDocSyntaxKind; 50 scan(): SyntaxKind; 51 52 getText(): string; 53 /* @internal */ 54 clearCommentDirectives(): void; 55 // Sets the text for the scanner to scan. An optional subrange starting point and length 56 // can be provided to have the scanner only scan a portion of the text. 57 setText(text: string | undefined, start?: number, length?: number): void; 58 setOnError(onError: ErrorCallback | undefined): void; 59 setScriptTarget(scriptTarget: ScriptTarget): void; 60 setLanguageVariant(variant: LanguageVariant): void; 61 setTextPos(textPos: number): void; 62 /* @internal */ 63 setInJSDocType(inType: boolean): void; 64 // Invokes the provided callback then unconditionally restores the scanner to the state it 65 // was in immediately prior to invoking the callback. The result of invoking the callback 66 // is returned from this function. 67 lookAhead<T>(callback: () => T): T; 68 69 // Invokes the callback with the scanner set to scan the specified range. When the callback 70 // returns, the scanner is restored to the state it was in before scanRange was called. 71 scanRange<T>(start: number, length: number, callback: () => T): T; 72 73 // Invokes the provided callback. If the callback returns something falsy, then it restores 74 // the scanner to the state it was in immediately prior to invoking the callback. If the 75 // callback returns something truthy, then the scanner state is not rolled back. The result 76 // of invoking the callback is returned from this function. 77 tryScan<T>(callback: () => T): T; 78 setEtsContext(isEtsContext: boolean): void; 79 } 80 81 /** @internal */ 82 export const textToKeywordObj: MapLike<KeywordSyntaxKind> = { 83 abstract: SyntaxKind.AbstractKeyword, 84 accessor: SyntaxKind.AccessorKeyword, 85 any: SyntaxKind.AnyKeyword, 86 as: SyntaxKind.AsKeyword, 87 asserts: SyntaxKind.AssertsKeyword, 88 assert: SyntaxKind.AssertKeyword, 89 bigint: SyntaxKind.BigIntKeyword, 90 boolean: SyntaxKind.BooleanKeyword, 91 break: SyntaxKind.BreakKeyword, 92 case: SyntaxKind.CaseKeyword, 93 catch: SyntaxKind.CatchKeyword, 94 class: SyntaxKind.ClassKeyword, 95 continue: SyntaxKind.ContinueKeyword, 96 const: SyntaxKind.ConstKeyword, 97 ["" + "constructor"]: SyntaxKind.ConstructorKeyword, 98 debugger: SyntaxKind.DebuggerKeyword, 99 declare: SyntaxKind.DeclareKeyword, 100 default: SyntaxKind.DefaultKeyword, 101 delete: SyntaxKind.DeleteKeyword, 102 do: SyntaxKind.DoKeyword, 103 else: SyntaxKind.ElseKeyword, 104 enum: SyntaxKind.EnumKeyword, 105 export: SyntaxKind.ExportKeyword, 106 extends: SyntaxKind.ExtendsKeyword, 107 false: SyntaxKind.FalseKeyword, 108 finally: SyntaxKind.FinallyKeyword, 109 for: SyntaxKind.ForKeyword, 110 from: SyntaxKind.FromKeyword, 111 function: SyntaxKind.FunctionKeyword, 112 get: SyntaxKind.GetKeyword, 113 if: SyntaxKind.IfKeyword, 114 implements: SyntaxKind.ImplementsKeyword, 115 import: SyntaxKind.ImportKeyword, 116 in: SyntaxKind.InKeyword, 117 infer: SyntaxKind.InferKeyword, 118 instanceof: SyntaxKind.InstanceOfKeyword, 119 interface: SyntaxKind.InterfaceKeyword, 120 intrinsic: SyntaxKind.IntrinsicKeyword, 121 is: SyntaxKind.IsKeyword, 122 keyof: SyntaxKind.KeyOfKeyword, 123 let: SyntaxKind.LetKeyword, 124 module: SyntaxKind.ModuleKeyword, 125 namespace: SyntaxKind.NamespaceKeyword, 126 never: SyntaxKind.NeverKeyword, 127 new: SyntaxKind.NewKeyword, 128 null: SyntaxKind.NullKeyword, 129 number: SyntaxKind.NumberKeyword, 130 object: SyntaxKind.ObjectKeyword, 131 package: SyntaxKind.PackageKeyword, 132 private: SyntaxKind.PrivateKeyword, 133 protected: SyntaxKind.ProtectedKeyword, 134 public: SyntaxKind.PublicKeyword, 135 override: SyntaxKind.OverrideKeyword, 136 out: SyntaxKind.OutKeyword, 137 readonly: SyntaxKind.ReadonlyKeyword, 138 require: SyntaxKind.RequireKeyword, 139 global: SyntaxKind.GlobalKeyword, 140 return: SyntaxKind.ReturnKeyword, 141 satisfies: SyntaxKind.SatisfiesKeyword, 142 set: SyntaxKind.SetKeyword, 143 static: SyntaxKind.StaticKeyword, 144 string: SyntaxKind.StringKeyword, 145 struct: SyntaxKind.StructKeyword, 146 super: SyntaxKind.SuperKeyword, 147 switch: SyntaxKind.SwitchKeyword, 148 symbol: SyntaxKind.SymbolKeyword, 149 this: SyntaxKind.ThisKeyword, 150 throw: SyntaxKind.ThrowKeyword, 151 true: SyntaxKind.TrueKeyword, 152 try: SyntaxKind.TryKeyword, 153 type: SyntaxKind.TypeKeyword, 154 typeof: SyntaxKind.TypeOfKeyword, 155 undefined: SyntaxKind.UndefinedKeyword, 156 unique: SyntaxKind.UniqueKeyword, 157 unknown: SyntaxKind.UnknownKeyword, 158 var: SyntaxKind.VarKeyword, 159 void: SyntaxKind.VoidKeyword, 160 while: SyntaxKind.WhileKeyword, 161 with: SyntaxKind.WithKeyword, 162 yield: SyntaxKind.YieldKeyword, 163 async: SyntaxKind.AsyncKeyword, 164 await: SyntaxKind.AwaitKeyword, 165 of: SyntaxKind.OfKeyword, 166 }; 167 168 const textToKeyword = new Map(getEntries(textToKeywordObj)); 169 170 const textToToken = new Map(getEntries({ 171 ...textToKeywordObj, 172 "{": SyntaxKind.OpenBraceToken, 173 "}": SyntaxKind.CloseBraceToken, 174 "(": SyntaxKind.OpenParenToken, 175 ")": SyntaxKind.CloseParenToken, 176 "[": SyntaxKind.OpenBracketToken, 177 "]": SyntaxKind.CloseBracketToken, 178 ".": SyntaxKind.DotToken, 179 "...": SyntaxKind.DotDotDotToken, 180 ";": SyntaxKind.SemicolonToken, 181 ",": SyntaxKind.CommaToken, 182 "<": SyntaxKind.LessThanToken, 183 ">": SyntaxKind.GreaterThanToken, 184 "<=": SyntaxKind.LessThanEqualsToken, 185 ">=": SyntaxKind.GreaterThanEqualsToken, 186 "==": SyntaxKind.EqualsEqualsToken, 187 "!=": SyntaxKind.ExclamationEqualsToken, 188 "===": SyntaxKind.EqualsEqualsEqualsToken, 189 "!==": SyntaxKind.ExclamationEqualsEqualsToken, 190 "=>": SyntaxKind.EqualsGreaterThanToken, 191 "+": SyntaxKind.PlusToken, 192 "-": SyntaxKind.MinusToken, 193 "**": SyntaxKind.AsteriskAsteriskToken, 194 "*": SyntaxKind.AsteriskToken, 195 "/": SyntaxKind.SlashToken, 196 "%": SyntaxKind.PercentToken, 197 "++": SyntaxKind.PlusPlusToken, 198 "--": SyntaxKind.MinusMinusToken, 199 "<<": SyntaxKind.LessThanLessThanToken, 200 "</": SyntaxKind.LessThanSlashToken, 201 ">>": SyntaxKind.GreaterThanGreaterThanToken, 202 ">>>": SyntaxKind.GreaterThanGreaterThanGreaterThanToken, 203 "&": SyntaxKind.AmpersandToken, 204 "|": SyntaxKind.BarToken, 205 "^": SyntaxKind.CaretToken, 206 "!": SyntaxKind.ExclamationToken, 207 "~": SyntaxKind.TildeToken, 208 "&&": SyntaxKind.AmpersandAmpersandToken, 209 "||": SyntaxKind.BarBarToken, 210 "?": SyntaxKind.QuestionToken, 211 "??": SyntaxKind.QuestionQuestionToken, 212 "?.": SyntaxKind.QuestionDotToken, 213 ":": SyntaxKind.ColonToken, 214 "=": SyntaxKind.EqualsToken, 215 "+=": SyntaxKind.PlusEqualsToken, 216 "-=": SyntaxKind.MinusEqualsToken, 217 "*=": SyntaxKind.AsteriskEqualsToken, 218 "**=": SyntaxKind.AsteriskAsteriskEqualsToken, 219 "/=": SyntaxKind.SlashEqualsToken, 220 "%=": SyntaxKind.PercentEqualsToken, 221 "<<=": SyntaxKind.LessThanLessThanEqualsToken, 222 ">>=": SyntaxKind.GreaterThanGreaterThanEqualsToken, 223 ">>>=": SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken, 224 "&=": SyntaxKind.AmpersandEqualsToken, 225 "|=": SyntaxKind.BarEqualsToken, 226 "^=": SyntaxKind.CaretEqualsToken, 227 "||=": SyntaxKind.BarBarEqualsToken, 228 "&&=": SyntaxKind.AmpersandAmpersandEqualsToken, 229 "??=": SyntaxKind.QuestionQuestionEqualsToken, 230 "@": SyntaxKind.AtToken, 231 "#": SyntaxKind.HashToken, 232 "`": SyntaxKind.BacktickToken, 233 })); 234 235 /* 236 As per ECMAScript Language Specification 3th Edition, Section 7.6: Identifiers 237 IdentifierStart :: 238 Can contain Unicode 3.0.0 categories: 239 Uppercase letter (Lu), 240 Lowercase letter (Ll), 241 Titlecase letter (Lt), 242 Modifier letter (Lm), 243 Other letter (Lo), or 244 Letter number (Nl). 245 IdentifierPart :: = 246 Can contain IdentifierStart + Unicode 3.0.0 categories: 247 Non-spacing mark (Mn), 248 Combining spacing mark (Mc), 249 Decimal number (Nd), or 250 Connector punctuation (Pc). 251 252 Codepoint ranges for ES3 Identifiers are extracted from the Unicode 3.0.0 specification at: 253 http://www.unicode.org/Public/3.0-Update/UnicodeData-3.0.0.txt 254 */ 255 const unicodeES3IdentifierStart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 543, 546, 563, 592, 685, 688, 696, 699, 705, 720, 721, 736, 740, 750, 750, 890, 890, 902, 902, 904, 906, 908, 908, 910, 929, 931, 974, 976, 983, 986, 1011, 1024, 1153, 1164, 1220, 1223, 1224, 1227, 1228, 1232, 1269, 1272, 1273, 1329, 1366, 1369, 1369, 1377, 1415, 1488, 1514, 1520, 1522, 1569, 1594, 1600, 1610, 1649, 1747, 1749, 1749, 1765, 1766, 1786, 1788, 1808, 1808, 1810, 1836, 1920, 1957, 2309, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2524, 2525, 2527, 2529, 2544, 2545, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2699, 2701, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2784, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2870, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 2997, 2999, 3001, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3168, 3169, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3294, 3294, 3296, 3297, 3333, 3340, 3342, 3344, 3346, 3368, 3370, 3385, 3424, 3425, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3805, 3840, 3840, 3904, 3911, 3913, 3946, 3976, 3979, 4096, 4129, 4131, 4135, 4137, 4138, 4176, 4181, 4256, 4293, 4304, 4342, 4352, 4441, 4447, 4514, 4520, 4601, 4608, 4614, 4616, 4678, 4680, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4742, 4744, 4744, 4746, 4749, 4752, 4782, 4784, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4814, 4816, 4822, 4824, 4846, 4848, 4878, 4880, 4880, 4882, 4885, 4888, 4894, 4896, 4934, 4936, 4954, 5024, 5108, 5121, 5740, 5743, 5750, 5761, 5786, 5792, 5866, 6016, 6067, 6176, 6263, 6272, 6312, 7680, 7835, 7840, 7929, 7936, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8319, 8319, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8497, 8499, 8505, 8544, 8579, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12346, 12353, 12436, 12445, 12446, 12449, 12538, 12540, 12542, 12549, 12588, 12593, 12686, 12704, 12727, 13312, 19893, 19968, 40869, 40960, 42124, 44032, 55203, 63744, 64045, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65138, 65140, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 256 const unicodeES3IdentifierPart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 543, 546, 563, 592, 685, 688, 696, 699, 705, 720, 721, 736, 740, 750, 750, 768, 846, 864, 866, 890, 890, 902, 902, 904, 906, 908, 908, 910, 929, 931, 974, 976, 983, 986, 1011, 1024, 1153, 1155, 1158, 1164, 1220, 1223, 1224, 1227, 1228, 1232, 1269, 1272, 1273, 1329, 1366, 1369, 1369, 1377, 1415, 1425, 1441, 1443, 1465, 1467, 1469, 1471, 1471, 1473, 1474, 1476, 1476, 1488, 1514, 1520, 1522, 1569, 1594, 1600, 1621, 1632, 1641, 1648, 1747, 1749, 1756, 1759, 1768, 1770, 1773, 1776, 1788, 1808, 1836, 1840, 1866, 1920, 1968, 2305, 2307, 2309, 2361, 2364, 2381, 2384, 2388, 2392, 2403, 2406, 2415, 2433, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2562, 2562, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2649, 2652, 2654, 2654, 2662, 2676, 2689, 2691, 2693, 2699, 2701, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2784, 2790, 2799, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2870, 2873, 2876, 2883, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2913, 2918, 2927, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 2997, 2999, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3031, 3031, 3047, 3055, 3073, 3075, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3168, 3169, 3174, 3183, 3202, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3262, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3297, 3302, 3311, 3330, 3331, 3333, 3340, 3342, 3344, 3346, 3368, 3370, 3385, 3390, 3395, 3398, 3400, 3402, 3405, 3415, 3415, 3424, 3425, 3430, 3439, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3769, 3771, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3805, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3946, 3953, 3972, 3974, 3979, 3984, 3991, 3993, 4028, 4038, 4038, 4096, 4129, 4131, 4135, 4137, 4138, 4140, 4146, 4150, 4153, 4160, 4169, 4176, 4185, 4256, 4293, 4304, 4342, 4352, 4441, 4447, 4514, 4520, 4601, 4608, 4614, 4616, 4678, 4680, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4742, 4744, 4744, 4746, 4749, 4752, 4782, 4784, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4814, 4816, 4822, 4824, 4846, 4848, 4878, 4880, 4880, 4882, 4885, 4888, 4894, 4896, 4934, 4936, 4954, 4969, 4977, 5024, 5108, 5121, 5740, 5743, 5750, 5761, 5786, 5792, 5866, 6016, 6099, 6112, 6121, 6160, 6169, 6176, 6263, 6272, 6313, 7680, 7835, 7840, 7929, 7936, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8255, 8256, 8319, 8319, 8400, 8412, 8417, 8417, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8497, 8499, 8505, 8544, 8579, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12346, 12353, 12436, 12441, 12442, 12445, 12446, 12449, 12542, 12549, 12588, 12593, 12686, 12704, 12727, 13312, 19893, 19968, 40869, 40960, 42124, 44032, 55203, 63744, 64045, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65056, 65059, 65075, 65076, 65101, 65103, 65136, 65138, 65140, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65381, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 257 258 /* 259 As per ECMAScript Language Specification 5th Edition, Section 7.6: ISyntaxToken Names and Identifiers 260 IdentifierStart :: 261 Can contain Unicode 6.2 categories: 262 Uppercase letter (Lu), 263 Lowercase letter (Ll), 264 Titlecase letter (Lt), 265 Modifier letter (Lm), 266 Other letter (Lo), or 267 Letter number (Nl). 268 IdentifierPart :: 269 Can contain IdentifierStart + Unicode 6.2 categories: 270 Non-spacing mark (Mn), 271 Combining spacing mark (Mc), 272 Decimal number (Nd), 273 Connector punctuation (Pc), 274 <ZWNJ>, or 275 <ZWJ>. 276 277 Codepoint ranges for ES5 Identifiers are extracted from the Unicode 6.2 specification at: 278 http://www.unicode.org/Public/6.2.0/ucd/UnicodeData.txt 279 */ 280 const unicodeES5IdentifierStart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1319, 1329, 1366, 1369, 1369, 1377, 1415, 1488, 1514, 1520, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2208, 2208, 2210, 2220, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2423, 2425, 2431, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3133, 3133, 3160, 3161, 3168, 3169, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3294, 3294, 3296, 3297, 3313, 3314, 3333, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3424, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5108, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5872, 5888, 5900, 5902, 5905, 5920, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6263, 6272, 6312, 6314, 6314, 6320, 6389, 6400, 6428, 6480, 6509, 6512, 6516, 6528, 6571, 6593, 6599, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6987, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7401, 7404, 7406, 7409, 7413, 7414, 7424, 7615, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12589, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40908, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42647, 42656, 42735, 42775, 42783, 42786, 42888, 42891, 42894, 42896, 42899, 42912, 42922, 43000, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642, 43648, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43968, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 281 const unicodeES5IdentifierPart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 768, 884, 886, 887, 890, 893, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1155, 1159, 1162, 1319, 1329, 1366, 1369, 1369, 1377, 1415, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1488, 1514, 1520, 1522, 1552, 1562, 1568, 1641, 1646, 1747, 1749, 1756, 1759, 1768, 1770, 1788, 1791, 1791, 1808, 1866, 1869, 1969, 1984, 2037, 2042, 2042, 2048, 2093, 2112, 2139, 2208, 2208, 2210, 2220, 2276, 2302, 2304, 2403, 2406, 2415, 2417, 2423, 2425, 2431, 2433, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504, 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2561, 2563, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2649, 2652, 2654, 2654, 2662, 2677, 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2799, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2876, 2884, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2915, 2918, 2927, 2929, 2929, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024, 3031, 3031, 3046, 3055, 3073, 3075, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3133, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3160, 3161, 3168, 3171, 3174, 3183, 3202, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3299, 3302, 3311, 3313, 3314, 3330, 3331, 3333, 3340, 3342, 3344, 3346, 3386, 3389, 3396, 3398, 3400, 3402, 3406, 3415, 3415, 3424, 3427, 3430, 3439, 3450, 3455, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3769, 3771, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3807, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3948, 3953, 3972, 3974, 3991, 3993, 4028, 4038, 4038, 4096, 4169, 4176, 4253, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4957, 4959, 4992, 5007, 5024, 5108, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5872, 5888, 5900, 5902, 5908, 5920, 5940, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6099, 6103, 6103, 6108, 6109, 6112, 6121, 6155, 6157, 6160, 6169, 6176, 6263, 6272, 6314, 6320, 6389, 6400, 6428, 6432, 6443, 6448, 6459, 6470, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6617, 6656, 6683, 6688, 6750, 6752, 6780, 6783, 6793, 6800, 6809, 6823, 6823, 6912, 6987, 6992, 7001, 7019, 7027, 7040, 7155, 7168, 7223, 7232, 7241, 7245, 7293, 7376, 7378, 7380, 7414, 7424, 7654, 7676, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8204, 8205, 8255, 8256, 8276, 8276, 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8412, 8417, 8417, 8421, 8432, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11647, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11744, 11775, 11823, 11823, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12348, 12353, 12438, 12441, 12442, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12589, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40908, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42539, 42560, 42607, 42612, 42621, 42623, 42647, 42655, 42737, 42775, 42783, 42786, 42888, 42891, 42894, 42896, 42899, 42912, 42922, 43000, 43047, 43072, 43123, 43136, 43204, 43216, 43225, 43232, 43255, 43259, 43259, 43264, 43309, 43312, 43347, 43360, 43388, 43392, 43456, 43471, 43481, 43520, 43574, 43584, 43597, 43600, 43609, 43616, 43638, 43642, 43643, 43648, 43714, 43739, 43741, 43744, 43759, 43762, 43766, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43968, 44010, 44012, 44013, 44016, 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65024, 65039, 65056, 65062, 65075, 65076, 65101, 65103, 65136, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 282 283 /** 284 * Generated by scripts/regenerate-unicode-identifier-parts.js on node v12.4.0 with unicode 12.1 285 * based on http://www.unicode.org/reports/tr31/ and https://www.ecma-international.org/ecma-262/6.0/#sec-names-and-keywords 286 * unicodeESNextIdentifierStart corresponds to the ID_Start and Other_ID_Start property, and 287 * unicodeESNextIdentifierPart corresponds to ID_Continue, Other_ID_Continue, plus ID_Start and Other_ID_Start 288 */ 289 const unicodeESNextIdentifierStart = [65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2208, 2228, 2230, 2237, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3294, 3294, 3296, 3297, 3313, 3314, 3333, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5880, 5888, 5900, 5902, 5905, 5920, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6264, 6272, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6987, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7304, 7312, 7354, 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8472, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, 12443, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40943, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42653, 42656, 42735, 42775, 42783, 42786, 42888, 42891, 42943, 42946, 42950, 42999, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43879, 43888, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65856, 65908, 66176, 66204, 66208, 66256, 66304, 66335, 66349, 66378, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66513, 66517, 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 67072, 67382, 67392, 67413, 67424, 67431, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68096, 68112, 68115, 68117, 68119, 68121, 68149, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68324, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, 68800, 68850, 68864, 68899, 69376, 69404, 69415, 69415, 69424, 69445, 69600, 69622, 69635, 69687, 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480, 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70751, 70784, 70831, 70852, 70853, 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352, 71424, 71450, 71680, 71723, 71840, 71903, 71935, 71935, 72096, 72103, 72106, 72144, 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349, 72349, 72384, 72440, 72704, 72712, 72714, 72750, 72768, 72768, 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73728, 74649, 74752, 74862, 74880, 75075, 77824, 78894, 82944, 83526, 92160, 92728, 92736, 92766, 92880, 92909, 92928, 92975, 92992, 92995, 93027, 93047, 93053, 93071, 93760, 93823, 93952, 94026, 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101106, 110592, 110878, 110928, 110930, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, 123136, 123180, 123191, 123197, 123214, 123214, 123584, 123627, 124928, 125124, 125184, 125251, 125259, 125259, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173782, 173824, 177972, 177984, 178205, 178208, 183969, 183984, 191456, 194560, 195101]; 290 const unicodeESNextIdentifierPart = [48, 57, 65, 90, 95, 95, 97, 122, 170, 170, 181, 181, 183, 183, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 768, 884, 886, 887, 890, 893, 895, 895, 902, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1155, 1159, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1488, 1514, 1519, 1522, 1552, 1562, 1568, 1641, 1646, 1747, 1749, 1756, 1759, 1768, 1770, 1788, 1791, 1791, 1808, 1866, 1869, 1969, 1984, 2037, 2042, 2042, 2045, 2045, 2048, 2093, 2112, 2139, 2144, 2154, 2208, 2228, 2230, 2237, 2259, 2273, 2275, 2403, 2406, 2415, 2417, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504, 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2556, 2556, 2558, 2558, 2561, 2563, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2649, 2652, 2654, 2654, 2662, 2677, 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2799, 2809, 2815, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2876, 2884, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2915, 2918, 2927, 2929, 2929, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024, 3031, 3031, 3046, 3055, 3072, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3160, 3162, 3168, 3171, 3174, 3183, 3200, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3299, 3302, 3311, 3313, 3314, 3328, 3331, 3333, 3340, 3342, 3344, 3346, 3396, 3398, 3400, 3402, 3406, 3412, 3415, 3423, 3427, 3430, 3439, 3450, 3455, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3558, 3567, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3807, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3948, 3953, 3972, 3974, 3991, 3993, 4028, 4038, 4038, 4096, 4169, 4176, 4253, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4957, 4959, 4969, 4977, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5880, 5888, 5900, 5902, 5908, 5920, 5940, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6099, 6103, 6103, 6108, 6109, 6112, 6121, 6155, 6157, 6160, 6169, 6176, 6264, 6272, 6314, 6320, 6389, 6400, 6430, 6432, 6443, 6448, 6459, 6470, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6618, 6656, 6683, 6688, 6750, 6752, 6780, 6783, 6793, 6800, 6809, 6823, 6823, 6832, 6845, 6912, 6987, 6992, 7001, 7019, 7027, 7040, 7155, 7168, 7223, 7232, 7241, 7245, 7293, 7296, 7304, 7312, 7354, 7357, 7359, 7376, 7378, 7380, 7418, 7424, 7673, 7675, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8255, 8256, 8276, 8276, 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8412, 8417, 8417, 8421, 8432, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8472, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11647, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11744, 11775, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12348, 12353, 12438, 12441, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40943, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42539, 42560, 42607, 42612, 42621, 42623, 42737, 42775, 42783, 42786, 42888, 42891, 42943, 42946, 42950, 42999, 43047, 43072, 43123, 43136, 43205, 43216, 43225, 43232, 43255, 43259, 43259, 43261, 43309, 43312, 43347, 43360, 43388, 43392, 43456, 43471, 43481, 43488, 43518, 43520, 43574, 43584, 43597, 43600, 43609, 43616, 43638, 43642, 43714, 43739, 43741, 43744, 43759, 43762, 43766, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43879, 43888, 44010, 44012, 44013, 44016, 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65024, 65039, 65056, 65071, 65075, 65076, 65101, 65103, 65136, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65856, 65908, 66045, 66045, 66176, 66204, 66208, 66256, 66272, 66272, 66304, 66335, 66349, 66378, 66384, 66426, 66432, 66461, 66464, 66499, 66504, 66511, 66513, 66517, 66560, 66717, 66720, 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 67072, 67382, 67392, 67413, 67424, 67431, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68099, 68101, 68102, 68108, 68115, 68117, 68119, 68121, 68149, 68152, 68154, 68159, 68159, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68326, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, 68800, 68850, 68864, 68903, 68912, 68921, 69376, 69404, 69415, 69415, 69424, 69456, 69600, 69622, 69632, 69702, 69734, 69743, 69759, 69818, 69840, 69864, 69872, 69881, 69888, 69940, 69942, 69951, 69956, 69958, 69968, 70003, 70006, 70006, 70016, 70084, 70089, 70092, 70096, 70106, 70108, 70108, 70144, 70161, 70163, 70199, 70206, 70206, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70378, 70384, 70393, 70400, 70403, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70459, 70468, 70471, 70472, 70475, 70477, 70480, 70480, 70487, 70487, 70493, 70499, 70502, 70508, 70512, 70516, 70656, 70730, 70736, 70745, 70750, 70751, 70784, 70853, 70855, 70855, 70864, 70873, 71040, 71093, 71096, 71104, 71128, 71133, 71168, 71232, 71236, 71236, 71248, 71257, 71296, 71352, 71360, 71369, 71424, 71450, 71453, 71467, 71472, 71481, 71680, 71738, 71840, 71913, 71935, 71935, 72096, 72103, 72106, 72151, 72154, 72161, 72163, 72164, 72192, 72254, 72263, 72263, 72272, 72345, 72349, 72349, 72384, 72440, 72704, 72712, 72714, 72758, 72760, 72768, 72784, 72793, 72818, 72847, 72850, 72871, 72873, 72886, 72960, 72966, 72968, 72969, 72971, 73014, 73018, 73018, 73020, 73021, 73023, 73031, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73102, 73104, 73105, 73107, 73112, 73120, 73129, 73440, 73462, 73728, 74649, 74752, 74862, 74880, 75075, 77824, 78894, 82944, 83526, 92160, 92728, 92736, 92766, 92768, 92777, 92880, 92909, 92912, 92916, 92928, 92982, 92992, 92995, 93008, 93017, 93027, 93047, 93053, 93071, 93760, 93823, 93952, 94026, 94031, 94087, 94095, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101106, 110592, 110878, 110928, 110930, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 113821, 113822, 119141, 119145, 119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, 120782, 120831, 121344, 121398, 121403, 121452, 121461, 121461, 121476, 121476, 121499, 121503, 121505, 121519, 122880, 122886, 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 123136, 123180, 123184, 123197, 123200, 123209, 123214, 123214, 123584, 123641, 124928, 125124, 125136, 125142, 125184, 125259, 125264, 125273, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173782, 173824, 177972, 177984, 178205, 178208, 183969, 183984, 191456, 194560, 195101, 917760, 917999]; 291 292 /** 293 * Test for whether a single line comment with leading whitespace trimmed's text contains a directive. 294 */ 295 const commentDirectiveRegExSingleLine = /^\/\/\/?\s*@(ts-expect-error|ts-ignore)/; 296 297 /** 298 * Test for whether a multi-line comment with leading whitespace trimmed's last line contains a directive. 299 */ 300 const commentDirectiveRegExMultiLine = /^(?:\/|\*)*\s*@(ts-expect-error|ts-ignore)/; 301 302 function lookupInUnicodeMap(code: number, map: readonly number[]): boolean { 303 // Bail out quickly if it couldn't possibly be in the map. 304 if (code < map[0]) { 305 return false; 306 } 307 308 // Perform binary search in one of the Unicode range maps 309 let lo = 0; 310 let hi: number = map.length; 311 let mid: number; 312 313 while (lo + 1 < hi) { 314 mid = lo + (hi - lo) / 2; 315 // mid has to be even to catch a range's beginning 316 mid -= mid % 2; 317 if (map[mid] <= code && code <= map[mid + 1]) { 318 return true; 319 } 320 321 if (code < map[mid]) { 322 hi = mid; 323 } 324 else { 325 lo = mid + 2; 326 } 327 } 328 329 return false; 330 } 331 332 /* @internal */ export function isUnicodeIdentifierStart(code: number, languageVersion: ScriptTarget | undefined) { 333 return languageVersion! >= ScriptTarget.ES2015 ? 334 lookupInUnicodeMap(code, unicodeESNextIdentifierStart) : 335 languageVersion === ScriptTarget.ES5 ? lookupInUnicodeMap(code, unicodeES5IdentifierStart) : 336 lookupInUnicodeMap(code, unicodeES3IdentifierStart); 337 } 338 339 function isUnicodeIdentifierPart(code: number, languageVersion: ScriptTarget | undefined) { 340 return languageVersion! >= ScriptTarget.ES2015 ? 341 lookupInUnicodeMap(code, unicodeESNextIdentifierPart) : 342 languageVersion === ScriptTarget.ES5 ? lookupInUnicodeMap(code, unicodeES5IdentifierPart) : 343 lookupInUnicodeMap(code, unicodeES3IdentifierPart); 344 } 345 346 function makeReverseMap(source: ESMap<string, number>): string[] { 347 const result: string[] = []; 348 source.forEach((value, name) => { 349 result[value] = name; 350 }); 351 return result; 352 } 353 354 const tokenStrings = makeReverseMap(textToToken); 355 export function tokenToString(t: SyntaxKind): string | undefined { 356 return tokenStrings[t]; 357 } 358 359 /* @internal */ 360 export function stringToToken(s: string): SyntaxKind | undefined { 361 return textToToken.get(s); 362 } 363 364 /* @internal */ 365 export function computeLineStarts(text: string): number[] { 366 const result: number[] = []; 367 let pos = 0; 368 let lineStart = 0; 369 while (pos < text.length) { 370 const ch = text.charCodeAt(pos); 371 pos++; 372 switch (ch) { 373 case CharacterCodes.carriageReturn: 374 if (text.charCodeAt(pos) === CharacterCodes.lineFeed) { 375 pos++; 376 } 377 // falls through 378 case CharacterCodes.lineFeed: 379 result.push(lineStart); 380 lineStart = pos; 381 break; 382 default: 383 if (ch > CharacterCodes.maxAsciiCharacter && isLineBreak(ch)) { 384 result.push(lineStart); 385 lineStart = pos; 386 } 387 break; 388 } 389 } 390 result.push(lineStart); 391 return result; 392 } 393 394 export function getPositionOfLineAndCharacter(sourceFile: SourceFileLike, line: number, character: number): number; 395 /* @internal */ 396 export function getPositionOfLineAndCharacter(sourceFile: SourceFileLike, line: number, character: number, allowEdits?: true): number; // eslint-disable-line @typescript-eslint/unified-signatures 397 export function getPositionOfLineAndCharacter(sourceFile: SourceFileLike, line: number, character: number, allowEdits?: true): number { 398 return sourceFile.getPositionOfLineAndCharacter ? 399 sourceFile.getPositionOfLineAndCharacter(line, character, allowEdits) : 400 computePositionOfLineAndCharacter(getLineStarts(sourceFile), line, character, sourceFile.text, allowEdits); 401 } 402 403 /* @internal */ 404 export function computePositionOfLineAndCharacter(lineStarts: readonly number[], line: number, character: number, debugText?: string, allowEdits?: true): number { 405 if (line < 0 || line >= lineStarts.length) { 406 if (allowEdits) { 407 // Clamp line to nearest allowable value 408 line = line < 0 ? 0 : line >= lineStarts.length ? lineStarts.length - 1 : line; 409 } 410 else { 411 Debug.fail(`Bad line number. Line: ${line}, lineStarts.length: ${lineStarts.length} , line map is correct? ${debugText !== undefined ? arraysEqual(lineStarts, computeLineStarts(debugText)) : "unknown"}`); 412 } 413 } 414 415 const res = lineStarts[line] + character; 416 if (allowEdits) { 417 // Clamp to nearest allowable values to allow the underlying to be edited without crashing (accuracy is lost, instead) 418 // TODO: Somehow track edits between file as it was during the creation of sourcemap we have and the current file and 419 // apply them to the computed position to improve accuracy 420 return res > lineStarts[line + 1] ? lineStarts[line + 1] : typeof debugText === "string" && res > debugText.length ? debugText.length : res; 421 } 422 if (line < lineStarts.length - 1) { 423 Debug.assert(res < lineStarts[line + 1]); 424 } 425 else if (debugText !== undefined) { 426 Debug.assert(res <= debugText.length); // Allow single character overflow for trailing newline 427 } 428 return res; 429 } 430 431 /* @internal */ 432 export function getLineStarts(sourceFile: SourceFileLike): readonly number[] { 433 return sourceFile.lineMap || (sourceFile.lineMap = computeLineStarts(sourceFile.text)); 434 } 435 436 /* @internal */ 437 export function computeLineAndCharacterOfPosition(lineStarts: readonly number[], position: number): LineAndCharacter { 438 const lineNumber = computeLineOfPosition(lineStarts, position); 439 return { 440 line: lineNumber, 441 character: position - lineStarts[lineNumber] 442 }; 443 } 444 445 /** 446 * @internal 447 * We assume the first line starts at position 0 and 'position' is non-negative. 448 */ 449 export function computeLineOfPosition(lineStarts: readonly number[], position: number, lowerBound?: number) { 450 let lineNumber = binarySearch(lineStarts, position, identity, compareValues, lowerBound); 451 if (lineNumber < 0) { 452 // If the actual position was not found, 453 // the binary search returns the 2's-complement of the next line start 454 // e.g. if the line starts at [5, 10, 23, 80] and the position requested was 20 455 // then the search will return -2. 456 // 457 // We want the index of the previous line start, so we subtract 1. 458 // Review 2's-complement if this is confusing. 459 lineNumber = ~lineNumber - 1; 460 Debug.assert(lineNumber !== -1, "position cannot precede the beginning of the file"); 461 } 462 return lineNumber; 463 } 464 465 /** @internal */ 466 export function getLinesBetweenPositions(sourceFile: SourceFileLike, pos1: number, pos2: number) { 467 if (pos1 === pos2) return 0; 468 const lineStarts = getLineStarts(sourceFile); 469 const lower = Math.min(pos1, pos2); 470 const isNegative = lower === pos2; 471 const upper = isNegative ? pos1 : pos2; 472 const lowerLine = computeLineOfPosition(lineStarts, lower); 473 const upperLine = computeLineOfPosition(lineStarts, upper, lowerLine); 474 return isNegative ? lowerLine - upperLine : upperLine - lowerLine; 475 } 476 477 export function getLineAndCharacterOfPosition(sourceFile: SourceFileLike, position: number): LineAndCharacter { 478 return computeLineAndCharacterOfPosition(getLineStarts(sourceFile), position); 479 } 480 481 export function isWhiteSpaceLike(ch: number): boolean { 482 return isWhiteSpaceSingleLine(ch) || isLineBreak(ch); 483 } 484 485 /** Does not include line breaks. For that, see isWhiteSpaceLike. */ 486 export function isWhiteSpaceSingleLine(ch: number): boolean { 487 // Note: nextLine is in the Zs space, and should be considered to be a whitespace. 488 // It is explicitly not a line-break as it isn't in the exact set specified by EcmaScript. 489 return ch === CharacterCodes.space || 490 ch === CharacterCodes.tab || 491 ch === CharacterCodes.verticalTab || 492 ch === CharacterCodes.formFeed || 493 ch === CharacterCodes.nonBreakingSpace || 494 ch === CharacterCodes.nextLine || 495 ch === CharacterCodes.ogham || 496 ch >= CharacterCodes.enQuad && ch <= CharacterCodes.zeroWidthSpace || 497 ch === CharacterCodes.narrowNoBreakSpace || 498 ch === CharacterCodes.mathematicalSpace || 499 ch === CharacterCodes.ideographicSpace || 500 ch === CharacterCodes.byteOrderMark; 501 } 502 503 export function isLineBreak(ch: number): boolean { 504 // ES5 7.3: 505 // The ECMAScript line terminator characters are listed in Table 3. 506 // Table 3: Line Terminator Characters 507 // Code Unit Value Name Formal Name 508 // \u000A Line Feed <LF> 509 // \u000D Carriage Return <CR> 510 // \u2028 Line separator <LS> 511 // \u2029 Paragraph separator <PS> 512 // Only the characters in Table 3 are treated as line terminators. Other new line or line 513 // breaking characters are treated as white space but not as line terminators. 514 515 return ch === CharacterCodes.lineFeed || 516 ch === CharacterCodes.carriageReturn || 517 ch === CharacterCodes.lineSeparator || 518 ch === CharacterCodes.paragraphSeparator; 519 } 520 521 function isDigit(ch: number): boolean { 522 return ch >= CharacterCodes._0 && ch <= CharacterCodes._9; 523 } 524 525 function isHexDigit(ch: number): boolean { 526 return isDigit(ch) || ch >= CharacterCodes.A && ch <= CharacterCodes.F || ch >= CharacterCodes.a && ch <= CharacterCodes.f; 527 } 528 529 function isCodePoint(code: number): boolean { 530 return code <= 0x10FFFF; 531 } 532 533 /* @internal */ 534 export function isOctalDigit(ch: number): boolean { 535 return ch >= CharacterCodes._0 && ch <= CharacterCodes._7; 536 } 537 538 export function couldStartTrivia(text: string, pos: number): boolean { 539 // Keep in sync with skipTrivia 540 const ch = text.charCodeAt(pos); 541 switch (ch) { 542 case CharacterCodes.carriageReturn: 543 case CharacterCodes.lineFeed: 544 case CharacterCodes.tab: 545 case CharacterCodes.verticalTab: 546 case CharacterCodes.formFeed: 547 case CharacterCodes.space: 548 case CharacterCodes.slash: 549 // starts of normal trivia 550 // falls through 551 case CharacterCodes.lessThan: 552 case CharacterCodes.bar: 553 case CharacterCodes.equals: 554 case CharacterCodes.greaterThan: 555 // Starts of conflict marker trivia 556 return true; 557 case CharacterCodes.hash: 558 // Only if its the beginning can we have #! trivia 559 return pos === 0; 560 default: 561 return ch > CharacterCodes.maxAsciiCharacter; 562 } 563 } 564 565 /* @internal */ 566 export function skipTrivia(text: string, pos: number, stopAfterLineBreak?: boolean, stopAtComments?: boolean, inJSDoc?: boolean): number { 567 if (positionIsSynthesized(pos)) { 568 return pos; 569 } 570 571 let canConsumeStar = false; 572 // Keep in sync with couldStartTrivia 573 while (true) { 574 const ch = text.charCodeAt(pos); 575 switch (ch) { 576 case CharacterCodes.carriageReturn: 577 if (text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { 578 pos++; 579 } 580 // falls through 581 case CharacterCodes.lineFeed: 582 pos++; 583 if (stopAfterLineBreak) { 584 return pos; 585 } 586 canConsumeStar = !!inJSDoc; 587 continue; 588 case CharacterCodes.tab: 589 case CharacterCodes.verticalTab: 590 case CharacterCodes.formFeed: 591 case CharacterCodes.space: 592 pos++; 593 continue; 594 case CharacterCodes.slash: 595 if (stopAtComments) { 596 break; 597 } 598 if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { 599 pos += 2; 600 while (pos < text.length) { 601 if (isLineBreak(text.charCodeAt(pos))) { 602 break; 603 } 604 pos++; 605 } 606 canConsumeStar = false; 607 continue; 608 } 609 if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { 610 pos += 2; 611 while (pos < text.length) { 612 if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { 613 pos += 2; 614 break; 615 } 616 pos++; 617 } 618 canConsumeStar = false; 619 continue; 620 } 621 break; 622 623 case CharacterCodes.lessThan: 624 case CharacterCodes.bar: 625 case CharacterCodes.equals: 626 case CharacterCodes.greaterThan: 627 if (isConflictMarkerTrivia(text, pos)) { 628 pos = scanConflictMarkerTrivia(text, pos); 629 canConsumeStar = false; 630 continue; 631 } 632 break; 633 634 case CharacterCodes.hash: 635 if (pos === 0 && isShebangTrivia(text, pos)) { 636 pos = scanShebangTrivia(text, pos); 637 canConsumeStar = false; 638 continue; 639 } 640 break; 641 642 case CharacterCodes.asterisk: 643 if (canConsumeStar) { 644 pos++; 645 canConsumeStar = false; 646 continue; 647 } 648 break; 649 650 default: 651 if (ch > CharacterCodes.maxAsciiCharacter && (isWhiteSpaceLike(ch))) { 652 pos++; 653 continue; 654 } 655 break; 656 } 657 return pos; 658 } 659 } 660 661 // All conflict markers consist of the same character repeated seven times. If it is 662 // a <<<<<<< or >>>>>>> marker then it is also followed by a space. 663 const mergeConflictMarkerLength = "<<<<<<<".length; 664 665 function isConflictMarkerTrivia(text: string, pos: number) { 666 Debug.assert(pos >= 0); 667 668 // Conflict markers must be at the start of a line. 669 if (pos === 0 || isLineBreak(text.charCodeAt(pos - 1))) { 670 const ch = text.charCodeAt(pos); 671 672 if ((pos + mergeConflictMarkerLength) < text.length) { 673 for (let i = 0; i < mergeConflictMarkerLength; i++) { 674 if (text.charCodeAt(pos + i) !== ch) { 675 return false; 676 } 677 } 678 679 return ch === CharacterCodes.equals || 680 text.charCodeAt(pos + mergeConflictMarkerLength) === CharacterCodes.space; 681 } 682 } 683 684 return false; 685 } 686 687 function scanConflictMarkerTrivia(text: string, pos: number, error?: (diag: DiagnosticMessage, pos?: number, len?: number) => void) { 688 if (error) { 689 error(Diagnostics.Merge_conflict_marker_encountered, pos, mergeConflictMarkerLength); 690 } 691 692 const ch = text.charCodeAt(pos); 693 const len = text.length; 694 695 if (ch === CharacterCodes.lessThan || ch === CharacterCodes.greaterThan) { 696 while (pos < len && !isLineBreak(text.charCodeAt(pos))) { 697 pos++; 698 } 699 } 700 else { 701 Debug.assert(ch === CharacterCodes.bar || ch === CharacterCodes.equals); 702 // Consume everything from the start of a ||||||| or ======= marker to the start 703 // of the next ======= or >>>>>>> marker. 704 while (pos < len) { 705 const currentChar = text.charCodeAt(pos); 706 if ((currentChar === CharacterCodes.equals || currentChar === CharacterCodes.greaterThan) && currentChar !== ch && isConflictMarkerTrivia(text, pos)) { 707 break; 708 } 709 710 pos++; 711 } 712 } 713 714 return pos; 715 } 716 717 const shebangTriviaRegex = /^#!.*/; 718 719 /*@internal*/ 720 export function isShebangTrivia(text: string, pos: number) { 721 // Shebangs check must only be done at the start of the file 722 Debug.assert(pos === 0); 723 return shebangTriviaRegex.test(text); 724 } 725 726 /*@internal*/ 727 export function scanShebangTrivia(text: string, pos: number) { 728 const shebang = shebangTriviaRegex.exec(text)![0]; 729 pos = pos + shebang.length; 730 return pos; 731 } 732 733 /** 734 * Invokes a callback for each comment range following the provided position. 735 * 736 * Single-line comment ranges include the leading double-slash characters but not the ending 737 * line break. Multi-line comment ranges include the leading slash-asterisk and trailing 738 * asterisk-slash characters. 739 * 740 * @param reduce If true, accumulates the result of calling the callback in a fashion similar 741 * to reduceLeft. If false, iteration stops when the callback returns a truthy value. 742 * @param text The source text to scan. 743 * @param pos The position at which to start scanning. 744 * @param trailing If false, whitespace is skipped until the first line break and comments 745 * between that location and the next token are returned. If true, comments occurring 746 * between the given position and the next line break are returned. 747 * @param cb The callback to execute as each comment range is encountered. 748 * @param state A state value to pass to each iteration of the callback. 749 * @param initial An initial value to pass when accumulating results (when "reduce" is true). 750 * @returns If "reduce" is true, the accumulated value. If "reduce" is false, the first truthy 751 * return value of the callback. 752 */ 753 function iterateCommentRanges<T, U>(reduce: boolean, text: string, pos: number, trailing: boolean, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U | undefined) => U, state: T, initial?: U): U | undefined { 754 let pendingPos!: number; 755 let pendingEnd!: number; 756 let pendingKind!: CommentKind; 757 let pendingHasTrailingNewLine!: boolean; 758 let hasPendingCommentRange = false; 759 let collecting = trailing; 760 let accumulator = initial; 761 if (pos === 0) { 762 collecting = true; 763 const shebang = getShebang(text); 764 if (shebang) { 765 pos = shebang.length; 766 } 767 } 768 scan: while (pos >= 0 && pos < text.length) { 769 const ch = text.charCodeAt(pos); 770 switch (ch) { 771 case CharacterCodes.carriageReturn: 772 if (text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { 773 pos++; 774 } 775 // falls through 776 case CharacterCodes.lineFeed: 777 pos++; 778 if (trailing) { 779 break scan; 780 } 781 782 collecting = true; 783 if (hasPendingCommentRange) { 784 pendingHasTrailingNewLine = true; 785 } 786 787 continue; 788 case CharacterCodes.tab: 789 case CharacterCodes.verticalTab: 790 case CharacterCodes.formFeed: 791 case CharacterCodes.space: 792 pos++; 793 continue; 794 case CharacterCodes.slash: 795 const nextChar = text.charCodeAt(pos + 1); 796 let hasTrailingNewLine = false; 797 if (nextChar === CharacterCodes.slash || nextChar === CharacterCodes.asterisk) { 798 const kind = nextChar === CharacterCodes.slash ? SyntaxKind.SingleLineCommentTrivia : SyntaxKind.MultiLineCommentTrivia; 799 const startPos = pos; 800 pos += 2; 801 if (nextChar === CharacterCodes.slash) { 802 while (pos < text.length) { 803 if (isLineBreak(text.charCodeAt(pos))) { 804 hasTrailingNewLine = true; 805 break; 806 } 807 pos++; 808 } 809 } 810 else { 811 while (pos < text.length) { 812 if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { 813 pos += 2; 814 break; 815 } 816 pos++; 817 } 818 } 819 820 if (collecting) { 821 if (hasPendingCommentRange) { 822 accumulator = cb(pendingPos, pendingEnd, pendingKind, pendingHasTrailingNewLine, state, accumulator); 823 if (!reduce && accumulator) { 824 // If we are not reducing and we have a truthy result, return it. 825 return accumulator; 826 } 827 } 828 829 pendingPos = startPos; 830 pendingEnd = pos; 831 pendingKind = kind; 832 pendingHasTrailingNewLine = hasTrailingNewLine; 833 hasPendingCommentRange = true; 834 } 835 836 continue; 837 } 838 break scan; 839 default: 840 if (ch > CharacterCodes.maxAsciiCharacter && (isWhiteSpaceLike(ch))) { 841 if (hasPendingCommentRange && isLineBreak(ch)) { 842 pendingHasTrailingNewLine = true; 843 } 844 pos++; 845 continue; 846 } 847 break scan; 848 } 849 } 850 851 if (hasPendingCommentRange) { 852 accumulator = cb(pendingPos, pendingEnd, pendingKind, pendingHasTrailingNewLine, state, accumulator); 853 } 854 855 return accumulator; 856 } 857 858 export function forEachLeadingCommentRange<U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean) => U): U | undefined; 859 export function forEachLeadingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state: T): U | undefined; 860 export function forEachLeadingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state?: T): U | undefined { 861 return iterateCommentRanges(/*reduce*/ false, text, pos, /*trailing*/ false, cb, state); 862 } 863 864 export function forEachTrailingCommentRange<U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean) => U): U | undefined; 865 export function forEachTrailingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state: T): U | undefined; 866 export function forEachTrailingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state?: T): U | undefined { 867 return iterateCommentRanges(/*reduce*/ false, text, pos, /*trailing*/ true, cb, state); 868 } 869 870 export function reduceEachLeadingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { 871 return iterateCommentRanges(/*reduce*/ true, text, pos, /*trailing*/ false, cb, state, initial); 872 } 873 874 export function reduceEachTrailingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { 875 return iterateCommentRanges(/*reduce*/ true, text, pos, /*trailing*/ true, cb, state, initial); 876 } 877 878 function appendCommentRange(pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, _state: any, comments: CommentRange[]) { 879 if (!comments) { 880 comments = []; 881 } 882 883 comments.push({ kind, pos, end, hasTrailingNewLine }); 884 return comments; 885 } 886 887 export function getLeadingCommentRanges(text: string, pos: number): CommentRange[] | undefined { 888 return reduceEachLeadingCommentRange(text, pos, appendCommentRange, /*state*/ undefined, /*initial*/ undefined); 889 } 890 891 export function getTrailingCommentRanges(text: string, pos: number): CommentRange[] | undefined { 892 return reduceEachTrailingCommentRange(text, pos, appendCommentRange, /*state*/ undefined, /*initial*/ undefined); 893 } 894 895 /** Optionally, get the shebang */ 896 export function getShebang(text: string): string | undefined { 897 const match = shebangTriviaRegex.exec(text); 898 if (match) { 899 return match[0]; 900 } 901 } 902 903 export function isIdentifierStart(ch: number, languageVersion: ScriptTarget | undefined): boolean { 904 return ch >= CharacterCodes.A && ch <= CharacterCodes.Z || ch >= CharacterCodes.a && ch <= CharacterCodes.z || 905 ch === CharacterCodes.$ || ch === CharacterCodes._ || 906 ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierStart(ch, languageVersion); 907 } 908 909 export function isIdentifierPart(ch: number, languageVersion: ScriptTarget | undefined, identifierVariant?: LanguageVariant): boolean { 910 return ch >= CharacterCodes.A && ch <= CharacterCodes.Z || ch >= CharacterCodes.a && ch <= CharacterCodes.z || 911 ch >= CharacterCodes._0 && ch <= CharacterCodes._9 || ch === CharacterCodes.$ || ch === CharacterCodes._ || 912 // "-" and ":" are valid in JSX Identifiers 913 (identifierVariant === LanguageVariant.JSX ? (ch === CharacterCodes.minus || ch === CharacterCodes.colon) : false) || 914 ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion); 915 } 916 917 /* @internal */ 918 export function isIdentifierText(name: string, languageVersion: ScriptTarget | undefined, identifierVariant?: LanguageVariant): boolean { 919 let ch = codePointAt(name, 0); 920 if (!isIdentifierStart(ch, languageVersion)) { 921 return false; 922 } 923 924 for (let i = charSize(ch); i < name.length; i += charSize(ch)) { 925 if (!isIdentifierPart(ch = codePointAt(name, i), languageVersion, identifierVariant)) { 926 return false; 927 } 928 } 929 930 return true; 931 } 932 933 // Creates a scanner over a (possibly unspecified) range of a piece of text. 934 export function createScanner(languageVersion: ScriptTarget, 935 skipTrivia: boolean, 936 languageVariant = LanguageVariant.Standard, 937 textInitial?: string, 938 onError?: ErrorCallback, 939 start?: number, 940 length?: number): Scanner { 941 942 let text = textInitial!; 943 944 // Current position (end position of text of current token) 945 let pos: number; 946 947 948 // end of text 949 let end: number; 950 951 // Start position of whitespace before current token 952 let startPos: number; 953 954 // Start position of text of current token 955 let tokenPos: number; 956 957 let token: SyntaxKind; 958 let tokenValue!: string; 959 let tokenFlags: TokenFlags; 960 961 let commentDirectives: CommentDirective[] | undefined; 962 let inJSDocType = 0; 963 let inEtsContext: boolean = false; 964 965 setText(text, start, length); 966 967 const scanner: Scanner = { 968 getStartPos: () => startPos, 969 getTextPos: () => pos, 970 getToken: () => token, 971 getTokenPos: () => tokenPos, 972 getTokenText: () => text.substring(tokenPos, pos), 973 getTokenValue: () => tokenValue, 974 hasUnicodeEscape: () => (tokenFlags & TokenFlags.UnicodeEscape) !== 0, 975 hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0, 976 hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0, 977 hasPrecedingJSDocComment: () => (tokenFlags & TokenFlags.PrecedingJSDocComment) !== 0, 978 isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord, 979 isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord, 980 isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0, 981 getCommentDirectives: () => commentDirectives, 982 getNumericLiteralFlags: () => tokenFlags & TokenFlags.NumericLiteralFlags, 983 getTokenFlags: () => tokenFlags, 984 reScanGreaterToken, 985 reScanAsteriskEqualsToken, 986 reScanSlashToken, 987 reScanTemplateToken, 988 reScanTemplateHeadOrNoSubstitutionTemplate, 989 scanJsxIdentifier, 990 scanJsxAttributeValue, 991 reScanJsxAttributeValue, 992 reScanJsxToken, 993 reScanLessThanToken, 994 reScanHashToken, 995 reScanQuestionToken, 996 reScanInvalidIdentifier, 997 scanJsxToken, 998 scanJsDocToken, 999 scan, 1000 getText, 1001 clearCommentDirectives, 1002 setText, 1003 setScriptTarget, 1004 setLanguageVariant, 1005 setOnError, 1006 setTextPos, 1007 setInJSDocType, 1008 tryScan, 1009 lookAhead, 1010 scanRange, 1011 setEtsContext 1012 }; 1013 1014 if (Debug.isDebugging) { 1015 Object.defineProperty(scanner, "__debugShowCurrentPositionInText", { 1016 get: () => { 1017 const text = scanner.getText(); 1018 return text.slice(0, scanner.getStartPos()) + "║" + text.slice(scanner.getStartPos()); 1019 }, 1020 }); 1021 } 1022 1023 return scanner; 1024 1025 function setEtsContext(isEtsContext: boolean): void { 1026 inEtsContext = isEtsContext; 1027 } 1028 1029 function error(message: DiagnosticMessage): void; 1030 function error(message: DiagnosticMessage, errPos: number, length: number): void; 1031 function error(message: DiagnosticMessage, errPos: number = pos, length?: number): void { 1032 if (onError) { 1033 const oldPos = pos; 1034 pos = errPos; 1035 onError(message, length || 0); 1036 pos = oldPos; 1037 } 1038 } 1039 1040 function scanNumberFragment(): string { 1041 let start = pos; 1042 let allowSeparator = false; 1043 let isPreviousTokenSeparator = false; 1044 let result = ""; 1045 while (true) { 1046 const ch = text.charCodeAt(pos); 1047 if (ch === CharacterCodes._) { 1048 tokenFlags |= TokenFlags.ContainsSeparator; 1049 if (allowSeparator) { 1050 allowSeparator = false; 1051 isPreviousTokenSeparator = true; 1052 result += text.substring(start, pos); 1053 } 1054 else if (isPreviousTokenSeparator) { 1055 error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1); 1056 } 1057 else { 1058 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1); 1059 } 1060 pos++; 1061 start = pos; 1062 continue; 1063 } 1064 if (isDigit(ch)) { 1065 allowSeparator = true; 1066 isPreviousTokenSeparator = false; 1067 pos++; 1068 continue; 1069 } 1070 break; 1071 } 1072 if (text.charCodeAt(pos - 1) === CharacterCodes._) { 1073 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1); 1074 } 1075 return result + text.substring(start, pos); 1076 } 1077 1078 function scanNumber(): { type: SyntaxKind, value: string } { 1079 const start = pos; 1080 const mainFragment = scanNumberFragment(); 1081 let decimalFragment: string | undefined; 1082 let scientificFragment: string | undefined; 1083 if (text.charCodeAt(pos) === CharacterCodes.dot) { 1084 pos++; 1085 decimalFragment = scanNumberFragment(); 1086 } 1087 let end = pos; 1088 if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) { 1089 pos++; 1090 tokenFlags |= TokenFlags.Scientific; 1091 if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++; 1092 const preNumericPart = pos; 1093 const finalFragment = scanNumberFragment(); 1094 if (!finalFragment) { 1095 error(Diagnostics.Digit_expected); 1096 } 1097 else { 1098 scientificFragment = text.substring(end, preNumericPart) + finalFragment; 1099 end = pos; 1100 } 1101 } 1102 let result: string; 1103 if (tokenFlags & TokenFlags.ContainsSeparator) { 1104 result = mainFragment; 1105 if (decimalFragment) { 1106 result += "." + decimalFragment; 1107 } 1108 if (scientificFragment) { 1109 result += scientificFragment; 1110 } 1111 } 1112 else { 1113 result = text.substring(start, end); // No need to use all the fragments; no _ removal needed 1114 } 1115 1116 if (decimalFragment !== undefined || tokenFlags & TokenFlags.Scientific) { 1117 checkForIdentifierStartAfterNumericLiteral(start, decimalFragment === undefined && !!(tokenFlags & TokenFlags.Scientific)); 1118 return { 1119 type: SyntaxKind.NumericLiteral, 1120 value: "" + +result // if value is not an integer, it can be safely coerced to a number 1121 }; 1122 } 1123 else { 1124 tokenValue = result; 1125 const type = checkBigIntSuffix(); // if value is an integer, check whether it is a bigint 1126 checkForIdentifierStartAfterNumericLiteral(start); 1127 return { type, value: tokenValue }; 1128 } 1129 } 1130 1131 function checkForIdentifierStartAfterNumericLiteral(numericStart: number, isScientific?: boolean) { 1132 if (!isIdentifierStart(codePointAt(text, pos), languageVersion)) { 1133 return; 1134 } 1135 1136 const identifierStart = pos; 1137 const { length } = scanIdentifierParts(); 1138 1139 if (length === 1 && text[identifierStart] === "n") { 1140 if (isScientific) { 1141 error(Diagnostics.A_bigint_literal_cannot_use_exponential_notation, numericStart, identifierStart - numericStart + 1); 1142 } 1143 else { 1144 error(Diagnostics.A_bigint_literal_must_be_an_integer, numericStart, identifierStart - numericStart + 1); 1145 } 1146 } 1147 else { 1148 error(Diagnostics.An_identifier_or_keyword_cannot_immediately_follow_a_numeric_literal, identifierStart, length); 1149 pos = identifierStart; 1150 } 1151 } 1152 1153 function scanOctalDigits(): number { 1154 const start = pos; 1155 while (isOctalDigit(text.charCodeAt(pos))) { 1156 pos++; 1157 } 1158 return +(text.substring(start, pos)); 1159 } 1160 1161 /** 1162 * Scans the given number of hexadecimal digits in the text, 1163 * returning -1 if the given number is unavailable. 1164 */ 1165 function scanExactNumberOfHexDigits(count: number, canHaveSeparators: boolean): number { 1166 const valueString = scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ false, canHaveSeparators); 1167 return valueString ? parseInt(valueString, 16) : -1; 1168 } 1169 1170 /** 1171 * Scans as many hexadecimal digits as are available in the text, 1172 * returning "" if the given number of digits was unavailable. 1173 */ 1174 function scanMinimumNumberOfHexDigits(count: number, canHaveSeparators: boolean): string { 1175 return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ true, canHaveSeparators); 1176 } 1177 1178 function scanHexDigits(minCount: number, scanAsManyAsPossible: boolean, canHaveSeparators: boolean): string { 1179 let valueChars: number[] = []; 1180 let allowSeparator = false; 1181 let isPreviousTokenSeparator = false; 1182 while (valueChars.length < minCount || scanAsManyAsPossible) { 1183 let ch = text.charCodeAt(pos); 1184 if (canHaveSeparators && ch === CharacterCodes._) { 1185 tokenFlags |= TokenFlags.ContainsSeparator; 1186 if (allowSeparator) { 1187 allowSeparator = false; 1188 isPreviousTokenSeparator = true; 1189 } 1190 else if (isPreviousTokenSeparator) { 1191 error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1); 1192 } 1193 else { 1194 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1); 1195 } 1196 pos++; 1197 continue; 1198 } 1199 allowSeparator = canHaveSeparators; 1200 if (ch >= CharacterCodes.A && ch <= CharacterCodes.F) { 1201 ch += CharacterCodes.a - CharacterCodes.A; // standardize hex literals to lowercase 1202 } 1203 else if (!((ch >= CharacterCodes._0 && ch <= CharacterCodes._9) || 1204 (ch >= CharacterCodes.a && ch <= CharacterCodes.f) 1205 )) { 1206 break; 1207 } 1208 valueChars.push(ch); 1209 pos++; 1210 isPreviousTokenSeparator = false; 1211 } 1212 if (valueChars.length < minCount) { 1213 valueChars = []; 1214 } 1215 if (text.charCodeAt(pos - 1) === CharacterCodes._) { 1216 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1); 1217 } 1218 return String.fromCharCode(...valueChars); 1219 } 1220 1221 function scanString(jsxAttributeString = false): string { 1222 const quote = text.charCodeAt(pos); 1223 pos++; 1224 let result = ""; 1225 let start = pos; 1226 while (true) { 1227 if (pos >= end) { 1228 result += text.substring(start, pos); 1229 tokenFlags |= TokenFlags.Unterminated; 1230 error(Diagnostics.Unterminated_string_literal); 1231 break; 1232 } 1233 const ch = text.charCodeAt(pos); 1234 if (ch === quote) { 1235 result += text.substring(start, pos); 1236 pos++; 1237 break; 1238 } 1239 if (ch === CharacterCodes.backslash && !jsxAttributeString) { 1240 result += text.substring(start, pos); 1241 result += scanEscapeSequence(); 1242 start = pos; 1243 continue; 1244 } 1245 if (isLineBreak(ch) && !jsxAttributeString) { 1246 result += text.substring(start, pos); 1247 tokenFlags |= TokenFlags.Unterminated; 1248 error(Diagnostics.Unterminated_string_literal); 1249 break; 1250 } 1251 pos++; 1252 } 1253 return result; 1254 } 1255 1256 /** 1257 * Sets the current 'tokenValue' and returns a NoSubstitutionTemplateLiteral or 1258 * a literal component of a TemplateExpression. 1259 */ 1260 function scanTemplateAndSetTokenValue(isTaggedTemplate: boolean): SyntaxKind { 1261 const startedWithBacktick = text.charCodeAt(pos) === CharacterCodes.backtick; 1262 1263 pos++; 1264 let start = pos; 1265 let contents = ""; 1266 let resultingToken: SyntaxKind; 1267 1268 while (true) { 1269 if (pos >= end) { 1270 contents += text.substring(start, pos); 1271 tokenFlags |= TokenFlags.Unterminated; 1272 error(Diagnostics.Unterminated_template_literal); 1273 resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail; 1274 break; 1275 } 1276 1277 const currChar = text.charCodeAt(pos); 1278 1279 // '`' 1280 if (currChar === CharacterCodes.backtick) { 1281 contents += text.substring(start, pos); 1282 pos++; 1283 resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail; 1284 break; 1285 } 1286 1287 // '${' 1288 if (currChar === CharacterCodes.$ && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.openBrace) { 1289 contents += text.substring(start, pos); 1290 pos += 2; 1291 resultingToken = startedWithBacktick ? SyntaxKind.TemplateHead : SyntaxKind.TemplateMiddle; 1292 break; 1293 } 1294 1295 // Escape character 1296 if (currChar === CharacterCodes.backslash) { 1297 contents += text.substring(start, pos); 1298 contents += scanEscapeSequence(isTaggedTemplate); 1299 start = pos; 1300 continue; 1301 } 1302 1303 // Speculated ECMAScript 6 Spec 11.8.6.1: 1304 // <CR><LF> and <CR> LineTerminatorSequences are normalized to <LF> for Template Values 1305 if (currChar === CharacterCodes.carriageReturn) { 1306 contents += text.substring(start, pos); 1307 pos++; 1308 1309 if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) { 1310 pos++; 1311 } 1312 1313 contents += "\n"; 1314 start = pos; 1315 continue; 1316 } 1317 1318 pos++; 1319 } 1320 1321 Debug.assert(resultingToken !== undefined); 1322 1323 tokenValue = contents; 1324 return resultingToken; 1325 } 1326 1327 function scanEscapeSequence(isTaggedTemplate?: boolean): string { 1328 const start = pos; 1329 pos++; 1330 if (pos >= end) { 1331 error(Diagnostics.Unexpected_end_of_text); 1332 return ""; 1333 } 1334 const ch = text.charCodeAt(pos); 1335 pos++; 1336 switch (ch) { 1337 case CharacterCodes._0: 1338 // '\01' 1339 if (isTaggedTemplate && pos < end && isDigit(text.charCodeAt(pos))) { 1340 pos++; 1341 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1342 return text.substring(start, pos); 1343 } 1344 return "\0"; 1345 case CharacterCodes.b: 1346 return "\b"; 1347 case CharacterCodes.t: 1348 return "\t"; 1349 case CharacterCodes.n: 1350 return "\n"; 1351 case CharacterCodes.v: 1352 return "\v"; 1353 case CharacterCodes.f: 1354 return "\f"; 1355 case CharacterCodes.r: 1356 return "\r"; 1357 case CharacterCodes.singleQuote: 1358 return "\'"; 1359 case CharacterCodes.doubleQuote: 1360 return "\""; 1361 case CharacterCodes.u: 1362 if (isTaggedTemplate) { 1363 // '\u' or '\u0' or '\u00' or '\u000' 1364 for (let escapePos = pos; escapePos < pos + 4; escapePos++) { 1365 if (escapePos < end && !isHexDigit(text.charCodeAt(escapePos)) && text.charCodeAt(escapePos) !== CharacterCodes.openBrace) { 1366 pos = escapePos; 1367 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1368 return text.substring(start, pos); 1369 } 1370 } 1371 } 1372 // '\u{DDDDDDDD}' 1373 if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) { 1374 pos++; 1375 1376 // '\u{' 1377 if (isTaggedTemplate && !isHexDigit(text.charCodeAt(pos))) { 1378 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1379 return text.substring(start, pos); 1380 } 1381 1382 if (isTaggedTemplate) { 1383 const savePos = pos; 1384 const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); 1385 const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1; 1386 1387 // '\u{Not Code Point' or '\u{CodePoint' 1388 if (!isCodePoint(escapedValue) || text.charCodeAt(pos) !== CharacterCodes.closeBrace) { 1389 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1390 return text.substring(start, pos); 1391 } 1392 else { 1393 pos = savePos; 1394 } 1395 } 1396 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 1397 return scanExtendedUnicodeEscape(); 1398 } 1399 1400 tokenFlags |= TokenFlags.UnicodeEscape; 1401 // '\uDDDD' 1402 return scanHexadecimalEscape(/*numDigits*/ 4); 1403 1404 case CharacterCodes.x: 1405 if (isTaggedTemplate) { 1406 if (!isHexDigit(text.charCodeAt(pos))) { 1407 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1408 return text.substring(start, pos); 1409 } 1410 else if (!isHexDigit(text.charCodeAt(pos + 1))) { 1411 pos++; 1412 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1413 return text.substring(start, pos); 1414 } 1415 } 1416 // '\xDD' 1417 return scanHexadecimalEscape(/*numDigits*/ 2); 1418 1419 // when encountering a LineContinuation (i.e. a backslash and a line terminator sequence), 1420 // the line terminator is interpreted to be "the empty code unit sequence". 1421 case CharacterCodes.carriageReturn: 1422 if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) { 1423 pos++; 1424 } 1425 // falls through 1426 case CharacterCodes.lineFeed: 1427 case CharacterCodes.lineSeparator: 1428 case CharacterCodes.paragraphSeparator: 1429 return ""; 1430 default: 1431 return String.fromCharCode(ch); 1432 } 1433 } 1434 1435 function scanHexadecimalEscape(numDigits: number): string { 1436 const escapedValue = scanExactNumberOfHexDigits(numDigits, /*canHaveSeparators*/ false); 1437 1438 if (escapedValue >= 0) { 1439 return String.fromCharCode(escapedValue); 1440 } 1441 else { 1442 error(Diagnostics.Hexadecimal_digit_expected); 1443 return ""; 1444 } 1445 } 1446 1447 function scanExtendedUnicodeEscape(): string { 1448 const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); 1449 const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1; 1450 let isInvalidExtendedEscape = false; 1451 1452 // Validate the value of the digit 1453 if (escapedValue < 0) { 1454 error(Diagnostics.Hexadecimal_digit_expected); 1455 isInvalidExtendedEscape = true; 1456 } 1457 else if (escapedValue > 0x10FFFF) { 1458 error(Diagnostics.An_extended_Unicode_escape_value_must_be_between_0x0_and_0x10FFFF_inclusive); 1459 isInvalidExtendedEscape = true; 1460 } 1461 1462 if (pos >= end) { 1463 error(Diagnostics.Unexpected_end_of_text); 1464 isInvalidExtendedEscape = true; 1465 } 1466 else if (text.charCodeAt(pos) === CharacterCodes.closeBrace) { 1467 // Only swallow the following character up if it's a '}'. 1468 pos++; 1469 } 1470 else { 1471 error(Diagnostics.Unterminated_Unicode_escape_sequence); 1472 isInvalidExtendedEscape = true; 1473 } 1474 1475 if (isInvalidExtendedEscape) { 1476 return ""; 1477 } 1478 1479 return utf16EncodeAsString(escapedValue); 1480 } 1481 1482 // Current character is known to be a backslash. Check for Unicode escape of the form '\uXXXX' 1483 // and return code point value if valid Unicode escape is found. Otherwise return -1. 1484 function peekUnicodeEscape(): number { 1485 if (pos + 5 < end && text.charCodeAt(pos + 1) === CharacterCodes.u) { 1486 const start = pos; 1487 pos += 2; 1488 const value = scanExactNumberOfHexDigits(4, /*canHaveSeparators*/ false); 1489 pos = start; 1490 return value; 1491 } 1492 return -1; 1493 } 1494 1495 1496 function peekExtendedUnicodeEscape(): number { 1497 if (codePointAt(text, pos + 1) === CharacterCodes.u && codePointAt(text, pos + 2) === CharacterCodes.openBrace) { 1498 const start = pos; 1499 pos += 3; 1500 const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); 1501 const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1; 1502 pos = start; 1503 return escapedValue; 1504 } 1505 return -1; 1506 } 1507 1508 function scanIdentifierParts(): string { 1509 let result = ""; 1510 let start = pos; 1511 while (pos < end) { 1512 let ch = codePointAt(text, pos); 1513 if (isIdentifierPart(ch, languageVersion)) { 1514 pos += charSize(ch); 1515 } 1516 else if (ch === CharacterCodes.backslash) { 1517 ch = peekExtendedUnicodeEscape(); 1518 if (ch >= 0 && isIdentifierPart(ch, languageVersion)) { 1519 pos += 3; 1520 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 1521 result += scanExtendedUnicodeEscape(); 1522 start = pos; 1523 continue; 1524 } 1525 ch = peekUnicodeEscape(); 1526 if (!(ch >= 0 && isIdentifierPart(ch, languageVersion))) { 1527 break; 1528 } 1529 tokenFlags |= TokenFlags.UnicodeEscape; 1530 result += text.substring(start, pos); 1531 result += utf16EncodeAsString(ch); 1532 // Valid Unicode escape is always six characters 1533 pos += 6; 1534 start = pos; 1535 } 1536 else { 1537 break; 1538 } 1539 } 1540 result += text.substring(start, pos); 1541 return result; 1542 } 1543 1544 function getIdentifierToken(): SyntaxKind.Identifier | KeywordSyntaxKind { 1545 // Reserved words are between 2 and 12 characters long and start with a lowercase letter 1546 const len = tokenValue.length; 1547 if (len >= 2 && len <= 12) { 1548 const ch = tokenValue.charCodeAt(0); 1549 if (ch >= CharacterCodes.a && ch <= CharacterCodes.z) { 1550 const keyword = textToKeyword.get(tokenValue); 1551 if (keyword !== undefined) { 1552 token = keyword; 1553 if (keyword === SyntaxKind.StructKeyword && !inEtsContext) { 1554 token = SyntaxKind.Identifier; 1555 } 1556 return token; 1557 } 1558 } 1559 } 1560 return token = SyntaxKind.Identifier; 1561 } 1562 1563 function scanBinaryOrOctalDigits(base: 2 | 8): string { 1564 let value = ""; 1565 // For counting number of digits; Valid binaryIntegerLiteral must have at least one binary digit following B or b. 1566 // Similarly valid octalIntegerLiteral must have at least one octal digit following o or O. 1567 let separatorAllowed = false; 1568 let isPreviousTokenSeparator = false; 1569 while (true) { 1570 const ch = text.charCodeAt(pos); 1571 // Numeric separators are allowed anywhere within a numeric literal, except not at the beginning, or following another separator 1572 if (ch === CharacterCodes._) { 1573 tokenFlags |= TokenFlags.ContainsSeparator; 1574 if (separatorAllowed) { 1575 separatorAllowed = false; 1576 isPreviousTokenSeparator = true; 1577 } 1578 else if (isPreviousTokenSeparator) { 1579 error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1); 1580 } 1581 else { 1582 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1); 1583 } 1584 pos++; 1585 continue; 1586 } 1587 separatorAllowed = true; 1588 if (!isDigit(ch) || ch - CharacterCodes._0 >= base) { 1589 break; 1590 } 1591 value += text[pos]; 1592 pos++; 1593 isPreviousTokenSeparator = false; 1594 } 1595 if (text.charCodeAt(pos - 1) === CharacterCodes._) { 1596 // Literal ends with underscore - not allowed 1597 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1); 1598 } 1599 return value; 1600 } 1601 1602 function checkBigIntSuffix(): SyntaxKind { 1603 if (text.charCodeAt(pos) === CharacterCodes.n) { 1604 tokenValue += "n"; 1605 // Use base 10 instead of base 2 or base 8 for shorter literals 1606 if (tokenFlags & TokenFlags.BinaryOrOctalSpecifier) { 1607 tokenValue = parsePseudoBigInt(tokenValue) + "n"; 1608 } 1609 pos++; 1610 return SyntaxKind.BigIntLiteral; 1611 } 1612 else { // not a bigint, so can convert to number in simplified form 1613 // Number() may not support 0b or 0o, so use parseInt() instead 1614 const numericValue = tokenFlags & TokenFlags.BinarySpecifier 1615 ? parseInt(tokenValue.slice(2), 2) // skip "0b" 1616 : tokenFlags & TokenFlags.OctalSpecifier 1617 ? parseInt(tokenValue.slice(2), 8) // skip "0o" 1618 : +tokenValue; 1619 tokenValue = "" + numericValue; 1620 return SyntaxKind.NumericLiteral; 1621 } 1622 } 1623 1624 function scan(): SyntaxKind { 1625 startPos = pos; 1626 tokenFlags = TokenFlags.None; 1627 let asteriskSeen = false; 1628 while (true) { 1629 tokenPos = pos; 1630 if (pos >= end) { 1631 return token = SyntaxKind.EndOfFileToken; 1632 } 1633 const ch = codePointAt(text, pos); 1634 1635 // Special handling for shebang 1636 if (ch === CharacterCodes.hash && pos === 0 && isShebangTrivia(text, pos)) { 1637 pos = scanShebangTrivia(text, pos); 1638 if (skipTrivia) { 1639 continue; 1640 } 1641 else { 1642 return token = SyntaxKind.ShebangTrivia; 1643 } 1644 } 1645 1646 switch (ch) { 1647 case CharacterCodes.lineFeed: 1648 case CharacterCodes.carriageReturn: 1649 tokenFlags |= TokenFlags.PrecedingLineBreak; 1650 if (skipTrivia) { 1651 pos++; 1652 continue; 1653 } 1654 else { 1655 if (ch === CharacterCodes.carriageReturn && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { 1656 // consume both CR and LF 1657 pos += 2; 1658 } 1659 else { 1660 pos++; 1661 } 1662 return token = SyntaxKind.NewLineTrivia; 1663 } 1664 case CharacterCodes.tab: 1665 case CharacterCodes.verticalTab: 1666 case CharacterCodes.formFeed: 1667 case CharacterCodes.space: 1668 case CharacterCodes.nonBreakingSpace: 1669 case CharacterCodes.ogham: 1670 case CharacterCodes.enQuad: 1671 case CharacterCodes.emQuad: 1672 case CharacterCodes.enSpace: 1673 case CharacterCodes.emSpace: 1674 case CharacterCodes.threePerEmSpace: 1675 case CharacterCodes.fourPerEmSpace: 1676 case CharacterCodes.sixPerEmSpace: 1677 case CharacterCodes.figureSpace: 1678 case CharacterCodes.punctuationSpace: 1679 case CharacterCodes.thinSpace: 1680 case CharacterCodes.hairSpace: 1681 case CharacterCodes.zeroWidthSpace: 1682 case CharacterCodes.narrowNoBreakSpace: 1683 case CharacterCodes.mathematicalSpace: 1684 case CharacterCodes.ideographicSpace: 1685 case CharacterCodes.byteOrderMark: 1686 if (skipTrivia) { 1687 pos++; 1688 continue; 1689 } 1690 else { 1691 while (pos < end && isWhiteSpaceSingleLine(text.charCodeAt(pos))) { 1692 pos++; 1693 } 1694 return token = SyntaxKind.WhitespaceTrivia; 1695 } 1696 case CharacterCodes.exclamation: 1697 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1698 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1699 return pos += 3, token = SyntaxKind.ExclamationEqualsEqualsToken; 1700 } 1701 return pos += 2, token = SyntaxKind.ExclamationEqualsToken; 1702 } 1703 pos++; 1704 return token = SyntaxKind.ExclamationToken; 1705 case CharacterCodes.doubleQuote: 1706 case CharacterCodes.singleQuote: 1707 tokenValue = scanString(); 1708 return token = SyntaxKind.StringLiteral; 1709 case CharacterCodes.backtick: 1710 return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ false); 1711 case CharacterCodes.percent: 1712 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1713 return pos += 2, token = SyntaxKind.PercentEqualsToken; 1714 } 1715 pos++; 1716 return token = SyntaxKind.PercentToken; 1717 case CharacterCodes.ampersand: 1718 if (text.charCodeAt(pos + 1) === CharacterCodes.ampersand) { 1719 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1720 return pos += 3, token = SyntaxKind.AmpersandAmpersandEqualsToken; 1721 } 1722 return pos += 2, token = SyntaxKind.AmpersandAmpersandToken; 1723 } 1724 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1725 return pos += 2, token = SyntaxKind.AmpersandEqualsToken; 1726 } 1727 pos++; 1728 return token = SyntaxKind.AmpersandToken; 1729 case CharacterCodes.openParen: 1730 pos++; 1731 return token = SyntaxKind.OpenParenToken; 1732 case CharacterCodes.closeParen: 1733 pos++; 1734 return token = SyntaxKind.CloseParenToken; 1735 case CharacterCodes.asterisk: 1736 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1737 return pos += 2, token = SyntaxKind.AsteriskEqualsToken; 1738 } 1739 if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { 1740 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1741 return pos += 3, token = SyntaxKind.AsteriskAsteriskEqualsToken; 1742 } 1743 return pos += 2, token = SyntaxKind.AsteriskAsteriskToken; 1744 } 1745 pos++; 1746 if (inJSDocType && !asteriskSeen && (tokenFlags & TokenFlags.PrecedingLineBreak)) { 1747 // decoration at the start of a JSDoc comment line 1748 asteriskSeen = true; 1749 continue; 1750 } 1751 return token = SyntaxKind.AsteriskToken; 1752 case CharacterCodes.plus: 1753 if (text.charCodeAt(pos + 1) === CharacterCodes.plus) { 1754 return pos += 2, token = SyntaxKind.PlusPlusToken; 1755 } 1756 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1757 return pos += 2, token = SyntaxKind.PlusEqualsToken; 1758 } 1759 pos++; 1760 return token = SyntaxKind.PlusToken; 1761 case CharacterCodes.comma: 1762 pos++; 1763 return token = SyntaxKind.CommaToken; 1764 case CharacterCodes.minus: 1765 if (text.charCodeAt(pos + 1) === CharacterCodes.minus) { 1766 return pos += 2, token = SyntaxKind.MinusMinusToken; 1767 } 1768 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1769 return pos += 2, token = SyntaxKind.MinusEqualsToken; 1770 } 1771 pos++; 1772 return token = SyntaxKind.MinusToken; 1773 case CharacterCodes.dot: 1774 if (isDigit(text.charCodeAt(pos + 1))) { 1775 tokenValue = scanNumber().value; 1776 return token = SyntaxKind.NumericLiteral; 1777 } 1778 if (text.charCodeAt(pos + 1) === CharacterCodes.dot && text.charCodeAt(pos + 2) === CharacterCodes.dot) { 1779 return pos += 3, token = SyntaxKind.DotDotDotToken; 1780 } 1781 pos++; 1782 return token = SyntaxKind.DotToken; 1783 case CharacterCodes.slash: 1784 // Single-line comment 1785 if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { 1786 pos += 2; 1787 1788 while (pos < end) { 1789 if (isLineBreak(text.charCodeAt(pos))) { 1790 break; 1791 } 1792 pos++; 1793 } 1794 1795 commentDirectives = appendIfCommentDirective( 1796 commentDirectives, 1797 text.slice(tokenPos, pos), 1798 commentDirectiveRegExSingleLine, 1799 tokenPos, 1800 ); 1801 1802 if (skipTrivia) { 1803 continue; 1804 } 1805 else { 1806 return token = SyntaxKind.SingleLineCommentTrivia; 1807 } 1808 } 1809 // Multi-line comment 1810 if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { 1811 pos += 2; 1812 if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) !== CharacterCodes.slash) { 1813 tokenFlags |= TokenFlags.PrecedingJSDocComment; 1814 } 1815 1816 let commentClosed = false; 1817 let lastLineStart = tokenPos; 1818 while (pos < end) { 1819 const ch = text.charCodeAt(pos); 1820 1821 if (ch === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { 1822 pos += 2; 1823 commentClosed = true; 1824 break; 1825 } 1826 1827 pos++; 1828 1829 if (isLineBreak(ch)) { 1830 lastLineStart = pos; 1831 tokenFlags |= TokenFlags.PrecedingLineBreak; 1832 } 1833 } 1834 1835 commentDirectives = appendIfCommentDirective(commentDirectives, text.slice(lastLineStart, pos), commentDirectiveRegExMultiLine, lastLineStart); 1836 1837 if (!commentClosed) { 1838 error(Diagnostics.Asterisk_Slash_expected); 1839 } 1840 1841 if (skipTrivia) { 1842 continue; 1843 } 1844 else { 1845 if (!commentClosed) { 1846 tokenFlags |= TokenFlags.Unterminated; 1847 } 1848 return token = SyntaxKind.MultiLineCommentTrivia; 1849 } 1850 } 1851 1852 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1853 return pos += 2, token = SyntaxKind.SlashEqualsToken; 1854 } 1855 1856 pos++; 1857 return token = SyntaxKind.SlashToken; 1858 1859 case CharacterCodes._0: 1860 if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) { 1861 pos += 2; 1862 tokenValue = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ true); 1863 if (!tokenValue) { 1864 error(Diagnostics.Hexadecimal_digit_expected); 1865 tokenValue = "0"; 1866 } 1867 tokenValue = "0x" + tokenValue; 1868 tokenFlags |= TokenFlags.HexSpecifier; 1869 return token = checkBigIntSuffix(); 1870 } 1871 else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) { 1872 pos += 2; 1873 tokenValue = scanBinaryOrOctalDigits(/* base */ 2); 1874 if (!tokenValue) { 1875 error(Diagnostics.Binary_digit_expected); 1876 tokenValue = "0"; 1877 } 1878 tokenValue = "0b" + tokenValue; 1879 tokenFlags |= TokenFlags.BinarySpecifier; 1880 return token = checkBigIntSuffix(); 1881 } 1882 else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) { 1883 pos += 2; 1884 tokenValue = scanBinaryOrOctalDigits(/* base */ 8); 1885 if (!tokenValue) { 1886 error(Diagnostics.Octal_digit_expected); 1887 tokenValue = "0"; 1888 } 1889 tokenValue = "0o" + tokenValue; 1890 tokenFlags |= TokenFlags.OctalSpecifier; 1891 return token = checkBigIntSuffix(); 1892 } 1893 // Try to parse as an octal 1894 if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) { 1895 tokenValue = "" + scanOctalDigits(); 1896 tokenFlags |= TokenFlags.Octal; 1897 return token = SyntaxKind.NumericLiteral; 1898 } 1899 // This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero 1900 // can only be followed by an octal digit, a dot, or the end of the number literal. However, we are being 1901 // permissive and allowing decimal digits of the form 08* and 09* (which many browsers also do). 1902 // falls through 1903 case CharacterCodes._1: 1904 case CharacterCodes._2: 1905 case CharacterCodes._3: 1906 case CharacterCodes._4: 1907 case CharacterCodes._5: 1908 case CharacterCodes._6: 1909 case CharacterCodes._7: 1910 case CharacterCodes._8: 1911 case CharacterCodes._9: 1912 ({ type: token, value: tokenValue } = scanNumber()); 1913 return token; 1914 case CharacterCodes.colon: 1915 pos++; 1916 return token = SyntaxKind.ColonToken; 1917 case CharacterCodes.semicolon: 1918 pos++; 1919 return token = SyntaxKind.SemicolonToken; 1920 case CharacterCodes.lessThan: 1921 if (isConflictMarkerTrivia(text, pos)) { 1922 pos = scanConflictMarkerTrivia(text, pos, error); 1923 if (skipTrivia) { 1924 continue; 1925 } 1926 else { 1927 return token = SyntaxKind.ConflictMarkerTrivia; 1928 } 1929 } 1930 1931 if (text.charCodeAt(pos + 1) === CharacterCodes.lessThan) { 1932 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1933 return pos += 3, token = SyntaxKind.LessThanLessThanEqualsToken; 1934 } 1935 return pos += 2, token = SyntaxKind.LessThanLessThanToken; 1936 } 1937 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1938 return pos += 2, token = SyntaxKind.LessThanEqualsToken; 1939 } 1940 if (languageVariant === LanguageVariant.JSX && 1941 text.charCodeAt(pos + 1) === CharacterCodes.slash && 1942 text.charCodeAt(pos + 2) !== CharacterCodes.asterisk) { 1943 return pos += 2, token = SyntaxKind.LessThanSlashToken; 1944 } 1945 pos++; 1946 return token = SyntaxKind.LessThanToken; 1947 case CharacterCodes.equals: 1948 if (isConflictMarkerTrivia(text, pos)) { 1949 pos = scanConflictMarkerTrivia(text, pos, error); 1950 if (skipTrivia) { 1951 continue; 1952 } 1953 else { 1954 return token = SyntaxKind.ConflictMarkerTrivia; 1955 } 1956 } 1957 1958 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1959 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1960 return pos += 3, token = SyntaxKind.EqualsEqualsEqualsToken; 1961 } 1962 return pos += 2, token = SyntaxKind.EqualsEqualsToken; 1963 } 1964 if (text.charCodeAt(pos + 1) === CharacterCodes.greaterThan) { 1965 return pos += 2, token = SyntaxKind.EqualsGreaterThanToken; 1966 } 1967 pos++; 1968 return token = SyntaxKind.EqualsToken; 1969 case CharacterCodes.greaterThan: 1970 if (isConflictMarkerTrivia(text, pos)) { 1971 pos = scanConflictMarkerTrivia(text, pos, error); 1972 if (skipTrivia) { 1973 continue; 1974 } 1975 else { 1976 return token = SyntaxKind.ConflictMarkerTrivia; 1977 } 1978 } 1979 1980 pos++; 1981 return token = SyntaxKind.GreaterThanToken; 1982 case CharacterCodes.question: 1983 if (text.charCodeAt(pos + 1) === CharacterCodes.dot && !isDigit(text.charCodeAt(pos + 2))) { 1984 return pos += 2, token = SyntaxKind.QuestionDotToken; 1985 } 1986 if (text.charCodeAt(pos + 1) === CharacterCodes.question) { 1987 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1988 return pos += 3, token = SyntaxKind.QuestionQuestionEqualsToken; 1989 } 1990 return pos += 2, token = SyntaxKind.QuestionQuestionToken; 1991 } 1992 pos++; 1993 return token = SyntaxKind.QuestionToken; 1994 case CharacterCodes.openBracket: 1995 pos++; 1996 return token = SyntaxKind.OpenBracketToken; 1997 case CharacterCodes.closeBracket: 1998 pos++; 1999 return token = SyntaxKind.CloseBracketToken; 2000 case CharacterCodes.caret: 2001 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 2002 return pos += 2, token = SyntaxKind.CaretEqualsToken; 2003 } 2004 pos++; 2005 return token = SyntaxKind.CaretToken; 2006 case CharacterCodes.openBrace: 2007 pos++; 2008 return token = SyntaxKind.OpenBraceToken; 2009 case CharacterCodes.bar: 2010 if (isConflictMarkerTrivia(text, pos)) { 2011 pos = scanConflictMarkerTrivia(text, pos, error); 2012 if (skipTrivia) { 2013 continue; 2014 } 2015 else { 2016 return token = SyntaxKind.ConflictMarkerTrivia; 2017 } 2018 } 2019 2020 if (text.charCodeAt(pos + 1) === CharacterCodes.bar) { 2021 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 2022 return pos += 3, token = SyntaxKind.BarBarEqualsToken; 2023 } 2024 return pos += 2, token = SyntaxKind.BarBarToken; 2025 } 2026 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 2027 return pos += 2, token = SyntaxKind.BarEqualsToken; 2028 } 2029 pos++; 2030 return token = SyntaxKind.BarToken; 2031 case CharacterCodes.closeBrace: 2032 pos++; 2033 return token = SyntaxKind.CloseBraceToken; 2034 case CharacterCodes.tilde: 2035 pos++; 2036 return token = SyntaxKind.TildeToken; 2037 case CharacterCodes.at: 2038 pos++; 2039 return token = SyntaxKind.AtToken; 2040 case CharacterCodes.backslash: 2041 const extendedCookedChar = peekExtendedUnicodeEscape(); 2042 if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) { 2043 pos += 3; 2044 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 2045 tokenValue = scanExtendedUnicodeEscape() + scanIdentifierParts(); 2046 return token = getIdentifierToken(); 2047 } 2048 2049 const cookedChar = peekUnicodeEscape(); 2050 if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) { 2051 pos += 6; 2052 tokenFlags |= TokenFlags.UnicodeEscape; 2053 tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts(); 2054 return token = getIdentifierToken(); 2055 } 2056 2057 error(Diagnostics.Invalid_character); 2058 pos++; 2059 return token = SyntaxKind.Unknown; 2060 case CharacterCodes.hash: 2061 if (pos !== 0 && text[pos + 1] === "!") { 2062 error(Diagnostics.can_only_be_used_at_the_start_of_a_file); 2063 pos++; 2064 return token = SyntaxKind.Unknown; 2065 } 2066 2067 const charAfterHash = codePointAt(text, pos + 1); 2068 if (charAfterHash === CharacterCodes.backslash) { 2069 pos++; 2070 const extendedCookedChar = peekExtendedUnicodeEscape(); 2071 if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) { 2072 pos += 3; 2073 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 2074 tokenValue = "#" + scanExtendedUnicodeEscape() + scanIdentifierParts(); 2075 return token = SyntaxKind.PrivateIdentifier; 2076 } 2077 2078 const cookedChar = peekUnicodeEscape(); 2079 if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) { 2080 pos += 6; 2081 tokenFlags |= TokenFlags.UnicodeEscape; 2082 tokenValue = "#" + String.fromCharCode(cookedChar) + scanIdentifierParts(); 2083 return token = SyntaxKind.PrivateIdentifier; 2084 } 2085 pos--; 2086 } 2087 2088 if (isIdentifierStart(charAfterHash, languageVersion)) { 2089 pos++; 2090 // We're relying on scanIdentifier's behavior and adjusting the token kind after the fact. 2091 // Notably absent from this block is the fact that calling a function named "scanIdentifier", 2092 // but identifiers don't include '#', and that function doesn't deal with it at all. 2093 // This works because 'scanIdentifier' tries to reuse source characters and builds up substrings; 2094 // however, it starts at the 'tokenPos' which includes the '#', and will "accidentally" prepend the '#' for us. 2095 scanIdentifier(charAfterHash, languageVersion); 2096 } 2097 else { 2098 tokenValue = "#"; 2099 error(Diagnostics.Invalid_character, pos++, charSize(ch)); 2100 } 2101 return token = SyntaxKind.PrivateIdentifier; 2102 default: 2103 const identifierKind = scanIdentifier(ch, languageVersion); 2104 if (identifierKind) { 2105 return token = identifierKind; 2106 } 2107 else if (isWhiteSpaceSingleLine(ch)) { 2108 pos += charSize(ch); 2109 continue; 2110 } 2111 else if (isLineBreak(ch)) { 2112 tokenFlags |= TokenFlags.PrecedingLineBreak; 2113 pos += charSize(ch); 2114 continue; 2115 } 2116 const size = charSize(ch); 2117 error(Diagnostics.Invalid_character, pos, size); 2118 pos += size; 2119 return token = SyntaxKind.Unknown; 2120 } 2121 } 2122 } 2123 2124 function reScanInvalidIdentifier(): SyntaxKind { 2125 Debug.assert(token === SyntaxKind.Unknown, "'reScanInvalidIdentifier' should only be called when the current token is 'SyntaxKind.Unknown'."); 2126 pos = tokenPos = startPos; 2127 tokenFlags = 0; 2128 const ch = codePointAt(text, pos); 2129 const identifierKind = scanIdentifier(ch, ScriptTarget.ESNext); 2130 if (identifierKind) { 2131 return token = identifierKind; 2132 } 2133 pos += charSize(ch); 2134 return token; // Still `SyntaKind.Unknown` 2135 } 2136 2137 function scanIdentifier(startCharacter: number, languageVersion: ScriptTarget) { 2138 let ch = startCharacter; 2139 if (isIdentifierStart(ch, languageVersion)) { 2140 pos += charSize(ch); 2141 while (pos < end && isIdentifierPart(ch = codePointAt(text, pos), languageVersion)) pos += charSize(ch); 2142 tokenValue = text.substring(tokenPos, pos); 2143 if (ch === CharacterCodes.backslash) { 2144 tokenValue += scanIdentifierParts(); 2145 } 2146 return getIdentifierToken(); 2147 } 2148 } 2149 2150 function reScanGreaterToken(): SyntaxKind { 2151 if (token === SyntaxKind.GreaterThanToken) { 2152 if (text.charCodeAt(pos) === CharacterCodes.greaterThan) { 2153 if (text.charCodeAt(pos + 1) === CharacterCodes.greaterThan) { 2154 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 2155 return pos += 3, token = SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken; 2156 } 2157 return pos += 2, token = SyntaxKind.GreaterThanGreaterThanGreaterThanToken; 2158 } 2159 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 2160 return pos += 2, token = SyntaxKind.GreaterThanGreaterThanEqualsToken; 2161 } 2162 pos++; 2163 return token = SyntaxKind.GreaterThanGreaterThanToken; 2164 } 2165 if (text.charCodeAt(pos) === CharacterCodes.equals) { 2166 pos++; 2167 return token = SyntaxKind.GreaterThanEqualsToken; 2168 } 2169 } 2170 return token; 2171 } 2172 2173 function reScanAsteriskEqualsToken(): SyntaxKind { 2174 Debug.assert(token === SyntaxKind.AsteriskEqualsToken, "'reScanAsteriskEqualsToken' should only be called on a '*='"); 2175 pos = tokenPos + 1; 2176 return token = SyntaxKind.EqualsToken; 2177 } 2178 2179 function reScanSlashToken(): SyntaxKind { 2180 if (token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) { 2181 let p = tokenPos + 1; 2182 let inEscape = false; 2183 let inCharacterClass = false; 2184 while (true) { 2185 // If we reach the end of a file, or hit a newline, then this is an unterminated 2186 // regex. Report error and return what we have so far. 2187 if (p >= end) { 2188 tokenFlags |= TokenFlags.Unterminated; 2189 error(Diagnostics.Unterminated_regular_expression_literal); 2190 break; 2191 } 2192 2193 const ch = text.charCodeAt(p); 2194 if (isLineBreak(ch)) { 2195 tokenFlags |= TokenFlags.Unterminated; 2196 error(Diagnostics.Unterminated_regular_expression_literal); 2197 break; 2198 } 2199 2200 if (inEscape) { 2201 // Parsing an escape character; 2202 // reset the flag and just advance to the next char. 2203 inEscape = false; 2204 } 2205 else if (ch === CharacterCodes.slash && !inCharacterClass) { 2206 // A slash within a character class is permissible, 2207 // but in general it signals the end of the regexp literal. 2208 p++; 2209 break; 2210 } 2211 else if (ch === CharacterCodes.openBracket) { 2212 inCharacterClass = true; 2213 } 2214 else if (ch === CharacterCodes.backslash) { 2215 inEscape = true; 2216 } 2217 else if (ch === CharacterCodes.closeBracket) { 2218 inCharacterClass = false; 2219 } 2220 p++; 2221 } 2222 2223 while (p < end && isIdentifierPart(text.charCodeAt(p), languageVersion)) { 2224 p++; 2225 } 2226 pos = p; 2227 tokenValue = text.substring(tokenPos, pos); 2228 token = SyntaxKind.RegularExpressionLiteral; 2229 } 2230 return token; 2231 } 2232 2233 function appendIfCommentDirective( 2234 commentDirectives: CommentDirective[] | undefined, 2235 text: string, 2236 commentDirectiveRegEx: RegExp, 2237 lineStart: number, 2238 ) { 2239 const type = getDirectiveFromComment(trimStringStart(text), commentDirectiveRegEx); 2240 if (type === undefined) { 2241 return commentDirectives; 2242 } 2243 2244 return append( 2245 commentDirectives, 2246 { 2247 range: { pos: lineStart, end: pos }, 2248 type, 2249 }, 2250 ); 2251 } 2252 2253 function getDirectiveFromComment(text: string, commentDirectiveRegEx: RegExp) { 2254 const match = commentDirectiveRegEx.exec(text); 2255 if (!match) { 2256 return undefined; 2257 } 2258 2259 switch (match[1]) { 2260 case "ts-expect-error": 2261 return CommentDirectiveType.ExpectError; 2262 2263 case "ts-ignore": 2264 return CommentDirectiveType.Ignore; 2265 } 2266 2267 return undefined; 2268 } 2269 2270 /** 2271 * Unconditionally back up and scan a template expression portion. 2272 */ 2273 function reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind { 2274 Debug.assert(token === SyntaxKind.CloseBraceToken, "'reScanTemplateToken' should only be called on a '}'"); 2275 pos = tokenPos; 2276 return token = scanTemplateAndSetTokenValue(isTaggedTemplate); 2277 } 2278 2279 function reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind { 2280 pos = tokenPos; 2281 return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ true); 2282 } 2283 2284 function reScanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind { 2285 pos = tokenPos = startPos; 2286 return token = scanJsxToken(allowMultilineJsxText); 2287 } 2288 2289 function reScanLessThanToken(): SyntaxKind { 2290 if (token === SyntaxKind.LessThanLessThanToken) { 2291 pos = tokenPos + 1; 2292 return token = SyntaxKind.LessThanToken; 2293 } 2294 return token; 2295 } 2296 2297 function reScanHashToken(): SyntaxKind { 2298 if (token === SyntaxKind.PrivateIdentifier) { 2299 pos = tokenPos + 1; 2300 return token = SyntaxKind.HashToken; 2301 } 2302 return token; 2303 } 2304 2305 function reScanQuestionToken(): SyntaxKind { 2306 Debug.assert(token === SyntaxKind.QuestionQuestionToken, "'reScanQuestionToken' should only be called on a '??'"); 2307 pos = tokenPos + 1; 2308 return token = SyntaxKind.QuestionToken; 2309 } 2310 2311 function scanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind { 2312 startPos = tokenPos = pos; 2313 2314 if (pos >= end) { 2315 return token = SyntaxKind.EndOfFileToken; 2316 } 2317 2318 let char = text.charCodeAt(pos); 2319 if (char === CharacterCodes.lessThan) { 2320 if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { 2321 pos += 2; 2322 return token = SyntaxKind.LessThanSlashToken; 2323 } 2324 pos++; 2325 return token = SyntaxKind.LessThanToken; 2326 } 2327 2328 if (char === CharacterCodes.openBrace) { 2329 pos++; 2330 return token = SyntaxKind.OpenBraceToken; 2331 } 2332 2333 // First non-whitespace character on this line. 2334 let firstNonWhitespace = 0; 2335 2336 // These initial values are special because the first line is: 2337 // firstNonWhitespace = 0 to indicate that we want leading whitespace, 2338 2339 while (pos < end) { 2340 char = text.charCodeAt(pos); 2341 if (char === CharacterCodes.openBrace) { 2342 break; 2343 } 2344 if (char === CharacterCodes.lessThan) { 2345 if (isConflictMarkerTrivia(text, pos)) { 2346 pos = scanConflictMarkerTrivia(text, pos, error); 2347 return token = SyntaxKind.ConflictMarkerTrivia; 2348 } 2349 break; 2350 } 2351 if (char === CharacterCodes.greaterThan) { 2352 error(Diagnostics.Unexpected_token_Did_you_mean_or_gt, pos, 1); 2353 } 2354 if (char === CharacterCodes.closeBrace) { 2355 error(Diagnostics.Unexpected_token_Did_you_mean_or_rbrace, pos, 1); 2356 } 2357 2358 // FirstNonWhitespace is 0, then we only see whitespaces so far. If we see a linebreak, we want to ignore that whitespaces. 2359 // i.e (- : whitespace) 2360 // <div>---- 2361 // </div> becomes <div></div> 2362 // 2363 // <div>----</div> becomes <div>----</div> 2364 if (isLineBreak(char) && firstNonWhitespace === 0) { 2365 firstNonWhitespace = -1; 2366 } 2367 else if (!allowMultilineJsxText && isLineBreak(char) && firstNonWhitespace > 0) { 2368 // Stop JsxText on each line during formatting. This allows the formatter to 2369 // indent each line correctly. 2370 break; 2371 } 2372 else if (!isWhiteSpaceLike(char)) { 2373 firstNonWhitespace = pos; 2374 } 2375 2376 pos++; 2377 } 2378 2379 tokenValue = text.substring(startPos, pos); 2380 2381 return firstNonWhitespace === -1 ? SyntaxKind.JsxTextAllWhiteSpaces : SyntaxKind.JsxText; 2382 } 2383 2384 // Scans a JSX identifier; these differ from normal identifiers in that 2385 // they allow dashes 2386 function scanJsxIdentifier(): SyntaxKind { 2387 if (tokenIsIdentifierOrKeyword(token)) { 2388 // An identifier or keyword has already been parsed - check for a `-` or a single instance of `:` and then append it and 2389 // everything after it to the token 2390 // Do note that this means that `scanJsxIdentifier` effectively _mutates_ the visible token without advancing to a new token 2391 // Any caller should be expecting this behavior and should only read the pos or token value after calling it. 2392 let namespaceSeparator = false; 2393 while (pos < end) { 2394 const ch = text.charCodeAt(pos); 2395 if (ch === CharacterCodes.minus) { 2396 tokenValue += "-"; 2397 pos++; 2398 continue; 2399 } 2400 else if (ch === CharacterCodes.colon && !namespaceSeparator) { 2401 tokenValue += ":"; 2402 pos++; 2403 namespaceSeparator = true; 2404 token = SyntaxKind.Identifier; // swap from keyword kind to identifier kind 2405 continue; 2406 } 2407 const oldPos = pos; 2408 tokenValue += scanIdentifierParts(); // reuse `scanIdentifierParts` so unicode escapes are handled 2409 if (pos === oldPos) { 2410 break; 2411 } 2412 } 2413 // Do not include a trailing namespace separator in the token, since this is against the spec. 2414 if (tokenValue.slice(-1) === ":") { 2415 tokenValue = tokenValue.slice(0, -1); 2416 pos--; 2417 } 2418 return getIdentifierToken(); 2419 } 2420 return token; 2421 } 2422 2423 function scanJsxAttributeValue(): SyntaxKind { 2424 startPos = pos; 2425 2426 switch (text.charCodeAt(pos)) { 2427 case CharacterCodes.doubleQuote: 2428 case CharacterCodes.singleQuote: 2429 tokenValue = scanString(/*jsxAttributeString*/ true); 2430 return token = SyntaxKind.StringLiteral; 2431 default: 2432 // If this scans anything other than `{`, it's a parse error. 2433 return scan(); 2434 } 2435 } 2436 2437 function reScanJsxAttributeValue(): SyntaxKind { 2438 pos = tokenPos = startPos; 2439 return scanJsxAttributeValue(); 2440 } 2441 2442 function scanJsDocToken(): JSDocSyntaxKind { 2443 startPos = tokenPos = pos; 2444 tokenFlags = TokenFlags.None; 2445 if (pos >= end) { 2446 return token = SyntaxKind.EndOfFileToken; 2447 } 2448 2449 const ch = codePointAt(text, pos); 2450 pos += charSize(ch); 2451 switch (ch) { 2452 case CharacterCodes.tab: 2453 case CharacterCodes.verticalTab: 2454 case CharacterCodes.formFeed: 2455 case CharacterCodes.space: 2456 while (pos < end && isWhiteSpaceSingleLine(text.charCodeAt(pos))) { 2457 pos++; 2458 } 2459 return token = SyntaxKind.WhitespaceTrivia; 2460 case CharacterCodes.at: 2461 return token = SyntaxKind.AtToken; 2462 case CharacterCodes.carriageReturn: 2463 if (text.charCodeAt(pos) === CharacterCodes.lineFeed) { 2464 pos++; 2465 } 2466 // falls through 2467 case CharacterCodes.lineFeed: 2468 tokenFlags |= TokenFlags.PrecedingLineBreak; 2469 return token = SyntaxKind.NewLineTrivia; 2470 case CharacterCodes.asterisk: 2471 return token = SyntaxKind.AsteriskToken; 2472 case CharacterCodes.openBrace: 2473 return token = SyntaxKind.OpenBraceToken; 2474 case CharacterCodes.closeBrace: 2475 return token = SyntaxKind.CloseBraceToken; 2476 case CharacterCodes.openBracket: 2477 return token = SyntaxKind.OpenBracketToken; 2478 case CharacterCodes.closeBracket: 2479 return token = SyntaxKind.CloseBracketToken; 2480 case CharacterCodes.lessThan: 2481 return token = SyntaxKind.LessThanToken; 2482 case CharacterCodes.greaterThan: 2483 return token = SyntaxKind.GreaterThanToken; 2484 case CharacterCodes.equals: 2485 return token = SyntaxKind.EqualsToken; 2486 case CharacterCodes.comma: 2487 return token = SyntaxKind.CommaToken; 2488 case CharacterCodes.dot: 2489 return token = SyntaxKind.DotToken; 2490 case CharacterCodes.backtick: 2491 return token = SyntaxKind.BacktickToken; 2492 case CharacterCodes.hash: 2493 return token = SyntaxKind.HashToken; 2494 case CharacterCodes.backslash: 2495 pos--; 2496 const extendedCookedChar = peekExtendedUnicodeEscape(); 2497 if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) { 2498 pos += 3; 2499 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 2500 tokenValue = scanExtendedUnicodeEscape() + scanIdentifierParts(); 2501 return token = getIdentifierToken(); 2502 } 2503 2504 const cookedChar = peekUnicodeEscape(); 2505 if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) { 2506 pos += 6; 2507 tokenFlags |= TokenFlags.UnicodeEscape; 2508 tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts(); 2509 return token = getIdentifierToken(); 2510 } 2511 pos++; 2512 return token = SyntaxKind.Unknown; 2513 } 2514 2515 if (isIdentifierStart(ch, languageVersion)) { 2516 let char = ch; 2517 while (pos < end && isIdentifierPart(char = codePointAt(text, pos), languageVersion) || text.charCodeAt(pos) === CharacterCodes.minus) pos += charSize(char); 2518 tokenValue = text.substring(tokenPos, pos); 2519 if (char === CharacterCodes.backslash) { 2520 tokenValue += scanIdentifierParts(); 2521 } 2522 return token = getIdentifierToken(); 2523 } 2524 else { 2525 return token = SyntaxKind.Unknown; 2526 } 2527 } 2528 2529 function speculationHelper<T>(callback: () => T, isLookahead: boolean): T { 2530 const savePos = pos; 2531 const saveStartPos = startPos; 2532 const saveTokenPos = tokenPos; 2533 const saveToken = token; 2534 const saveTokenValue = tokenValue; 2535 const saveTokenFlags = tokenFlags; 2536 const result = callback(); 2537 2538 // If our callback returned something 'falsy' or we're just looking ahead, 2539 // then unconditionally restore us to where we were. 2540 if (!result || isLookahead) { 2541 pos = savePos; 2542 startPos = saveStartPos; 2543 tokenPos = saveTokenPos; 2544 token = saveToken; 2545 tokenValue = saveTokenValue; 2546 tokenFlags = saveTokenFlags; 2547 } 2548 return result; 2549 } 2550 2551 function scanRange<T>(start: number, length: number, callback: () => T): T { 2552 const saveEnd = end; 2553 const savePos = pos; 2554 const saveStartPos = startPos; 2555 const saveTokenPos = tokenPos; 2556 const saveToken = token; 2557 const saveTokenValue = tokenValue; 2558 const saveTokenFlags = tokenFlags; 2559 const saveErrorExpectations = commentDirectives; 2560 2561 setText(text, start, length); 2562 const result = callback(); 2563 2564 end = saveEnd; 2565 pos = savePos; 2566 startPos = saveStartPos; 2567 tokenPos = saveTokenPos; 2568 token = saveToken; 2569 tokenValue = saveTokenValue; 2570 tokenFlags = saveTokenFlags; 2571 commentDirectives = saveErrorExpectations; 2572 2573 return result; 2574 } 2575 2576 function lookAhead<T>(callback: () => T): T { 2577 return speculationHelper(callback, /*isLookahead*/ true); 2578 } 2579 2580 function tryScan<T>(callback: () => T): T { 2581 return speculationHelper(callback, /*isLookahead*/ false); 2582 } 2583 2584 function getText(): string { 2585 return text; 2586 } 2587 2588 function clearCommentDirectives() { 2589 commentDirectives = undefined; 2590 } 2591 2592 function setText(newText: string | undefined, start: number | undefined, length: number | undefined) { 2593 text = newText || ""; 2594 end = length === undefined ? text.length : start! + length; 2595 setTextPos(start || 0); 2596 } 2597 2598 function setOnError(errorCallback: ErrorCallback | undefined) { 2599 onError = errorCallback; 2600 } 2601 2602 function setScriptTarget(scriptTarget: ScriptTarget) { 2603 languageVersion = scriptTarget; 2604 } 2605 2606 function setLanguageVariant(variant: LanguageVariant) { 2607 languageVariant = variant; 2608 } 2609 2610 function setTextPos(textPos: number) { 2611 Debug.assert(textPos >= 0); 2612 pos = textPos; 2613 startPos = textPos; 2614 tokenPos = textPos; 2615 token = SyntaxKind.Unknown; 2616 tokenValue = undefined!; 2617 tokenFlags = TokenFlags.None; 2618 } 2619 2620 function setInJSDocType(inType: boolean) { 2621 inJSDocType += inType ? 1 : -1; 2622 } 2623 } 2624 2625 /* @internal */ 2626 const codePointAt: (s: string, i: number) => number = (String.prototype as any).codePointAt ? (s, i) => (s as any).codePointAt(i) : function codePointAt(str, i): number { 2627 // from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/codePointAt 2628 const size = str.length; 2629 // Account for out-of-bounds indices: 2630 if (i < 0 || i >= size) { 2631 return undefined!; // String.codePointAt returns `undefined` for OOB indexes 2632 } 2633 // Get the first code unit 2634 const first = str.charCodeAt(i); 2635 // check if it’s the start of a surrogate pair 2636 if (first >= 0xD800 && first <= 0xDBFF && size > i + 1) { // high surrogate and there is a next code unit 2637 const second = str.charCodeAt(i + 1); 2638 if (second >= 0xDC00 && second <= 0xDFFF) { // low surrogate 2639 // https://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae 2640 return (first - 0xD800) * 0x400 + second - 0xDC00 + 0x10000; 2641 } 2642 } 2643 return first; 2644 }; 2645 2646 /* @internal */ 2647 function charSize(ch: number) { 2648 if (ch >= 0x10000) { 2649 return 2; 2650 } 2651 return 1; 2652 } 2653 2654 // Derived from the 10.1.1 UTF16Encoding of the ES6 Spec. 2655 function utf16EncodeAsStringFallback(codePoint: number) { 2656 Debug.assert(0x0 <= codePoint && codePoint <= 0x10FFFF); 2657 2658 if (codePoint <= 65535) { 2659 return String.fromCharCode(codePoint); 2660 } 2661 2662 const codeUnit1 = Math.floor((codePoint - 65536) / 1024) + 0xD800; 2663 const codeUnit2 = ((codePoint - 65536) % 1024) + 0xDC00; 2664 2665 return String.fromCharCode(codeUnit1, codeUnit2); 2666 } 2667 2668 const utf16EncodeAsStringWorker: (codePoint: number) => string = (String as any).fromCodePoint ? codePoint => (String as any).fromCodePoint(codePoint) : utf16EncodeAsStringFallback; 2669 2670 /* @internal */ 2671 export function utf16EncodeAsString(codePoint: number) { 2672 return utf16EncodeAsStringWorker(codePoint); 2673 } 2674} 2675