1namespace ts { 2 export type ErrorCallback = (message: DiagnosticMessage, length: number) => void; 3 4 /* @internal */ 5 export function tokenIsIdentifierOrKeyword(token: SyntaxKind): boolean { 6 return token >= SyntaxKind.Identifier; 7 } 8 9 /* @internal */ 10 export function tokenIsIdentifierOrKeywordOrGreaterThan(token: SyntaxKind): boolean { 11 return token === SyntaxKind.GreaterThanToken || tokenIsIdentifierOrKeyword(token); 12 } 13 14 export interface Scanner { 15 getStartPos(): number; 16 getToken(): SyntaxKind; 17 getTextPos(): number; 18 getTokenPos(): number; 19 getTokenText(): string; 20 getTokenValue(): string; 21 hasUnicodeEscape(): boolean; 22 hasExtendedUnicodeEscape(): boolean; 23 hasPrecedingLineBreak(): boolean; 24 /* @internal */ 25 hasPrecedingJSDocComment(): boolean; 26 isIdentifier(): boolean; 27 isReservedWord(): boolean; 28 isUnterminated(): boolean; 29 /* @internal */ 30 getNumericLiteralFlags(): TokenFlags; 31 /* @internal */ 32 getCommentDirectives(): CommentDirective[] | undefined; 33 /* @internal */ 34 getTokenFlags(): TokenFlags; 35 reScanGreaterToken(): SyntaxKind; 36 reScanSlashToken(): SyntaxKind; 37 reScanAsteriskEqualsToken(): SyntaxKind; 38 reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind; 39 reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind; 40 scanJsxIdentifier(): SyntaxKind; 41 scanJsxAttributeValue(): SyntaxKind; 42 reScanJsxAttributeValue(): SyntaxKind; 43 reScanJsxToken(allowMultilineJsxText?: boolean): JsxTokenSyntaxKind; 44 reScanLessThanToken(): SyntaxKind; 45 reScanHashToken(): SyntaxKind; 46 reScanQuestionToken(): SyntaxKind; 47 reScanInvalidIdentifier(): SyntaxKind; 48 scanJsxToken(): JsxTokenSyntaxKind; 49 scanJsDocToken(): JSDocSyntaxKind; 50 scan(): SyntaxKind; 51 52 getText(): string; 53 /* @internal */ 54 clearCommentDirectives(): void; 55 // Sets the text for the scanner to scan. An optional subrange starting point and length 56 // can be provided to have the scanner only scan a portion of the text. 57 setText(text: string | undefined, start?: number, length?: number): void; 58 setOnError(onError: ErrorCallback | undefined): void; 59 setScriptTarget(scriptTarget: ScriptTarget): void; 60 setLanguageVariant(variant: LanguageVariant): void; 61 setTextPos(textPos: number): void; 62 /* @internal */ 63 setInJSDocType(inType: boolean): void; 64 // Invokes the provided callback then unconditionally restores the scanner to the state it 65 // was in immediately prior to invoking the callback. The result of invoking the callback 66 // is returned from this function. 67 lookAhead<T>(callback: () => T): T; 68 69 // Invokes the callback with the scanner set to scan the specified range. When the callback 70 // returns, the scanner is restored to the state it was in before scanRange was called. 71 scanRange<T>(start: number, length: number, callback: () => T): T; 72 73 // Invokes the provided callback. If the callback returns something falsy, then it restores 74 // the scanner to the state it was in immediately prior to invoking the callback. If the 75 // callback returns something truthy, then the scanner state is not rolled back. The result 76 // of invoking the callback is returned from this function. 77 tryScan<T>(callback: () => T): T; 78 setEtsContext(isEtsContext: boolean): void; 79 } 80 81 /** @internal */ 82 export const textToKeywordObj: MapLike<KeywordSyntaxKind> = { 83 abstract: SyntaxKind.AbstractKeyword, 84 accessor: SyntaxKind.AccessorKeyword, 85 any: SyntaxKind.AnyKeyword, 86 as: SyntaxKind.AsKeyword, 87 asserts: SyntaxKind.AssertsKeyword, 88 assert: SyntaxKind.AssertKeyword, 89 bigint: SyntaxKind.BigIntKeyword, 90 boolean: SyntaxKind.BooleanKeyword, 91 break: SyntaxKind.BreakKeyword, 92 case: SyntaxKind.CaseKeyword, 93 catch: SyntaxKind.CatchKeyword, 94 class: SyntaxKind.ClassKeyword, 95 continue: SyntaxKind.ContinueKeyword, 96 const: SyntaxKind.ConstKeyword, 97 ["" + "constructor"]: SyntaxKind.ConstructorKeyword, 98 debugger: SyntaxKind.DebuggerKeyword, 99 declare: SyntaxKind.DeclareKeyword, 100 default: SyntaxKind.DefaultKeyword, 101 delete: SyntaxKind.DeleteKeyword, 102 do: SyntaxKind.DoKeyword, 103 else: SyntaxKind.ElseKeyword, 104 enum: SyntaxKind.EnumKeyword, 105 export: SyntaxKind.ExportKeyword, 106 extends: SyntaxKind.ExtendsKeyword, 107 false: SyntaxKind.FalseKeyword, 108 finally: SyntaxKind.FinallyKeyword, 109 for: SyntaxKind.ForKeyword, 110 from: SyntaxKind.FromKeyword, 111 function: SyntaxKind.FunctionKeyword, 112 get: SyntaxKind.GetKeyword, 113 if: SyntaxKind.IfKeyword, 114 implements: SyntaxKind.ImplementsKeyword, 115 import: SyntaxKind.ImportKeyword, 116 in: SyntaxKind.InKeyword, 117 infer: SyntaxKind.InferKeyword, 118 instanceof: SyntaxKind.InstanceOfKeyword, 119 interface: SyntaxKind.InterfaceKeyword, 120 intrinsic: SyntaxKind.IntrinsicKeyword, 121 is: SyntaxKind.IsKeyword, 122 keyof: SyntaxKind.KeyOfKeyword, 123 let: SyntaxKind.LetKeyword, 124 module: SyntaxKind.ModuleKeyword, 125 namespace: SyntaxKind.NamespaceKeyword, 126 never: SyntaxKind.NeverKeyword, 127 new: SyntaxKind.NewKeyword, 128 null: SyntaxKind.NullKeyword, 129 number: SyntaxKind.NumberKeyword, 130 object: SyntaxKind.ObjectKeyword, 131 package: SyntaxKind.PackageKeyword, 132 private: SyntaxKind.PrivateKeyword, 133 protected: SyntaxKind.ProtectedKeyword, 134 public: SyntaxKind.PublicKeyword, 135 override: SyntaxKind.OverrideKeyword, 136 out: SyntaxKind.OutKeyword, 137 readonly: SyntaxKind.ReadonlyKeyword, 138 require: SyntaxKind.RequireKeyword, 139 global: SyntaxKind.GlobalKeyword, 140 return: SyntaxKind.ReturnKeyword, 141 satisfies: SyntaxKind.SatisfiesKeyword, 142 set: SyntaxKind.SetKeyword, 143 static: SyntaxKind.StaticKeyword, 144 string: SyntaxKind.StringKeyword, 145 struct: SyntaxKind.StructKeyword, 146 super: SyntaxKind.SuperKeyword, 147 switch: SyntaxKind.SwitchKeyword, 148 symbol: SyntaxKind.SymbolKeyword, 149 this: SyntaxKind.ThisKeyword, 150 throw: SyntaxKind.ThrowKeyword, 151 true: SyntaxKind.TrueKeyword, 152 try: SyntaxKind.TryKeyword, 153 type: SyntaxKind.TypeKeyword, 154 lazy: SyntaxKind.LazyKeyword, 155 typeof: SyntaxKind.TypeOfKeyword, 156 undefined: SyntaxKind.UndefinedKeyword, 157 unique: SyntaxKind.UniqueKeyword, 158 unknown: SyntaxKind.UnknownKeyword, 159 var: SyntaxKind.VarKeyword, 160 void: SyntaxKind.VoidKeyword, 161 while: SyntaxKind.WhileKeyword, 162 with: SyntaxKind.WithKeyword, 163 yield: SyntaxKind.YieldKeyword, 164 async: SyntaxKind.AsyncKeyword, 165 await: SyntaxKind.AwaitKeyword, 166 of: SyntaxKind.OfKeyword, 167 }; 168 169 const textToKeyword = new Map(getEntries(textToKeywordObj)); 170 171 const textToToken = new Map(getEntries({ 172 ...textToKeywordObj, 173 "{": SyntaxKind.OpenBraceToken, 174 "}": SyntaxKind.CloseBraceToken, 175 "(": SyntaxKind.OpenParenToken, 176 ")": SyntaxKind.CloseParenToken, 177 "[": SyntaxKind.OpenBracketToken, 178 "]": SyntaxKind.CloseBracketToken, 179 ".": SyntaxKind.DotToken, 180 "...": SyntaxKind.DotDotDotToken, 181 ";": SyntaxKind.SemicolonToken, 182 ",": SyntaxKind.CommaToken, 183 "<": SyntaxKind.LessThanToken, 184 ">": SyntaxKind.GreaterThanToken, 185 "<=": SyntaxKind.LessThanEqualsToken, 186 ">=": SyntaxKind.GreaterThanEqualsToken, 187 "==": SyntaxKind.EqualsEqualsToken, 188 "!=": SyntaxKind.ExclamationEqualsToken, 189 "===": SyntaxKind.EqualsEqualsEqualsToken, 190 "!==": SyntaxKind.ExclamationEqualsEqualsToken, 191 "=>": SyntaxKind.EqualsGreaterThanToken, 192 "+": SyntaxKind.PlusToken, 193 "-": SyntaxKind.MinusToken, 194 "**": SyntaxKind.AsteriskAsteriskToken, 195 "*": SyntaxKind.AsteriskToken, 196 "/": SyntaxKind.SlashToken, 197 "%": SyntaxKind.PercentToken, 198 "++": SyntaxKind.PlusPlusToken, 199 "--": SyntaxKind.MinusMinusToken, 200 "<<": SyntaxKind.LessThanLessThanToken, 201 "</": SyntaxKind.LessThanSlashToken, 202 ">>": SyntaxKind.GreaterThanGreaterThanToken, 203 ">>>": SyntaxKind.GreaterThanGreaterThanGreaterThanToken, 204 "&": SyntaxKind.AmpersandToken, 205 "|": SyntaxKind.BarToken, 206 "^": SyntaxKind.CaretToken, 207 "!": SyntaxKind.ExclamationToken, 208 "~": SyntaxKind.TildeToken, 209 "&&": SyntaxKind.AmpersandAmpersandToken, 210 "||": SyntaxKind.BarBarToken, 211 "?": SyntaxKind.QuestionToken, 212 "??": SyntaxKind.QuestionQuestionToken, 213 "?.": SyntaxKind.QuestionDotToken, 214 ":": SyntaxKind.ColonToken, 215 "=": SyntaxKind.EqualsToken, 216 "+=": SyntaxKind.PlusEqualsToken, 217 "-=": SyntaxKind.MinusEqualsToken, 218 "*=": SyntaxKind.AsteriskEqualsToken, 219 "**=": SyntaxKind.AsteriskAsteriskEqualsToken, 220 "/=": SyntaxKind.SlashEqualsToken, 221 "%=": SyntaxKind.PercentEqualsToken, 222 "<<=": SyntaxKind.LessThanLessThanEqualsToken, 223 ">>=": SyntaxKind.GreaterThanGreaterThanEqualsToken, 224 ">>>=": SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken, 225 "&=": SyntaxKind.AmpersandEqualsToken, 226 "|=": SyntaxKind.BarEqualsToken, 227 "^=": SyntaxKind.CaretEqualsToken, 228 "||=": SyntaxKind.BarBarEqualsToken, 229 "&&=": SyntaxKind.AmpersandAmpersandEqualsToken, 230 "??=": SyntaxKind.QuestionQuestionEqualsToken, 231 "@": SyntaxKind.AtToken, 232 "#": SyntaxKind.HashToken, 233 "`": SyntaxKind.BacktickToken, 234 })); 235 236 /* 237 As per ECMAScript Language Specification 3th Edition, Section 7.6: Identifiers 238 IdentifierStart :: 239 Can contain Unicode 3.0.0 categories: 240 Uppercase letter (Lu), 241 Lowercase letter (Ll), 242 Titlecase letter (Lt), 243 Modifier letter (Lm), 244 Other letter (Lo), or 245 Letter number (Nl). 246 IdentifierPart :: = 247 Can contain IdentifierStart + Unicode 3.0.0 categories: 248 Non-spacing mark (Mn), 249 Combining spacing mark (Mc), 250 Decimal number (Nd), or 251 Connector punctuation (Pc). 252 253 Codepoint ranges for ES3 Identifiers are extracted from the Unicode 3.0.0 specification at: 254 http://www.unicode.org/Public/3.0-Update/UnicodeData-3.0.0.txt 255 */ 256 const unicodeES3IdentifierStart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 543, 546, 563, 592, 685, 688, 696, 699, 705, 720, 721, 736, 740, 750, 750, 890, 890, 902, 902, 904, 906, 908, 908, 910, 929, 931, 974, 976, 983, 986, 1011, 1024, 1153, 1164, 1220, 1223, 1224, 1227, 1228, 1232, 1269, 1272, 1273, 1329, 1366, 1369, 1369, 1377, 1415, 1488, 1514, 1520, 1522, 1569, 1594, 1600, 1610, 1649, 1747, 1749, 1749, 1765, 1766, 1786, 1788, 1808, 1808, 1810, 1836, 1920, 1957, 2309, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2524, 2525, 2527, 2529, 2544, 2545, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2699, 2701, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2784, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2870, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 2997, 2999, 3001, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3168, 3169, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3294, 3294, 3296, 3297, 3333, 3340, 3342, 3344, 3346, 3368, 3370, 3385, 3424, 3425, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3805, 3840, 3840, 3904, 3911, 3913, 3946, 3976, 3979, 4096, 4129, 4131, 4135, 4137, 4138, 4176, 4181, 4256, 4293, 4304, 4342, 4352, 4441, 4447, 4514, 4520, 4601, 4608, 4614, 4616, 4678, 4680, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4742, 4744, 4744, 4746, 4749, 4752, 4782, 4784, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4814, 4816, 4822, 4824, 4846, 4848, 4878, 4880, 4880, 4882, 4885, 4888, 4894, 4896, 4934, 4936, 4954, 5024, 5108, 5121, 5740, 5743, 5750, 5761, 5786, 5792, 5866, 6016, 6067, 6176, 6263, 6272, 6312, 7680, 7835, 7840, 7929, 7936, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8319, 8319, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8497, 8499, 8505, 8544, 8579, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12346, 12353, 12436, 12445, 12446, 12449, 12538, 12540, 12542, 12549, 12588, 12593, 12686, 12704, 12727, 13312, 19893, 19968, 40869, 40960, 42124, 44032, 55203, 63744, 64045, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65138, 65140, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 257 const unicodeES3IdentifierPart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 543, 546, 563, 592, 685, 688, 696, 699, 705, 720, 721, 736, 740, 750, 750, 768, 846, 864, 866, 890, 890, 902, 902, 904, 906, 908, 908, 910, 929, 931, 974, 976, 983, 986, 1011, 1024, 1153, 1155, 1158, 1164, 1220, 1223, 1224, 1227, 1228, 1232, 1269, 1272, 1273, 1329, 1366, 1369, 1369, 1377, 1415, 1425, 1441, 1443, 1465, 1467, 1469, 1471, 1471, 1473, 1474, 1476, 1476, 1488, 1514, 1520, 1522, 1569, 1594, 1600, 1621, 1632, 1641, 1648, 1747, 1749, 1756, 1759, 1768, 1770, 1773, 1776, 1788, 1808, 1836, 1840, 1866, 1920, 1968, 2305, 2307, 2309, 2361, 2364, 2381, 2384, 2388, 2392, 2403, 2406, 2415, 2433, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2562, 2562, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2649, 2652, 2654, 2654, 2662, 2676, 2689, 2691, 2693, 2699, 2701, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2784, 2790, 2799, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2870, 2873, 2876, 2883, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2913, 2918, 2927, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 2997, 2999, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3031, 3031, 3047, 3055, 3073, 3075, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3168, 3169, 3174, 3183, 3202, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3262, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3297, 3302, 3311, 3330, 3331, 3333, 3340, 3342, 3344, 3346, 3368, 3370, 3385, 3390, 3395, 3398, 3400, 3402, 3405, 3415, 3415, 3424, 3425, 3430, 3439, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3769, 3771, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3805, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3946, 3953, 3972, 3974, 3979, 3984, 3991, 3993, 4028, 4038, 4038, 4096, 4129, 4131, 4135, 4137, 4138, 4140, 4146, 4150, 4153, 4160, 4169, 4176, 4185, 4256, 4293, 4304, 4342, 4352, 4441, 4447, 4514, 4520, 4601, 4608, 4614, 4616, 4678, 4680, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4742, 4744, 4744, 4746, 4749, 4752, 4782, 4784, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4814, 4816, 4822, 4824, 4846, 4848, 4878, 4880, 4880, 4882, 4885, 4888, 4894, 4896, 4934, 4936, 4954, 4969, 4977, 5024, 5108, 5121, 5740, 5743, 5750, 5761, 5786, 5792, 5866, 6016, 6099, 6112, 6121, 6160, 6169, 6176, 6263, 6272, 6313, 7680, 7835, 7840, 7929, 7936, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8255, 8256, 8319, 8319, 8400, 8412, 8417, 8417, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8497, 8499, 8505, 8544, 8579, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12346, 12353, 12436, 12441, 12442, 12445, 12446, 12449, 12542, 12549, 12588, 12593, 12686, 12704, 12727, 13312, 19893, 19968, 40869, 40960, 42124, 44032, 55203, 63744, 64045, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65056, 65059, 65075, 65076, 65101, 65103, 65136, 65138, 65140, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65381, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 258 259 /* 260 As per ECMAScript Language Specification 5th Edition, Section 7.6: ISyntaxToken Names and Identifiers 261 IdentifierStart :: 262 Can contain Unicode 6.2 categories: 263 Uppercase letter (Lu), 264 Lowercase letter (Ll), 265 Titlecase letter (Lt), 266 Modifier letter (Lm), 267 Other letter (Lo), or 268 Letter number (Nl). 269 IdentifierPart :: 270 Can contain IdentifierStart + Unicode 6.2 categories: 271 Non-spacing mark (Mn), 272 Combining spacing mark (Mc), 273 Decimal number (Nd), 274 Connector punctuation (Pc), 275 <ZWNJ>, or 276 <ZWJ>. 277 278 Codepoint ranges for ES5 Identifiers are extracted from the Unicode 6.2 specification at: 279 http://www.unicode.org/Public/6.2.0/ucd/UnicodeData.txt 280 */ 281 const unicodeES5IdentifierStart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1319, 1329, 1366, 1369, 1369, 1377, 1415, 1488, 1514, 1520, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2208, 2208, 2210, 2220, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2423, 2425, 2431, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3133, 3133, 3160, 3161, 3168, 3169, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3294, 3294, 3296, 3297, 3313, 3314, 3333, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3424, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5108, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5872, 5888, 5900, 5902, 5905, 5920, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6263, 6272, 6312, 6314, 6314, 6320, 6389, 6400, 6428, 6480, 6509, 6512, 6516, 6528, 6571, 6593, 6599, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6987, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7401, 7404, 7406, 7409, 7413, 7414, 7424, 7615, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12589, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40908, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42647, 42656, 42735, 42775, 42783, 42786, 42888, 42891, 42894, 42896, 42899, 42912, 42922, 43000, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642, 43648, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43968, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 282 const unicodeES5IdentifierPart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 768, 884, 886, 887, 890, 893, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1155, 1159, 1162, 1319, 1329, 1366, 1369, 1369, 1377, 1415, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1488, 1514, 1520, 1522, 1552, 1562, 1568, 1641, 1646, 1747, 1749, 1756, 1759, 1768, 1770, 1788, 1791, 1791, 1808, 1866, 1869, 1969, 1984, 2037, 2042, 2042, 2048, 2093, 2112, 2139, 2208, 2208, 2210, 2220, 2276, 2302, 2304, 2403, 2406, 2415, 2417, 2423, 2425, 2431, 2433, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504, 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2561, 2563, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2649, 2652, 2654, 2654, 2662, 2677, 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2799, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2876, 2884, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2915, 2918, 2927, 2929, 2929, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024, 3031, 3031, 3046, 3055, 3073, 3075, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3133, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3160, 3161, 3168, 3171, 3174, 3183, 3202, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3299, 3302, 3311, 3313, 3314, 3330, 3331, 3333, 3340, 3342, 3344, 3346, 3386, 3389, 3396, 3398, 3400, 3402, 3406, 3415, 3415, 3424, 3427, 3430, 3439, 3450, 3455, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3769, 3771, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3807, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3948, 3953, 3972, 3974, 3991, 3993, 4028, 4038, 4038, 4096, 4169, 4176, 4253, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4957, 4959, 4992, 5007, 5024, 5108, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5872, 5888, 5900, 5902, 5908, 5920, 5940, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6099, 6103, 6103, 6108, 6109, 6112, 6121, 6155, 6157, 6160, 6169, 6176, 6263, 6272, 6314, 6320, 6389, 6400, 6428, 6432, 6443, 6448, 6459, 6470, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6617, 6656, 6683, 6688, 6750, 6752, 6780, 6783, 6793, 6800, 6809, 6823, 6823, 6912, 6987, 6992, 7001, 7019, 7027, 7040, 7155, 7168, 7223, 7232, 7241, 7245, 7293, 7376, 7378, 7380, 7414, 7424, 7654, 7676, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8204, 8205, 8255, 8256, 8276, 8276, 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8412, 8417, 8417, 8421, 8432, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11647, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11744, 11775, 11823, 11823, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12348, 12353, 12438, 12441, 12442, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12589, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40908, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42539, 42560, 42607, 42612, 42621, 42623, 42647, 42655, 42737, 42775, 42783, 42786, 42888, 42891, 42894, 42896, 42899, 42912, 42922, 43000, 43047, 43072, 43123, 43136, 43204, 43216, 43225, 43232, 43255, 43259, 43259, 43264, 43309, 43312, 43347, 43360, 43388, 43392, 43456, 43471, 43481, 43520, 43574, 43584, 43597, 43600, 43609, 43616, 43638, 43642, 43643, 43648, 43714, 43739, 43741, 43744, 43759, 43762, 43766, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43968, 44010, 44012, 44013, 44016, 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65024, 65039, 65056, 65062, 65075, 65076, 65101, 65103, 65136, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 283 284 /** 285 * Generated by scripts/regenerate-unicode-identifier-parts.js on node v12.4.0 with unicode 12.1 286 * based on http://www.unicode.org/reports/tr31/ and https://www.ecma-international.org/ecma-262/6.0/#sec-names-and-keywords 287 * unicodeESNextIdentifierStart corresponds to the ID_Start and Other_ID_Start property, and 288 * unicodeESNextIdentifierPart corresponds to ID_Continue, Other_ID_Continue, plus ID_Start and Other_ID_Start 289 */ 290 const unicodeESNextIdentifierStart = [65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2208, 2228, 2230, 2237, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3294, 3294, 3296, 3297, 3313, 3314, 3333, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5880, 5888, 5900, 5902, 5905, 5920, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6264, 6272, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6987, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7304, 7312, 7354, 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8472, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, 12443, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40943, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42653, 42656, 42735, 42775, 42783, 42786, 42888, 42891, 42943, 42946, 42950, 42999, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43879, 43888, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65856, 65908, 66176, 66204, 66208, 66256, 66304, 66335, 66349, 66378, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66513, 66517, 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 67072, 67382, 67392, 67413, 67424, 67431, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68096, 68112, 68115, 68117, 68119, 68121, 68149, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68324, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, 68800, 68850, 68864, 68899, 69376, 69404, 69415, 69415, 69424, 69445, 69600, 69622, 69635, 69687, 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480, 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70751, 70784, 70831, 70852, 70853, 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352, 71424, 71450, 71680, 71723, 71840, 71903, 71935, 71935, 72096, 72103, 72106, 72144, 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349, 72349, 72384, 72440, 72704, 72712, 72714, 72750, 72768, 72768, 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73728, 74649, 74752, 74862, 74880, 75075, 77824, 78894, 82944, 83526, 92160, 92728, 92736, 92766, 92880, 92909, 92928, 92975, 92992, 92995, 93027, 93047, 93053, 93071, 93760, 93823, 93952, 94026, 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101106, 110592, 110878, 110928, 110930, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, 123136, 123180, 123191, 123197, 123214, 123214, 123584, 123627, 124928, 125124, 125184, 125251, 125259, 125259, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173782, 173824, 177972, 177984, 178205, 178208, 183969, 183984, 191456, 194560, 195101]; 291 const unicodeESNextIdentifierPart = [48, 57, 65, 90, 95, 95, 97, 122, 170, 170, 181, 181, 183, 183, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 768, 884, 886, 887, 890, 893, 895, 895, 902, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1155, 1159, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1488, 1514, 1519, 1522, 1552, 1562, 1568, 1641, 1646, 1747, 1749, 1756, 1759, 1768, 1770, 1788, 1791, 1791, 1808, 1866, 1869, 1969, 1984, 2037, 2042, 2042, 2045, 2045, 2048, 2093, 2112, 2139, 2144, 2154, 2208, 2228, 2230, 2237, 2259, 2273, 2275, 2403, 2406, 2415, 2417, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504, 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2556, 2556, 2558, 2558, 2561, 2563, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2649, 2652, 2654, 2654, 2662, 2677, 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2799, 2809, 2815, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2876, 2884, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2915, 2918, 2927, 2929, 2929, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024, 3031, 3031, 3046, 3055, 3072, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3160, 3162, 3168, 3171, 3174, 3183, 3200, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3299, 3302, 3311, 3313, 3314, 3328, 3331, 3333, 3340, 3342, 3344, 3346, 3396, 3398, 3400, 3402, 3406, 3412, 3415, 3423, 3427, 3430, 3439, 3450, 3455, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3558, 3567, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3807, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3948, 3953, 3972, 3974, 3991, 3993, 4028, 4038, 4038, 4096, 4169, 4176, 4253, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4957, 4959, 4969, 4977, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5880, 5888, 5900, 5902, 5908, 5920, 5940, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6099, 6103, 6103, 6108, 6109, 6112, 6121, 6155, 6157, 6160, 6169, 6176, 6264, 6272, 6314, 6320, 6389, 6400, 6430, 6432, 6443, 6448, 6459, 6470, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6618, 6656, 6683, 6688, 6750, 6752, 6780, 6783, 6793, 6800, 6809, 6823, 6823, 6832, 6845, 6912, 6987, 6992, 7001, 7019, 7027, 7040, 7155, 7168, 7223, 7232, 7241, 7245, 7293, 7296, 7304, 7312, 7354, 7357, 7359, 7376, 7378, 7380, 7418, 7424, 7673, 7675, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8255, 8256, 8276, 8276, 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8412, 8417, 8417, 8421, 8432, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8472, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11647, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11744, 11775, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12348, 12353, 12438, 12441, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40943, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42539, 42560, 42607, 42612, 42621, 42623, 42737, 42775, 42783, 42786, 42888, 42891, 42943, 42946, 42950, 42999, 43047, 43072, 43123, 43136, 43205, 43216, 43225, 43232, 43255, 43259, 43259, 43261, 43309, 43312, 43347, 43360, 43388, 43392, 43456, 43471, 43481, 43488, 43518, 43520, 43574, 43584, 43597, 43600, 43609, 43616, 43638, 43642, 43714, 43739, 43741, 43744, 43759, 43762, 43766, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43879, 43888, 44010, 44012, 44013, 44016, 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65024, 65039, 65056, 65071, 65075, 65076, 65101, 65103, 65136, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65856, 65908, 66045, 66045, 66176, 66204, 66208, 66256, 66272, 66272, 66304, 66335, 66349, 66378, 66384, 66426, 66432, 66461, 66464, 66499, 66504, 66511, 66513, 66517, 66560, 66717, 66720, 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 67072, 67382, 67392, 67413, 67424, 67431, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68099, 68101, 68102, 68108, 68115, 68117, 68119, 68121, 68149, 68152, 68154, 68159, 68159, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68326, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, 68800, 68850, 68864, 68903, 68912, 68921, 69376, 69404, 69415, 69415, 69424, 69456, 69600, 69622, 69632, 69702, 69734, 69743, 69759, 69818, 69840, 69864, 69872, 69881, 69888, 69940, 69942, 69951, 69956, 69958, 69968, 70003, 70006, 70006, 70016, 70084, 70089, 70092, 70096, 70106, 70108, 70108, 70144, 70161, 70163, 70199, 70206, 70206, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70378, 70384, 70393, 70400, 70403, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70459, 70468, 70471, 70472, 70475, 70477, 70480, 70480, 70487, 70487, 70493, 70499, 70502, 70508, 70512, 70516, 70656, 70730, 70736, 70745, 70750, 70751, 70784, 70853, 70855, 70855, 70864, 70873, 71040, 71093, 71096, 71104, 71128, 71133, 71168, 71232, 71236, 71236, 71248, 71257, 71296, 71352, 71360, 71369, 71424, 71450, 71453, 71467, 71472, 71481, 71680, 71738, 71840, 71913, 71935, 71935, 72096, 72103, 72106, 72151, 72154, 72161, 72163, 72164, 72192, 72254, 72263, 72263, 72272, 72345, 72349, 72349, 72384, 72440, 72704, 72712, 72714, 72758, 72760, 72768, 72784, 72793, 72818, 72847, 72850, 72871, 72873, 72886, 72960, 72966, 72968, 72969, 72971, 73014, 73018, 73018, 73020, 73021, 73023, 73031, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73102, 73104, 73105, 73107, 73112, 73120, 73129, 73440, 73462, 73728, 74649, 74752, 74862, 74880, 75075, 77824, 78894, 82944, 83526, 92160, 92728, 92736, 92766, 92768, 92777, 92880, 92909, 92912, 92916, 92928, 92982, 92992, 92995, 93008, 93017, 93027, 93047, 93053, 93071, 93760, 93823, 93952, 94026, 94031, 94087, 94095, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101106, 110592, 110878, 110928, 110930, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 113821, 113822, 119141, 119145, 119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, 120782, 120831, 121344, 121398, 121403, 121452, 121461, 121461, 121476, 121476, 121499, 121503, 121505, 121519, 122880, 122886, 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 123136, 123180, 123184, 123197, 123200, 123209, 123214, 123214, 123584, 123641, 124928, 125124, 125136, 125142, 125184, 125259, 125264, 125273, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173782, 173824, 177972, 177984, 178205, 178208, 183969, 183984, 191456, 194560, 195101, 917760, 917999]; 292 293 /** 294 * Test for whether a single line comment with leading whitespace trimmed's text contains a directive. 295 */ 296 const commentDirectiveRegExSingleLine = /^\/\/\/?\s*@(ts-expect-error|ts-ignore)/; 297 298 /** 299 * Test for whether a multi-line comment with leading whitespace trimmed's last line contains a directive. 300 */ 301 const commentDirectiveRegExMultiLine = /^(?:\/|\*)*\s*@(ts-expect-error|ts-ignore)/; 302 303 function lookupInUnicodeMap(code: number, map: readonly number[]): boolean { 304 // Bail out quickly if it couldn't possibly be in the map. 305 if (code < map[0]) { 306 return false; 307 } 308 309 // Perform binary search in one of the Unicode range maps 310 let lo = 0; 311 let hi: number = map.length; 312 let mid: number; 313 314 while (lo + 1 < hi) { 315 mid = lo + (hi - lo) / 2; 316 // mid has to be even to catch a range's beginning 317 mid -= mid % 2; 318 if (map[mid] <= code && code <= map[mid + 1]) { 319 return true; 320 } 321 322 if (code < map[mid]) { 323 hi = mid; 324 } 325 else { 326 lo = mid + 2; 327 } 328 } 329 330 return false; 331 } 332 333 /* @internal */ export function isUnicodeIdentifierStart(code: number, languageVersion: ScriptTarget | undefined) { 334 return languageVersion! >= ScriptTarget.ES2015 ? 335 lookupInUnicodeMap(code, unicodeESNextIdentifierStart) : 336 languageVersion === ScriptTarget.ES5 ? lookupInUnicodeMap(code, unicodeES5IdentifierStart) : 337 lookupInUnicodeMap(code, unicodeES3IdentifierStart); 338 } 339 340 function isUnicodeIdentifierPart(code: number, languageVersion: ScriptTarget | undefined) { 341 return languageVersion! >= ScriptTarget.ES2015 ? 342 lookupInUnicodeMap(code, unicodeESNextIdentifierPart) : 343 languageVersion === ScriptTarget.ES5 ? lookupInUnicodeMap(code, unicodeES5IdentifierPart) : 344 lookupInUnicodeMap(code, unicodeES3IdentifierPart); 345 } 346 347 function makeReverseMap(source: ESMap<string, number>): string[] { 348 const result: string[] = []; 349 source.forEach((value, name) => { 350 result[value] = name; 351 }); 352 return result; 353 } 354 355 const tokenStrings = makeReverseMap(textToToken); 356 export function tokenToString(t: SyntaxKind): string | undefined { 357 return tokenStrings[t]; 358 } 359 360 /* @internal */ 361 export function stringToToken(s: string): SyntaxKind | undefined { 362 return textToToken.get(s); 363 } 364 365 /* @internal */ 366 export function computeLineStarts(text: string): number[] { 367 const result: number[] = []; 368 let pos = 0; 369 let lineStart = 0; 370 while (pos < text.length) { 371 const ch = text.charCodeAt(pos); 372 pos++; 373 switch (ch) { 374 case CharacterCodes.carriageReturn: 375 if (text.charCodeAt(pos) === CharacterCodes.lineFeed) { 376 pos++; 377 } 378 // falls through 379 case CharacterCodes.lineFeed: 380 result.push(lineStart); 381 lineStart = pos; 382 break; 383 default: 384 if (ch > CharacterCodes.maxAsciiCharacter && isLineBreak(ch)) { 385 result.push(lineStart); 386 lineStart = pos; 387 } 388 break; 389 } 390 } 391 result.push(lineStart); 392 return result; 393 } 394 395 export function getPositionOfLineAndCharacter(sourceFile: SourceFileLike, line: number, character: number): number; 396 /* @internal */ 397 export function getPositionOfLineAndCharacter(sourceFile: SourceFileLike, line: number, character: number, allowEdits?: true): number; // eslint-disable-line @typescript-eslint/unified-signatures 398 export function getPositionOfLineAndCharacter(sourceFile: SourceFileLike, line: number, character: number, allowEdits?: true): number { 399 return sourceFile.getPositionOfLineAndCharacter ? 400 sourceFile.getPositionOfLineAndCharacter(line, character, allowEdits) : 401 computePositionOfLineAndCharacter(getLineStarts(sourceFile), line, character, sourceFile.text, allowEdits); 402 } 403 404 /* @internal */ 405 export function computePositionOfLineAndCharacter(lineStarts: readonly number[], line: number, character: number, debugText?: string, allowEdits?: true): number { 406 if (line < 0 || line >= lineStarts.length) { 407 if (allowEdits) { 408 // Clamp line to nearest allowable value 409 line = line < 0 ? 0 : line >= lineStarts.length ? lineStarts.length - 1 : line; 410 } 411 else { 412 Debug.fail(`Bad line number. Line: ${line}, lineStarts.length: ${lineStarts.length} , line map is correct? ${debugText !== undefined ? arraysEqual(lineStarts, computeLineStarts(debugText)) : "unknown"}`); 413 } 414 } 415 416 const res = lineStarts[line] + character; 417 if (allowEdits) { 418 // Clamp to nearest allowable values to allow the underlying to be edited without crashing (accuracy is lost, instead) 419 // TODO: Somehow track edits between file as it was during the creation of sourcemap we have and the current file and 420 // apply them to the computed position to improve accuracy 421 return res > lineStarts[line + 1] ? lineStarts[line + 1] : typeof debugText === "string" && res > debugText.length ? debugText.length : res; 422 } 423 if (line < lineStarts.length - 1) { 424 Debug.assert(res < lineStarts[line + 1]); 425 } 426 else if (debugText !== undefined) { 427 Debug.assert(res <= debugText.length); // Allow single character overflow for trailing newline 428 } 429 return res; 430 } 431 432 /* @internal */ 433 export function getLineStarts(sourceFile: SourceFileLike): readonly number[] { 434 return sourceFile.lineMap || (sourceFile.lineMap = computeLineStarts(sourceFile.text)); 435 } 436 437 /* @internal */ 438 export function computeLineAndCharacterOfPosition(lineStarts: readonly number[], position: number): LineAndCharacter { 439 const lineNumber = computeLineOfPosition(lineStarts, position); 440 return { 441 line: lineNumber, 442 character: position - lineStarts[lineNumber] 443 }; 444 } 445 446 /** 447 * @internal 448 * We assume the first line starts at position 0 and 'position' is non-negative. 449 */ 450 export function computeLineOfPosition(lineStarts: readonly number[], position: number, lowerBound?: number) { 451 let lineNumber = binarySearch(lineStarts, position, identity, compareValues, lowerBound); 452 if (lineNumber < 0) { 453 // If the actual position was not found, 454 // the binary search returns the 2's-complement of the next line start 455 // e.g. if the line starts at [5, 10, 23, 80] and the position requested was 20 456 // then the search will return -2. 457 // 458 // We want the index of the previous line start, so we subtract 1. 459 // Review 2's-complement if this is confusing. 460 lineNumber = ~lineNumber - 1; 461 Debug.assert(lineNumber !== -1, "position cannot precede the beginning of the file"); 462 } 463 return lineNumber; 464 } 465 466 /** @internal */ 467 export function getLinesBetweenPositions(sourceFile: SourceFileLike, pos1: number, pos2: number) { 468 if (pos1 === pos2) return 0; 469 const lineStarts = getLineStarts(sourceFile); 470 const lower = Math.min(pos1, pos2); 471 const isNegative = lower === pos2; 472 const upper = isNegative ? pos1 : pos2; 473 const lowerLine = computeLineOfPosition(lineStarts, lower); 474 const upperLine = computeLineOfPosition(lineStarts, upper, lowerLine); 475 return isNegative ? lowerLine - upperLine : upperLine - lowerLine; 476 } 477 478 export function getLineAndCharacterOfPosition(sourceFile: SourceFileLike, position: number): LineAndCharacter { 479 return computeLineAndCharacterOfPosition(getLineStarts(sourceFile), position); 480 } 481 482 export function isWhiteSpaceLike(ch: number): boolean { 483 return isWhiteSpaceSingleLine(ch) || isLineBreak(ch); 484 } 485 486 /** Does not include line breaks. For that, see isWhiteSpaceLike. */ 487 export function isWhiteSpaceSingleLine(ch: number): boolean { 488 // Note: nextLine is in the Zs space, and should be considered to be a whitespace. 489 // It is explicitly not a line-break as it isn't in the exact set specified by EcmaScript. 490 return ch === CharacterCodes.space || 491 ch === CharacterCodes.tab || 492 ch === CharacterCodes.verticalTab || 493 ch === CharacterCodes.formFeed || 494 ch === CharacterCodes.nonBreakingSpace || 495 ch === CharacterCodes.nextLine || 496 ch === CharacterCodes.ogham || 497 ch >= CharacterCodes.enQuad && ch <= CharacterCodes.zeroWidthSpace || 498 ch === CharacterCodes.narrowNoBreakSpace || 499 ch === CharacterCodes.mathematicalSpace || 500 ch === CharacterCodes.ideographicSpace || 501 ch === CharacterCodes.byteOrderMark; 502 } 503 504 export function isLineBreak(ch: number): boolean { 505 // ES5 7.3: 506 // The ECMAScript line terminator characters are listed in Table 3. 507 // Table 3: Line Terminator Characters 508 // Code Unit Value Name Formal Name 509 // \u000A Line Feed <LF> 510 // \u000D Carriage Return <CR> 511 // \u2028 Line separator <LS> 512 // \u2029 Paragraph separator <PS> 513 // Only the characters in Table 3 are treated as line terminators. Other new line or line 514 // breaking characters are treated as white space but not as line terminators. 515 516 return ch === CharacterCodes.lineFeed || 517 ch === CharacterCodes.carriageReturn || 518 ch === CharacterCodes.lineSeparator || 519 ch === CharacterCodes.paragraphSeparator; 520 } 521 522 function isDigit(ch: number): boolean { 523 return ch >= CharacterCodes._0 && ch <= CharacterCodes._9; 524 } 525 526 function isHexDigit(ch: number): boolean { 527 return isDigit(ch) || ch >= CharacterCodes.A && ch <= CharacterCodes.F || ch >= CharacterCodes.a && ch <= CharacterCodes.f; 528 } 529 530 function isCodePoint(code: number): boolean { 531 return code <= 0x10FFFF; 532 } 533 534 /* @internal */ 535 export function isOctalDigit(ch: number): boolean { 536 return ch >= CharacterCodes._0 && ch <= CharacterCodes._7; 537 } 538 539 export function couldStartTrivia(text: string, pos: number): boolean { 540 // Keep in sync with skipTrivia 541 const ch = text.charCodeAt(pos); 542 switch (ch) { 543 case CharacterCodes.carriageReturn: 544 case CharacterCodes.lineFeed: 545 case CharacterCodes.tab: 546 case CharacterCodes.verticalTab: 547 case CharacterCodes.formFeed: 548 case CharacterCodes.space: 549 case CharacterCodes.slash: 550 // starts of normal trivia 551 // falls through 552 case CharacterCodes.lessThan: 553 case CharacterCodes.bar: 554 case CharacterCodes.equals: 555 case CharacterCodes.greaterThan: 556 // Starts of conflict marker trivia 557 return true; 558 case CharacterCodes.hash: 559 // Only if its the beginning can we have #! trivia 560 return pos === 0; 561 default: 562 return ch > CharacterCodes.maxAsciiCharacter; 563 } 564 } 565 566 /* @internal */ 567 export function skipTrivia(text: string, pos: number, stopAfterLineBreak?: boolean, stopAtComments?: boolean, inJSDoc?: boolean): number { 568 if (positionIsSynthesized(pos)) { 569 return pos; 570 } 571 572 let canConsumeStar = false; 573 // Keep in sync with couldStartTrivia 574 while (true) { 575 const ch = text.charCodeAt(pos); 576 switch (ch) { 577 case CharacterCodes.carriageReturn: 578 if (text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { 579 pos++; 580 } 581 // falls through 582 case CharacterCodes.lineFeed: 583 pos++; 584 if (stopAfterLineBreak) { 585 return pos; 586 } 587 canConsumeStar = !!inJSDoc; 588 continue; 589 case CharacterCodes.tab: 590 case CharacterCodes.verticalTab: 591 case CharacterCodes.formFeed: 592 case CharacterCodes.space: 593 pos++; 594 continue; 595 case CharacterCodes.slash: 596 if (stopAtComments) { 597 break; 598 } 599 if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { 600 pos += 2; 601 while (pos < text.length) { 602 if (isLineBreak(text.charCodeAt(pos))) { 603 break; 604 } 605 pos++; 606 } 607 canConsumeStar = false; 608 continue; 609 } 610 if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { 611 pos += 2; 612 while (pos < text.length) { 613 if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { 614 pos += 2; 615 break; 616 } 617 pos++; 618 } 619 canConsumeStar = false; 620 continue; 621 } 622 break; 623 624 case CharacterCodes.lessThan: 625 case CharacterCodes.bar: 626 case CharacterCodes.equals: 627 case CharacterCodes.greaterThan: 628 if (isConflictMarkerTrivia(text, pos)) { 629 pos = scanConflictMarkerTrivia(text, pos); 630 canConsumeStar = false; 631 continue; 632 } 633 break; 634 635 case CharacterCodes.hash: 636 if (pos === 0 && isShebangTrivia(text, pos)) { 637 pos = scanShebangTrivia(text, pos); 638 canConsumeStar = false; 639 continue; 640 } 641 break; 642 643 case CharacterCodes.asterisk: 644 if (canConsumeStar) { 645 pos++; 646 canConsumeStar = false; 647 continue; 648 } 649 break; 650 651 default: 652 if (ch > CharacterCodes.maxAsciiCharacter && (isWhiteSpaceLike(ch))) { 653 pos++; 654 continue; 655 } 656 break; 657 } 658 return pos; 659 } 660 } 661 662 // All conflict markers consist of the same character repeated seven times. If it is 663 // a <<<<<<< or >>>>>>> marker then it is also followed by a space. 664 const mergeConflictMarkerLength = "<<<<<<<".length; 665 666 function isConflictMarkerTrivia(text: string, pos: number) { 667 Debug.assert(pos >= 0); 668 669 // Conflict markers must be at the start of a line. 670 if (pos === 0 || isLineBreak(text.charCodeAt(pos - 1))) { 671 const ch = text.charCodeAt(pos); 672 673 if ((pos + mergeConflictMarkerLength) < text.length) { 674 for (let i = 0; i < mergeConflictMarkerLength; i++) { 675 if (text.charCodeAt(pos + i) !== ch) { 676 return false; 677 } 678 } 679 680 return ch === CharacterCodes.equals || 681 text.charCodeAt(pos + mergeConflictMarkerLength) === CharacterCodes.space; 682 } 683 } 684 685 return false; 686 } 687 688 function scanConflictMarkerTrivia(text: string, pos: number, error?: (diag: DiagnosticMessage, pos?: number, len?: number) => void) { 689 if (error) { 690 error(Diagnostics.Merge_conflict_marker_encountered, pos, mergeConflictMarkerLength); 691 } 692 693 const ch = text.charCodeAt(pos); 694 const len = text.length; 695 696 if (ch === CharacterCodes.lessThan || ch === CharacterCodes.greaterThan) { 697 while (pos < len && !isLineBreak(text.charCodeAt(pos))) { 698 pos++; 699 } 700 } 701 else { 702 Debug.assert(ch === CharacterCodes.bar || ch === CharacterCodes.equals); 703 // Consume everything from the start of a ||||||| or ======= marker to the start 704 // of the next ======= or >>>>>>> marker. 705 while (pos < len) { 706 const currentChar = text.charCodeAt(pos); 707 if ((currentChar === CharacterCodes.equals || currentChar === CharacterCodes.greaterThan) && currentChar !== ch && isConflictMarkerTrivia(text, pos)) { 708 break; 709 } 710 711 pos++; 712 } 713 } 714 715 return pos; 716 } 717 718 const shebangTriviaRegex = /^#!.*/; 719 720 /*@internal*/ 721 export function isShebangTrivia(text: string, pos: number) { 722 // Shebangs check must only be done at the start of the file 723 Debug.assert(pos === 0); 724 return shebangTriviaRegex.test(text); 725 } 726 727 /*@internal*/ 728 export function scanShebangTrivia(text: string, pos: number) { 729 const shebang = shebangTriviaRegex.exec(text)![0]; 730 pos = pos + shebang.length; 731 return pos; 732 } 733 734 /** 735 * Invokes a callback for each comment range following the provided position. 736 * 737 * Single-line comment ranges include the leading double-slash characters but not the ending 738 * line break. Multi-line comment ranges include the leading slash-asterisk and trailing 739 * asterisk-slash characters. 740 * 741 * @param reduce If true, accumulates the result of calling the callback in a fashion similar 742 * to reduceLeft. If false, iteration stops when the callback returns a truthy value. 743 * @param text The source text to scan. 744 * @param pos The position at which to start scanning. 745 * @param trailing If false, whitespace is skipped until the first line break and comments 746 * between that location and the next token are returned. If true, comments occurring 747 * between the given position and the next line break are returned. 748 * @param cb The callback to execute as each comment range is encountered. 749 * @param state A state value to pass to each iteration of the callback. 750 * @param initial An initial value to pass when accumulating results (when "reduce" is true). 751 * @returns If "reduce" is true, the accumulated value. If "reduce" is false, the first truthy 752 * return value of the callback. 753 */ 754 function iterateCommentRanges<T, U>(reduce: boolean, text: string, pos: number, trailing: boolean, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U | undefined) => U, state: T, initial?: U): U | undefined { 755 let pendingPos!: number; 756 let pendingEnd!: number; 757 let pendingKind!: CommentKind; 758 let pendingHasTrailingNewLine!: boolean; 759 let hasPendingCommentRange = false; 760 let collecting = trailing; 761 let accumulator = initial; 762 if (pos === 0) { 763 collecting = true; 764 const shebang = getShebang(text); 765 if (shebang) { 766 pos = shebang.length; 767 } 768 } 769 scan: while (pos >= 0 && pos < text.length) { 770 const ch = text.charCodeAt(pos); 771 switch (ch) { 772 case CharacterCodes.carriageReturn: 773 if (text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { 774 pos++; 775 } 776 // falls through 777 case CharacterCodes.lineFeed: 778 pos++; 779 if (trailing) { 780 break scan; 781 } 782 783 collecting = true; 784 if (hasPendingCommentRange) { 785 pendingHasTrailingNewLine = true; 786 } 787 788 continue; 789 case CharacterCodes.tab: 790 case CharacterCodes.verticalTab: 791 case CharacterCodes.formFeed: 792 case CharacterCodes.space: 793 pos++; 794 continue; 795 case CharacterCodes.slash: 796 const nextChar = text.charCodeAt(pos + 1); 797 let hasTrailingNewLine = false; 798 if (nextChar === CharacterCodes.slash || nextChar === CharacterCodes.asterisk) { 799 const kind = nextChar === CharacterCodes.slash ? SyntaxKind.SingleLineCommentTrivia : SyntaxKind.MultiLineCommentTrivia; 800 const startPos = pos; 801 pos += 2; 802 if (nextChar === CharacterCodes.slash) { 803 while (pos < text.length) { 804 if (isLineBreak(text.charCodeAt(pos))) { 805 hasTrailingNewLine = true; 806 break; 807 } 808 pos++; 809 } 810 } 811 else { 812 while (pos < text.length) { 813 if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { 814 pos += 2; 815 break; 816 } 817 pos++; 818 } 819 } 820 821 if (collecting) { 822 if (hasPendingCommentRange) { 823 accumulator = cb(pendingPos, pendingEnd, pendingKind, pendingHasTrailingNewLine, state, accumulator); 824 if (!reduce && accumulator) { 825 // If we are not reducing and we have a truthy result, return it. 826 return accumulator; 827 } 828 } 829 830 pendingPos = startPos; 831 pendingEnd = pos; 832 pendingKind = kind; 833 pendingHasTrailingNewLine = hasTrailingNewLine; 834 hasPendingCommentRange = true; 835 } 836 837 continue; 838 } 839 break scan; 840 default: 841 if (ch > CharacterCodes.maxAsciiCharacter && (isWhiteSpaceLike(ch))) { 842 if (hasPendingCommentRange && isLineBreak(ch)) { 843 pendingHasTrailingNewLine = true; 844 } 845 pos++; 846 continue; 847 } 848 break scan; 849 } 850 } 851 852 if (hasPendingCommentRange) { 853 accumulator = cb(pendingPos, pendingEnd, pendingKind, pendingHasTrailingNewLine, state, accumulator); 854 } 855 856 return accumulator; 857 } 858 859 export function forEachLeadingCommentRange<U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean) => U): U | undefined; 860 export function forEachLeadingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state: T): U | undefined; 861 export function forEachLeadingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state?: T): U | undefined { 862 return iterateCommentRanges(/*reduce*/ false, text, pos, /*trailing*/ false, cb, state); 863 } 864 865 export function forEachTrailingCommentRange<U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean) => U): U | undefined; 866 export function forEachTrailingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state: T): U | undefined; 867 export function forEachTrailingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state?: T): U | undefined { 868 return iterateCommentRanges(/*reduce*/ false, text, pos, /*trailing*/ true, cb, state); 869 } 870 871 export function reduceEachLeadingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { 872 return iterateCommentRanges(/*reduce*/ true, text, pos, /*trailing*/ false, cb, state, initial); 873 } 874 875 export function reduceEachTrailingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { 876 return iterateCommentRanges(/*reduce*/ true, text, pos, /*trailing*/ true, cb, state, initial); 877 } 878 879 function appendCommentRange(pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, _state: any, comments: CommentRange[]) { 880 if (!comments) { 881 comments = []; 882 } 883 884 comments.push({ kind, pos, end, hasTrailingNewLine }); 885 return comments; 886 } 887 888 export function getLeadingCommentRanges(text: string, pos: number): CommentRange[] | undefined { 889 return reduceEachLeadingCommentRange(text, pos, appendCommentRange, /*state*/ undefined, /*initial*/ undefined); 890 } 891 892 export function getTrailingCommentRanges(text: string, pos: number): CommentRange[] | undefined { 893 return reduceEachTrailingCommentRange(text, pos, appendCommentRange, /*state*/ undefined, /*initial*/ undefined); 894 } 895 896 /** Optionally, get the shebang */ 897 export function getShebang(text: string): string | undefined { 898 const match = shebangTriviaRegex.exec(text); 899 if (match) { 900 return match[0]; 901 } 902 } 903 904 export function isIdentifierStart(ch: number, languageVersion: ScriptTarget | undefined): boolean { 905 return ch >= CharacterCodes.A && ch <= CharacterCodes.Z || ch >= CharacterCodes.a && ch <= CharacterCodes.z || 906 ch === CharacterCodes.$ || ch === CharacterCodes._ || 907 ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierStart(ch, languageVersion); 908 } 909 910 export function isIdentifierPart(ch: number, languageVersion: ScriptTarget | undefined, identifierVariant?: LanguageVariant): boolean { 911 return ch >= CharacterCodes.A && ch <= CharacterCodes.Z || ch >= CharacterCodes.a && ch <= CharacterCodes.z || 912 ch >= CharacterCodes._0 && ch <= CharacterCodes._9 || ch === CharacterCodes.$ || ch === CharacterCodes._ || 913 // "-" and ":" are valid in JSX Identifiers 914 (identifierVariant === LanguageVariant.JSX ? (ch === CharacterCodes.minus || ch === CharacterCodes.colon) : false) || 915 ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion); 916 } 917 918 /* @internal */ 919 export function isIdentifierText(name: string, languageVersion: ScriptTarget | undefined, identifierVariant?: LanguageVariant): boolean { 920 let ch = codePointAt(name, 0); 921 if (!isIdentifierStart(ch, languageVersion)) { 922 return false; 923 } 924 925 for (let i = charSize(ch); i < name.length; i += charSize(ch)) { 926 if (!isIdentifierPart(ch = codePointAt(name, i), languageVersion, identifierVariant)) { 927 return false; 928 } 929 } 930 931 return true; 932 } 933 934 // Creates a scanner over a (possibly unspecified) range of a piece of text. 935 export function createScanner(languageVersion: ScriptTarget, 936 skipTrivia: boolean, 937 languageVariant = LanguageVariant.Standard, 938 textInitial?: string, 939 onError?: ErrorCallback, 940 start?: number, 941 length?: number): Scanner { 942 943 let text = textInitial!; 944 945 // Current position (end position of text of current token) 946 let pos: number; 947 948 949 // end of text 950 let end: number; 951 952 // Start position of whitespace before current token 953 let startPos: number; 954 955 // Start position of text of current token 956 let tokenPos: number; 957 958 let token: SyntaxKind; 959 let tokenValue!: string; 960 let tokenFlags: TokenFlags; 961 962 let commentDirectives: CommentDirective[] | undefined; 963 let inJSDocType = 0; 964 let inEtsContext: boolean = false; 965 966 setText(text, start, length); 967 968 const scanner: Scanner = { 969 getStartPos: () => startPos, 970 getTextPos: () => pos, 971 getToken: () => token, 972 getTokenPos: () => tokenPos, 973 getTokenText: () => text.substring(tokenPos, pos), 974 getTokenValue: () => tokenValue, 975 hasUnicodeEscape: () => (tokenFlags & TokenFlags.UnicodeEscape) !== 0, 976 hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0, 977 hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0, 978 hasPrecedingJSDocComment: () => (tokenFlags & TokenFlags.PrecedingJSDocComment) !== 0, 979 isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord, 980 isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord, 981 isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0, 982 getCommentDirectives: () => commentDirectives, 983 getNumericLiteralFlags: () => tokenFlags & TokenFlags.NumericLiteralFlags, 984 getTokenFlags: () => tokenFlags, 985 reScanGreaterToken, 986 reScanAsteriskEqualsToken, 987 reScanSlashToken, 988 reScanTemplateToken, 989 reScanTemplateHeadOrNoSubstitutionTemplate, 990 scanJsxIdentifier, 991 scanJsxAttributeValue, 992 reScanJsxAttributeValue, 993 reScanJsxToken, 994 reScanLessThanToken, 995 reScanHashToken, 996 reScanQuestionToken, 997 reScanInvalidIdentifier, 998 scanJsxToken, 999 scanJsDocToken, 1000 scan, 1001 getText, 1002 clearCommentDirectives, 1003 setText, 1004 setScriptTarget, 1005 setLanguageVariant, 1006 setOnError, 1007 setTextPos, 1008 setInJSDocType, 1009 tryScan, 1010 lookAhead, 1011 scanRange, 1012 setEtsContext 1013 }; 1014 1015 if (Debug.isDebugging) { 1016 Object.defineProperty(scanner, "__debugShowCurrentPositionInText", { 1017 get: () => { 1018 const text = scanner.getText(); 1019 return text.slice(0, scanner.getStartPos()) + "║" + text.slice(scanner.getStartPos()); 1020 }, 1021 }); 1022 } 1023 1024 return scanner; 1025 1026 function setEtsContext(isEtsContext: boolean): void { 1027 inEtsContext = isEtsContext; 1028 } 1029 1030 function error(message: DiagnosticMessage): void; 1031 function error(message: DiagnosticMessage, errPos: number, length: number): void; 1032 function error(message: DiagnosticMessage, errPos: number = pos, length?: number): void { 1033 if (onError) { 1034 const oldPos = pos; 1035 pos = errPos; 1036 onError(message, length || 0); 1037 pos = oldPos; 1038 } 1039 } 1040 1041 function scanNumberFragment(): string { 1042 let start = pos; 1043 let allowSeparator = false; 1044 let isPreviousTokenSeparator = false; 1045 let result = ""; 1046 while (true) { 1047 const ch = text.charCodeAt(pos); 1048 if (ch === CharacterCodes._) { 1049 tokenFlags |= TokenFlags.ContainsSeparator; 1050 if (allowSeparator) { 1051 allowSeparator = false; 1052 isPreviousTokenSeparator = true; 1053 result += text.substring(start, pos); 1054 } 1055 else if (isPreviousTokenSeparator) { 1056 error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1); 1057 } 1058 else { 1059 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1); 1060 } 1061 pos++; 1062 start = pos; 1063 continue; 1064 } 1065 if (isDigit(ch)) { 1066 allowSeparator = true; 1067 isPreviousTokenSeparator = false; 1068 pos++; 1069 continue; 1070 } 1071 break; 1072 } 1073 if (text.charCodeAt(pos - 1) === CharacterCodes._) { 1074 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1); 1075 } 1076 return result + text.substring(start, pos); 1077 } 1078 1079 function scanNumber(): { type: SyntaxKind, value: string } { 1080 const start = pos; 1081 const mainFragment = scanNumberFragment(); 1082 let decimalFragment: string | undefined; 1083 let scientificFragment: string | undefined; 1084 if (text.charCodeAt(pos) === CharacterCodes.dot) { 1085 pos++; 1086 decimalFragment = scanNumberFragment(); 1087 } 1088 let end = pos; 1089 if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) { 1090 pos++; 1091 tokenFlags |= TokenFlags.Scientific; 1092 if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++; 1093 const preNumericPart = pos; 1094 const finalFragment = scanNumberFragment(); 1095 if (!finalFragment) { 1096 error(Diagnostics.Digit_expected); 1097 } 1098 else { 1099 scientificFragment = text.substring(end, preNumericPart) + finalFragment; 1100 end = pos; 1101 } 1102 } 1103 let result: string; 1104 if (tokenFlags & TokenFlags.ContainsSeparator) { 1105 result = mainFragment; 1106 if (decimalFragment) { 1107 result += "." + decimalFragment; 1108 } 1109 if (scientificFragment) { 1110 result += scientificFragment; 1111 } 1112 } 1113 else { 1114 result = text.substring(start, end); // No need to use all the fragments; no _ removal needed 1115 } 1116 1117 if (decimalFragment !== undefined || tokenFlags & TokenFlags.Scientific) { 1118 checkForIdentifierStartAfterNumericLiteral(start, decimalFragment === undefined && !!(tokenFlags & TokenFlags.Scientific)); 1119 return { 1120 type: SyntaxKind.NumericLiteral, 1121 value: "" + +result // if value is not an integer, it can be safely coerced to a number 1122 }; 1123 } 1124 else { 1125 tokenValue = result; 1126 const type = checkBigIntSuffix(); // if value is an integer, check whether it is a bigint 1127 checkForIdentifierStartAfterNumericLiteral(start); 1128 return { type, value: tokenValue }; 1129 } 1130 } 1131 1132 function checkForIdentifierStartAfterNumericLiteral(numericStart: number, isScientific?: boolean) { 1133 if (!isIdentifierStart(codePointAt(text, pos), languageVersion)) { 1134 return; 1135 } 1136 1137 const identifierStart = pos; 1138 const { length } = scanIdentifierParts(); 1139 1140 if (length === 1 && text[identifierStart] === "n") { 1141 if (isScientific) { 1142 error(Diagnostics.A_bigint_literal_cannot_use_exponential_notation, numericStart, identifierStart - numericStart + 1); 1143 } 1144 else { 1145 error(Diagnostics.A_bigint_literal_must_be_an_integer, numericStart, identifierStart - numericStart + 1); 1146 } 1147 } 1148 else { 1149 error(Diagnostics.An_identifier_or_keyword_cannot_immediately_follow_a_numeric_literal, identifierStart, length); 1150 pos = identifierStart; 1151 } 1152 } 1153 1154 function scanOctalDigits(): number { 1155 const start = pos; 1156 while (isOctalDigit(text.charCodeAt(pos))) { 1157 pos++; 1158 } 1159 return +(text.substring(start, pos)); 1160 } 1161 1162 /** 1163 * Scans the given number of hexadecimal digits in the text, 1164 * returning -1 if the given number is unavailable. 1165 */ 1166 function scanExactNumberOfHexDigits(count: number, canHaveSeparators: boolean): number { 1167 const valueString = scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ false, canHaveSeparators); 1168 return valueString ? parseInt(valueString, 16) : -1; 1169 } 1170 1171 /** 1172 * Scans as many hexadecimal digits as are available in the text, 1173 * returning "" if the given number of digits was unavailable. 1174 */ 1175 function scanMinimumNumberOfHexDigits(count: number, canHaveSeparators: boolean): string { 1176 return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ true, canHaveSeparators); 1177 } 1178 1179 function scanHexDigits(minCount: number, scanAsManyAsPossible: boolean, canHaveSeparators: boolean): string { 1180 let valueChars: number[] = []; 1181 let allowSeparator = false; 1182 let isPreviousTokenSeparator = false; 1183 while (valueChars.length < minCount || scanAsManyAsPossible) { 1184 let ch = text.charCodeAt(pos); 1185 if (canHaveSeparators && ch === CharacterCodes._) { 1186 tokenFlags |= TokenFlags.ContainsSeparator; 1187 if (allowSeparator) { 1188 allowSeparator = false; 1189 isPreviousTokenSeparator = true; 1190 } 1191 else if (isPreviousTokenSeparator) { 1192 error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1); 1193 } 1194 else { 1195 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1); 1196 } 1197 pos++; 1198 continue; 1199 } 1200 allowSeparator = canHaveSeparators; 1201 if (ch >= CharacterCodes.A && ch <= CharacterCodes.F) { 1202 ch += CharacterCodes.a - CharacterCodes.A; // standardize hex literals to lowercase 1203 } 1204 else if (!((ch >= CharacterCodes._0 && ch <= CharacterCodes._9) || 1205 (ch >= CharacterCodes.a && ch <= CharacterCodes.f) 1206 )) { 1207 break; 1208 } 1209 valueChars.push(ch); 1210 pos++; 1211 isPreviousTokenSeparator = false; 1212 } 1213 if (valueChars.length < minCount) { 1214 valueChars = []; 1215 } 1216 if (text.charCodeAt(pos - 1) === CharacterCodes._) { 1217 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1); 1218 } 1219 return String.fromCharCode(...valueChars); 1220 } 1221 1222 function scanString(jsxAttributeString = false): string { 1223 const quote = text.charCodeAt(pos); 1224 pos++; 1225 let result = ""; 1226 let start = pos; 1227 while (true) { 1228 if (pos >= end) { 1229 result += text.substring(start, pos); 1230 tokenFlags |= TokenFlags.Unterminated; 1231 error(Diagnostics.Unterminated_string_literal); 1232 break; 1233 } 1234 const ch = text.charCodeAt(pos); 1235 if (ch === quote) { 1236 result += text.substring(start, pos); 1237 pos++; 1238 break; 1239 } 1240 if (ch === CharacterCodes.backslash && !jsxAttributeString) { 1241 result += text.substring(start, pos); 1242 result += scanEscapeSequence(); 1243 start = pos; 1244 continue; 1245 } 1246 if (isLineBreak(ch) && !jsxAttributeString) { 1247 result += text.substring(start, pos); 1248 tokenFlags |= TokenFlags.Unterminated; 1249 error(Diagnostics.Unterminated_string_literal); 1250 break; 1251 } 1252 pos++; 1253 } 1254 return result; 1255 } 1256 1257 /** 1258 * Sets the current 'tokenValue' and returns a NoSubstitutionTemplateLiteral or 1259 * a literal component of a TemplateExpression. 1260 */ 1261 function scanTemplateAndSetTokenValue(isTaggedTemplate: boolean): SyntaxKind { 1262 const startedWithBacktick = text.charCodeAt(pos) === CharacterCodes.backtick; 1263 1264 pos++; 1265 let start = pos; 1266 let contents = ""; 1267 let resultingToken: SyntaxKind; 1268 1269 while (true) { 1270 if (pos >= end) { 1271 contents += text.substring(start, pos); 1272 tokenFlags |= TokenFlags.Unterminated; 1273 error(Diagnostics.Unterminated_template_literal); 1274 resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail; 1275 break; 1276 } 1277 1278 const currChar = text.charCodeAt(pos); 1279 1280 // '`' 1281 if (currChar === CharacterCodes.backtick) { 1282 contents += text.substring(start, pos); 1283 pos++; 1284 resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail; 1285 break; 1286 } 1287 1288 // '${' 1289 if (currChar === CharacterCodes.$ && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.openBrace) { 1290 contents += text.substring(start, pos); 1291 pos += 2; 1292 resultingToken = startedWithBacktick ? SyntaxKind.TemplateHead : SyntaxKind.TemplateMiddle; 1293 break; 1294 } 1295 1296 // Escape character 1297 if (currChar === CharacterCodes.backslash) { 1298 contents += text.substring(start, pos); 1299 contents += scanEscapeSequence(isTaggedTemplate); 1300 start = pos; 1301 continue; 1302 } 1303 1304 // Speculated ECMAScript 6 Spec 11.8.6.1: 1305 // <CR><LF> and <CR> LineTerminatorSequences are normalized to <LF> for Template Values 1306 if (currChar === CharacterCodes.carriageReturn) { 1307 contents += text.substring(start, pos); 1308 pos++; 1309 1310 if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) { 1311 pos++; 1312 } 1313 1314 contents += "\n"; 1315 start = pos; 1316 continue; 1317 } 1318 1319 pos++; 1320 } 1321 1322 Debug.assert(resultingToken !== undefined); 1323 1324 tokenValue = contents; 1325 return resultingToken; 1326 } 1327 1328 function scanEscapeSequence(isTaggedTemplate?: boolean): string { 1329 const start = pos; 1330 pos++; 1331 if (pos >= end) { 1332 error(Diagnostics.Unexpected_end_of_text); 1333 return ""; 1334 } 1335 const ch = text.charCodeAt(pos); 1336 pos++; 1337 switch (ch) { 1338 case CharacterCodes._0: 1339 // '\01' 1340 if (isTaggedTemplate && pos < end && isDigit(text.charCodeAt(pos))) { 1341 pos++; 1342 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1343 return text.substring(start, pos); 1344 } 1345 return "\0"; 1346 case CharacterCodes.b: 1347 return "\b"; 1348 case CharacterCodes.t: 1349 return "\t"; 1350 case CharacterCodes.n: 1351 return "\n"; 1352 case CharacterCodes.v: 1353 return "\v"; 1354 case CharacterCodes.f: 1355 return "\f"; 1356 case CharacterCodes.r: 1357 return "\r"; 1358 case CharacterCodes.singleQuote: 1359 return "\'"; 1360 case CharacterCodes.doubleQuote: 1361 return "\""; 1362 case CharacterCodes.u: 1363 if (isTaggedTemplate) { 1364 // '\u' or '\u0' or '\u00' or '\u000' 1365 for (let escapePos = pos; escapePos < pos + 4; escapePos++) { 1366 if (escapePos < end && !isHexDigit(text.charCodeAt(escapePos)) && text.charCodeAt(escapePos) !== CharacterCodes.openBrace) { 1367 pos = escapePos; 1368 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1369 return text.substring(start, pos); 1370 } 1371 } 1372 } 1373 // '\u{DDDDDDDD}' 1374 if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) { 1375 pos++; 1376 1377 // '\u{' 1378 if (isTaggedTemplate && !isHexDigit(text.charCodeAt(pos))) { 1379 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1380 return text.substring(start, pos); 1381 } 1382 1383 if (isTaggedTemplate) { 1384 const savePos = pos; 1385 const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); 1386 const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1; 1387 1388 // '\u{Not Code Point' or '\u{CodePoint' 1389 if (!isCodePoint(escapedValue) || text.charCodeAt(pos) !== CharacterCodes.closeBrace) { 1390 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1391 return text.substring(start, pos); 1392 } 1393 else { 1394 pos = savePos; 1395 } 1396 } 1397 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 1398 return scanExtendedUnicodeEscape(); 1399 } 1400 1401 tokenFlags |= TokenFlags.UnicodeEscape; 1402 // '\uDDDD' 1403 return scanHexadecimalEscape(/*numDigits*/ 4); 1404 1405 case CharacterCodes.x: 1406 if (isTaggedTemplate) { 1407 if (!isHexDigit(text.charCodeAt(pos))) { 1408 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1409 return text.substring(start, pos); 1410 } 1411 else if (!isHexDigit(text.charCodeAt(pos + 1))) { 1412 pos++; 1413 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1414 return text.substring(start, pos); 1415 } 1416 } 1417 // '\xDD' 1418 return scanHexadecimalEscape(/*numDigits*/ 2); 1419 1420 // when encountering a LineContinuation (i.e. a backslash and a line terminator sequence), 1421 // the line terminator is interpreted to be "the empty code unit sequence". 1422 case CharacterCodes.carriageReturn: 1423 if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) { 1424 pos++; 1425 } 1426 // falls through 1427 case CharacterCodes.lineFeed: 1428 case CharacterCodes.lineSeparator: 1429 case CharacterCodes.paragraphSeparator: 1430 return ""; 1431 default: 1432 return String.fromCharCode(ch); 1433 } 1434 } 1435 1436 function scanHexadecimalEscape(numDigits: number): string { 1437 const escapedValue = scanExactNumberOfHexDigits(numDigits, /*canHaveSeparators*/ false); 1438 1439 if (escapedValue >= 0) { 1440 return String.fromCharCode(escapedValue); 1441 } 1442 else { 1443 error(Diagnostics.Hexadecimal_digit_expected); 1444 return ""; 1445 } 1446 } 1447 1448 function scanExtendedUnicodeEscape(): string { 1449 const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); 1450 const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1; 1451 let isInvalidExtendedEscape = false; 1452 1453 // Validate the value of the digit 1454 if (escapedValue < 0) { 1455 error(Diagnostics.Hexadecimal_digit_expected); 1456 isInvalidExtendedEscape = true; 1457 } 1458 else if (escapedValue > 0x10FFFF) { 1459 error(Diagnostics.An_extended_Unicode_escape_value_must_be_between_0x0_and_0x10FFFF_inclusive); 1460 isInvalidExtendedEscape = true; 1461 } 1462 1463 if (pos >= end) { 1464 error(Diagnostics.Unexpected_end_of_text); 1465 isInvalidExtendedEscape = true; 1466 } 1467 else if (text.charCodeAt(pos) === CharacterCodes.closeBrace) { 1468 // Only swallow the following character up if it's a '}'. 1469 pos++; 1470 } 1471 else { 1472 error(Diagnostics.Unterminated_Unicode_escape_sequence); 1473 isInvalidExtendedEscape = true; 1474 } 1475 1476 if (isInvalidExtendedEscape) { 1477 return ""; 1478 } 1479 1480 return utf16EncodeAsString(escapedValue); 1481 } 1482 1483 // Current character is known to be a backslash. Check for Unicode escape of the form '\uXXXX' 1484 // and return code point value if valid Unicode escape is found. Otherwise return -1. 1485 function peekUnicodeEscape(): number { 1486 if (pos + 5 < end && text.charCodeAt(pos + 1) === CharacterCodes.u) { 1487 const start = pos; 1488 pos += 2; 1489 const value = scanExactNumberOfHexDigits(4, /*canHaveSeparators*/ false); 1490 pos = start; 1491 return value; 1492 } 1493 return -1; 1494 } 1495 1496 1497 function peekExtendedUnicodeEscape(): number { 1498 if (codePointAt(text, pos + 1) === CharacterCodes.u && codePointAt(text, pos + 2) === CharacterCodes.openBrace) { 1499 const start = pos; 1500 pos += 3; 1501 const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); 1502 const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1; 1503 pos = start; 1504 return escapedValue; 1505 } 1506 return -1; 1507 } 1508 1509 function scanIdentifierParts(): string { 1510 let result = ""; 1511 let start = pos; 1512 while (pos < end) { 1513 let ch = codePointAt(text, pos); 1514 if (isIdentifierPart(ch, languageVersion)) { 1515 pos += charSize(ch); 1516 } 1517 else if (ch === CharacterCodes.backslash) { 1518 ch = peekExtendedUnicodeEscape(); 1519 if (ch >= 0 && isIdentifierPart(ch, languageVersion)) { 1520 pos += 3; 1521 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 1522 result += scanExtendedUnicodeEscape(); 1523 start = pos; 1524 continue; 1525 } 1526 ch = peekUnicodeEscape(); 1527 if (!(ch >= 0 && isIdentifierPart(ch, languageVersion))) { 1528 break; 1529 } 1530 tokenFlags |= TokenFlags.UnicodeEscape; 1531 result += text.substring(start, pos); 1532 result += utf16EncodeAsString(ch); 1533 // Valid Unicode escape is always six characters 1534 pos += 6; 1535 start = pos; 1536 } 1537 else { 1538 break; 1539 } 1540 } 1541 result += text.substring(start, pos); 1542 return result; 1543 } 1544 1545 function getIdentifierToken(): SyntaxKind.Identifier | KeywordSyntaxKind { 1546 // Reserved words are between 2 and 12 characters long and start with a lowercase letter 1547 const len = tokenValue.length; 1548 if (len >= 2 && len <= 12) { 1549 const ch = tokenValue.charCodeAt(0); 1550 if (ch >= CharacterCodes.a && ch <= CharacterCodes.z) { 1551 const keyword = textToKeyword.get(tokenValue); 1552 if (keyword !== undefined) { 1553 token = keyword; 1554 if (keyword === SyntaxKind.StructKeyword && !inEtsContext) { 1555 token = SyntaxKind.Identifier; 1556 } 1557 return token; 1558 } 1559 } 1560 } 1561 return token = SyntaxKind.Identifier; 1562 } 1563 1564 function scanBinaryOrOctalDigits(base: 2 | 8): string { 1565 let value = ""; 1566 // For counting number of digits; Valid binaryIntegerLiteral must have at least one binary digit following B or b. 1567 // Similarly valid octalIntegerLiteral must have at least one octal digit following o or O. 1568 let separatorAllowed = false; 1569 let isPreviousTokenSeparator = false; 1570 while (true) { 1571 const ch = text.charCodeAt(pos); 1572 // Numeric separators are allowed anywhere within a numeric literal, except not at the beginning, or following another separator 1573 if (ch === CharacterCodes._) { 1574 tokenFlags |= TokenFlags.ContainsSeparator; 1575 if (separatorAllowed) { 1576 separatorAllowed = false; 1577 isPreviousTokenSeparator = true; 1578 } 1579 else if (isPreviousTokenSeparator) { 1580 error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1); 1581 } 1582 else { 1583 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1); 1584 } 1585 pos++; 1586 continue; 1587 } 1588 separatorAllowed = true; 1589 if (!isDigit(ch) || ch - CharacterCodes._0 >= base) { 1590 break; 1591 } 1592 value += text[pos]; 1593 pos++; 1594 isPreviousTokenSeparator = false; 1595 } 1596 if (text.charCodeAt(pos - 1) === CharacterCodes._) { 1597 // Literal ends with underscore - not allowed 1598 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1); 1599 } 1600 return value; 1601 } 1602 1603 function checkBigIntSuffix(): SyntaxKind { 1604 if (text.charCodeAt(pos) === CharacterCodes.n) { 1605 tokenValue += "n"; 1606 // Use base 10 instead of base 2 or base 8 for shorter literals 1607 if (tokenFlags & TokenFlags.BinaryOrOctalSpecifier) { 1608 tokenValue = parsePseudoBigInt(tokenValue) + "n"; 1609 } 1610 pos++; 1611 return SyntaxKind.BigIntLiteral; 1612 } 1613 else { // not a bigint, so can convert to number in simplified form 1614 // Number() may not support 0b or 0o, so use parseInt() instead 1615 const numericValue = tokenFlags & TokenFlags.BinarySpecifier 1616 ? parseInt(tokenValue.slice(2), 2) // skip "0b" 1617 : tokenFlags & TokenFlags.OctalSpecifier 1618 ? parseInt(tokenValue.slice(2), 8) // skip "0o" 1619 : +tokenValue; 1620 tokenValue = "" + numericValue; 1621 return SyntaxKind.NumericLiteral; 1622 } 1623 } 1624 1625 function scan(): SyntaxKind { 1626 startPos = pos; 1627 tokenFlags = TokenFlags.None; 1628 let asteriskSeen = false; 1629 while (true) { 1630 tokenPos = pos; 1631 if (pos >= end) { 1632 return token = SyntaxKind.EndOfFileToken; 1633 } 1634 const ch = codePointAt(text, pos); 1635 1636 // Special handling for shebang 1637 if (ch === CharacterCodes.hash && pos === 0 && isShebangTrivia(text, pos)) { 1638 pos = scanShebangTrivia(text, pos); 1639 if (skipTrivia) { 1640 continue; 1641 } 1642 else { 1643 return token = SyntaxKind.ShebangTrivia; 1644 } 1645 } 1646 1647 switch (ch) { 1648 case CharacterCodes.lineFeed: 1649 case CharacterCodes.carriageReturn: 1650 tokenFlags |= TokenFlags.PrecedingLineBreak; 1651 if (skipTrivia) { 1652 pos++; 1653 continue; 1654 } 1655 else { 1656 if (ch === CharacterCodes.carriageReturn && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { 1657 // consume both CR and LF 1658 pos += 2; 1659 } 1660 else { 1661 pos++; 1662 } 1663 return token = SyntaxKind.NewLineTrivia; 1664 } 1665 case CharacterCodes.tab: 1666 case CharacterCodes.verticalTab: 1667 case CharacterCodes.formFeed: 1668 case CharacterCodes.space: 1669 case CharacterCodes.nonBreakingSpace: 1670 case CharacterCodes.ogham: 1671 case CharacterCodes.enQuad: 1672 case CharacterCodes.emQuad: 1673 case CharacterCodes.enSpace: 1674 case CharacterCodes.emSpace: 1675 case CharacterCodes.threePerEmSpace: 1676 case CharacterCodes.fourPerEmSpace: 1677 case CharacterCodes.sixPerEmSpace: 1678 case CharacterCodes.figureSpace: 1679 case CharacterCodes.punctuationSpace: 1680 case CharacterCodes.thinSpace: 1681 case CharacterCodes.hairSpace: 1682 case CharacterCodes.zeroWidthSpace: 1683 case CharacterCodes.narrowNoBreakSpace: 1684 case CharacterCodes.mathematicalSpace: 1685 case CharacterCodes.ideographicSpace: 1686 case CharacterCodes.byteOrderMark: 1687 if (skipTrivia) { 1688 pos++; 1689 continue; 1690 } 1691 else { 1692 while (pos < end && isWhiteSpaceSingleLine(text.charCodeAt(pos))) { 1693 pos++; 1694 } 1695 return token = SyntaxKind.WhitespaceTrivia; 1696 } 1697 case CharacterCodes.exclamation: 1698 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1699 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1700 return pos += 3, token = SyntaxKind.ExclamationEqualsEqualsToken; 1701 } 1702 return pos += 2, token = SyntaxKind.ExclamationEqualsToken; 1703 } 1704 pos++; 1705 return token = SyntaxKind.ExclamationToken; 1706 case CharacterCodes.doubleQuote: 1707 case CharacterCodes.singleQuote: 1708 tokenValue = scanString(); 1709 return token = SyntaxKind.StringLiteral; 1710 case CharacterCodes.backtick: 1711 return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ false); 1712 case CharacterCodes.percent: 1713 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1714 return pos += 2, token = SyntaxKind.PercentEqualsToken; 1715 } 1716 pos++; 1717 return token = SyntaxKind.PercentToken; 1718 case CharacterCodes.ampersand: 1719 if (text.charCodeAt(pos + 1) === CharacterCodes.ampersand) { 1720 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1721 return pos += 3, token = SyntaxKind.AmpersandAmpersandEqualsToken; 1722 } 1723 return pos += 2, token = SyntaxKind.AmpersandAmpersandToken; 1724 } 1725 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1726 return pos += 2, token = SyntaxKind.AmpersandEqualsToken; 1727 } 1728 pos++; 1729 return token = SyntaxKind.AmpersandToken; 1730 case CharacterCodes.openParen: 1731 pos++; 1732 return token = SyntaxKind.OpenParenToken; 1733 case CharacterCodes.closeParen: 1734 pos++; 1735 return token = SyntaxKind.CloseParenToken; 1736 case CharacterCodes.asterisk: 1737 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1738 return pos += 2, token = SyntaxKind.AsteriskEqualsToken; 1739 } 1740 if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { 1741 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1742 return pos += 3, token = SyntaxKind.AsteriskAsteriskEqualsToken; 1743 } 1744 return pos += 2, token = SyntaxKind.AsteriskAsteriskToken; 1745 } 1746 pos++; 1747 if (inJSDocType && !asteriskSeen && (tokenFlags & TokenFlags.PrecedingLineBreak)) { 1748 // decoration at the start of a JSDoc comment line 1749 asteriskSeen = true; 1750 continue; 1751 } 1752 return token = SyntaxKind.AsteriskToken; 1753 case CharacterCodes.plus: 1754 if (text.charCodeAt(pos + 1) === CharacterCodes.plus) { 1755 return pos += 2, token = SyntaxKind.PlusPlusToken; 1756 } 1757 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1758 return pos += 2, token = SyntaxKind.PlusEqualsToken; 1759 } 1760 pos++; 1761 return token = SyntaxKind.PlusToken; 1762 case CharacterCodes.comma: 1763 pos++; 1764 return token = SyntaxKind.CommaToken; 1765 case CharacterCodes.minus: 1766 if (text.charCodeAt(pos + 1) === CharacterCodes.minus) { 1767 return pos += 2, token = SyntaxKind.MinusMinusToken; 1768 } 1769 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1770 return pos += 2, token = SyntaxKind.MinusEqualsToken; 1771 } 1772 pos++; 1773 return token = SyntaxKind.MinusToken; 1774 case CharacterCodes.dot: 1775 if (isDigit(text.charCodeAt(pos + 1))) { 1776 tokenValue = scanNumber().value; 1777 return token = SyntaxKind.NumericLiteral; 1778 } 1779 if (text.charCodeAt(pos + 1) === CharacterCodes.dot && text.charCodeAt(pos + 2) === CharacterCodes.dot) { 1780 return pos += 3, token = SyntaxKind.DotDotDotToken; 1781 } 1782 pos++; 1783 return token = SyntaxKind.DotToken; 1784 case CharacterCodes.slash: 1785 // Single-line comment 1786 if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { 1787 pos += 2; 1788 1789 while (pos < end) { 1790 if (isLineBreak(text.charCodeAt(pos))) { 1791 break; 1792 } 1793 pos++; 1794 } 1795 1796 commentDirectives = appendIfCommentDirective( 1797 commentDirectives, 1798 text.slice(tokenPos, pos), 1799 commentDirectiveRegExSingleLine, 1800 tokenPos, 1801 ); 1802 1803 if (skipTrivia) { 1804 continue; 1805 } 1806 else { 1807 return token = SyntaxKind.SingleLineCommentTrivia; 1808 } 1809 } 1810 // Multi-line comment 1811 if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { 1812 pos += 2; 1813 if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) !== CharacterCodes.slash) { 1814 tokenFlags |= TokenFlags.PrecedingJSDocComment; 1815 } 1816 1817 let commentClosed = false; 1818 let lastLineStart = tokenPos; 1819 while (pos < end) { 1820 const ch = text.charCodeAt(pos); 1821 1822 if (ch === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { 1823 pos += 2; 1824 commentClosed = true; 1825 break; 1826 } 1827 1828 pos++; 1829 1830 if (isLineBreak(ch)) { 1831 lastLineStart = pos; 1832 tokenFlags |= TokenFlags.PrecedingLineBreak; 1833 } 1834 } 1835 1836 commentDirectives = appendIfCommentDirective(commentDirectives, text.slice(lastLineStart, pos), commentDirectiveRegExMultiLine, lastLineStart); 1837 1838 if (!commentClosed) { 1839 error(Diagnostics.Asterisk_Slash_expected); 1840 } 1841 1842 if (skipTrivia) { 1843 continue; 1844 } 1845 else { 1846 if (!commentClosed) { 1847 tokenFlags |= TokenFlags.Unterminated; 1848 } 1849 return token = SyntaxKind.MultiLineCommentTrivia; 1850 } 1851 } 1852 1853 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1854 return pos += 2, token = SyntaxKind.SlashEqualsToken; 1855 } 1856 1857 pos++; 1858 return token = SyntaxKind.SlashToken; 1859 1860 case CharacterCodes._0: 1861 if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) { 1862 pos += 2; 1863 tokenValue = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ true); 1864 if (!tokenValue) { 1865 error(Diagnostics.Hexadecimal_digit_expected); 1866 tokenValue = "0"; 1867 } 1868 tokenValue = "0x" + tokenValue; 1869 tokenFlags |= TokenFlags.HexSpecifier; 1870 return token = checkBigIntSuffix(); 1871 } 1872 else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) { 1873 pos += 2; 1874 tokenValue = scanBinaryOrOctalDigits(/* base */ 2); 1875 if (!tokenValue) { 1876 error(Diagnostics.Binary_digit_expected); 1877 tokenValue = "0"; 1878 } 1879 tokenValue = "0b" + tokenValue; 1880 tokenFlags |= TokenFlags.BinarySpecifier; 1881 return token = checkBigIntSuffix(); 1882 } 1883 else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) { 1884 pos += 2; 1885 tokenValue = scanBinaryOrOctalDigits(/* base */ 8); 1886 if (!tokenValue) { 1887 error(Diagnostics.Octal_digit_expected); 1888 tokenValue = "0"; 1889 } 1890 tokenValue = "0o" + tokenValue; 1891 tokenFlags |= TokenFlags.OctalSpecifier; 1892 return token = checkBigIntSuffix(); 1893 } 1894 // Try to parse as an octal 1895 if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) { 1896 tokenValue = "" + scanOctalDigits(); 1897 tokenFlags |= TokenFlags.Octal; 1898 return token = SyntaxKind.NumericLiteral; 1899 } 1900 // This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero 1901 // can only be followed by an octal digit, a dot, or the end of the number literal. However, we are being 1902 // permissive and allowing decimal digits of the form 08* and 09* (which many browsers also do). 1903 // falls through 1904 case CharacterCodes._1: 1905 case CharacterCodes._2: 1906 case CharacterCodes._3: 1907 case CharacterCodes._4: 1908 case CharacterCodes._5: 1909 case CharacterCodes._6: 1910 case CharacterCodes._7: 1911 case CharacterCodes._8: 1912 case CharacterCodes._9: 1913 ({ type: token, value: tokenValue } = scanNumber()); 1914 return token; 1915 case CharacterCodes.colon: 1916 pos++; 1917 return token = SyntaxKind.ColonToken; 1918 case CharacterCodes.semicolon: 1919 pos++; 1920 return token = SyntaxKind.SemicolonToken; 1921 case CharacterCodes.lessThan: 1922 if (isConflictMarkerTrivia(text, pos)) { 1923 pos = scanConflictMarkerTrivia(text, pos, error); 1924 if (skipTrivia) { 1925 continue; 1926 } 1927 else { 1928 return token = SyntaxKind.ConflictMarkerTrivia; 1929 } 1930 } 1931 1932 if (text.charCodeAt(pos + 1) === CharacterCodes.lessThan) { 1933 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1934 return pos += 3, token = SyntaxKind.LessThanLessThanEqualsToken; 1935 } 1936 return pos += 2, token = SyntaxKind.LessThanLessThanToken; 1937 } 1938 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1939 return pos += 2, token = SyntaxKind.LessThanEqualsToken; 1940 } 1941 if (languageVariant === LanguageVariant.JSX && 1942 text.charCodeAt(pos + 1) === CharacterCodes.slash && 1943 text.charCodeAt(pos + 2) !== CharacterCodes.asterisk) { 1944 return pos += 2, token = SyntaxKind.LessThanSlashToken; 1945 } 1946 pos++; 1947 return token = SyntaxKind.LessThanToken; 1948 case CharacterCodes.equals: 1949 if (isConflictMarkerTrivia(text, pos)) { 1950 pos = scanConflictMarkerTrivia(text, pos, error); 1951 if (skipTrivia) { 1952 continue; 1953 } 1954 else { 1955 return token = SyntaxKind.ConflictMarkerTrivia; 1956 } 1957 } 1958 1959 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1960 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1961 return pos += 3, token = SyntaxKind.EqualsEqualsEqualsToken; 1962 } 1963 return pos += 2, token = SyntaxKind.EqualsEqualsToken; 1964 } 1965 if (text.charCodeAt(pos + 1) === CharacterCodes.greaterThan) { 1966 return pos += 2, token = SyntaxKind.EqualsGreaterThanToken; 1967 } 1968 pos++; 1969 return token = SyntaxKind.EqualsToken; 1970 case CharacterCodes.greaterThan: 1971 if (isConflictMarkerTrivia(text, pos)) { 1972 pos = scanConflictMarkerTrivia(text, pos, error); 1973 if (skipTrivia) { 1974 continue; 1975 } 1976 else { 1977 return token = SyntaxKind.ConflictMarkerTrivia; 1978 } 1979 } 1980 1981 pos++; 1982 return token = SyntaxKind.GreaterThanToken; 1983 case CharacterCodes.question: 1984 if (text.charCodeAt(pos + 1) === CharacterCodes.dot && !isDigit(text.charCodeAt(pos + 2))) { 1985 return pos += 2, token = SyntaxKind.QuestionDotToken; 1986 } 1987 if (text.charCodeAt(pos + 1) === CharacterCodes.question) { 1988 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1989 return pos += 3, token = SyntaxKind.QuestionQuestionEqualsToken; 1990 } 1991 return pos += 2, token = SyntaxKind.QuestionQuestionToken; 1992 } 1993 pos++; 1994 return token = SyntaxKind.QuestionToken; 1995 case CharacterCodes.openBracket: 1996 pos++; 1997 return token = SyntaxKind.OpenBracketToken; 1998 case CharacterCodes.closeBracket: 1999 pos++; 2000 return token = SyntaxKind.CloseBracketToken; 2001 case CharacterCodes.caret: 2002 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 2003 return pos += 2, token = SyntaxKind.CaretEqualsToken; 2004 } 2005 pos++; 2006 return token = SyntaxKind.CaretToken; 2007 case CharacterCodes.openBrace: 2008 pos++; 2009 return token = SyntaxKind.OpenBraceToken; 2010 case CharacterCodes.bar: 2011 if (isConflictMarkerTrivia(text, pos)) { 2012 pos = scanConflictMarkerTrivia(text, pos, error); 2013 if (skipTrivia) { 2014 continue; 2015 } 2016 else { 2017 return token = SyntaxKind.ConflictMarkerTrivia; 2018 } 2019 } 2020 2021 if (text.charCodeAt(pos + 1) === CharacterCodes.bar) { 2022 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 2023 return pos += 3, token = SyntaxKind.BarBarEqualsToken; 2024 } 2025 return pos += 2, token = SyntaxKind.BarBarToken; 2026 } 2027 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 2028 return pos += 2, token = SyntaxKind.BarEqualsToken; 2029 } 2030 pos++; 2031 return token = SyntaxKind.BarToken; 2032 case CharacterCodes.closeBrace: 2033 pos++; 2034 return token = SyntaxKind.CloseBraceToken; 2035 case CharacterCodes.tilde: 2036 pos++; 2037 return token = SyntaxKind.TildeToken; 2038 case CharacterCodes.at: 2039 pos++; 2040 return token = SyntaxKind.AtToken; 2041 case CharacterCodes.backslash: 2042 const extendedCookedChar = peekExtendedUnicodeEscape(); 2043 if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) { 2044 pos += 3; 2045 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 2046 tokenValue = scanExtendedUnicodeEscape() + scanIdentifierParts(); 2047 return token = getIdentifierToken(); 2048 } 2049 2050 const cookedChar = peekUnicodeEscape(); 2051 if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) { 2052 pos += 6; 2053 tokenFlags |= TokenFlags.UnicodeEscape; 2054 tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts(); 2055 return token = getIdentifierToken(); 2056 } 2057 2058 error(Diagnostics.Invalid_character); 2059 pos++; 2060 return token = SyntaxKind.Unknown; 2061 case CharacterCodes.hash: 2062 if (pos !== 0 && text[pos + 1] === "!") { 2063 error(Diagnostics.can_only_be_used_at_the_start_of_a_file); 2064 pos++; 2065 return token = SyntaxKind.Unknown; 2066 } 2067 2068 const charAfterHash = codePointAt(text, pos + 1); 2069 if (charAfterHash === CharacterCodes.backslash) { 2070 pos++; 2071 const extendedCookedChar = peekExtendedUnicodeEscape(); 2072 if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) { 2073 pos += 3; 2074 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 2075 tokenValue = "#" + scanExtendedUnicodeEscape() + scanIdentifierParts(); 2076 return token = SyntaxKind.PrivateIdentifier; 2077 } 2078 2079 const cookedChar = peekUnicodeEscape(); 2080 if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) { 2081 pos += 6; 2082 tokenFlags |= TokenFlags.UnicodeEscape; 2083 tokenValue = "#" + String.fromCharCode(cookedChar) + scanIdentifierParts(); 2084 return token = SyntaxKind.PrivateIdentifier; 2085 } 2086 pos--; 2087 } 2088 2089 if (isIdentifierStart(charAfterHash, languageVersion)) { 2090 pos++; 2091 // We're relying on scanIdentifier's behavior and adjusting the token kind after the fact. 2092 // Notably absent from this block is the fact that calling a function named "scanIdentifier", 2093 // but identifiers don't include '#', and that function doesn't deal with it at all. 2094 // This works because 'scanIdentifier' tries to reuse source characters and builds up substrings; 2095 // however, it starts at the 'tokenPos' which includes the '#', and will "accidentally" prepend the '#' for us. 2096 scanIdentifier(charAfterHash, languageVersion); 2097 } 2098 else { 2099 tokenValue = "#"; 2100 error(Diagnostics.Invalid_character, pos++, charSize(ch)); 2101 } 2102 return token = SyntaxKind.PrivateIdentifier; 2103 default: 2104 const identifierKind = scanIdentifier(ch, languageVersion); 2105 if (identifierKind) { 2106 return token = identifierKind; 2107 } 2108 else if (isWhiteSpaceSingleLine(ch)) { 2109 pos += charSize(ch); 2110 continue; 2111 } 2112 else if (isLineBreak(ch)) { 2113 tokenFlags |= TokenFlags.PrecedingLineBreak; 2114 pos += charSize(ch); 2115 continue; 2116 } 2117 const size = charSize(ch); 2118 error(Diagnostics.Invalid_character, pos, size); 2119 pos += size; 2120 return token = SyntaxKind.Unknown; 2121 } 2122 } 2123 } 2124 2125 function reScanInvalidIdentifier(): SyntaxKind { 2126 Debug.assert(token === SyntaxKind.Unknown, "'reScanInvalidIdentifier' should only be called when the current token is 'SyntaxKind.Unknown'."); 2127 pos = tokenPos = startPos; 2128 tokenFlags = 0; 2129 const ch = codePointAt(text, pos); 2130 const identifierKind = scanIdentifier(ch, ScriptTarget.ESNext); 2131 if (identifierKind) { 2132 return token = identifierKind; 2133 } 2134 pos += charSize(ch); 2135 return token; // Still `SyntaKind.Unknown` 2136 } 2137 2138 function scanIdentifier(startCharacter: number, languageVersion: ScriptTarget) { 2139 let ch = startCharacter; 2140 if (isIdentifierStart(ch, languageVersion)) { 2141 pos += charSize(ch); 2142 while (pos < end && isIdentifierPart(ch = codePointAt(text, pos), languageVersion)) pos += charSize(ch); 2143 tokenValue = text.substring(tokenPos, pos); 2144 if (ch === CharacterCodes.backslash) { 2145 tokenValue += scanIdentifierParts(); 2146 } 2147 return getIdentifierToken(); 2148 } 2149 } 2150 2151 function reScanGreaterToken(): SyntaxKind { 2152 if (token === SyntaxKind.GreaterThanToken) { 2153 if (text.charCodeAt(pos) === CharacterCodes.greaterThan) { 2154 if (text.charCodeAt(pos + 1) === CharacterCodes.greaterThan) { 2155 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 2156 return pos += 3, token = SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken; 2157 } 2158 return pos += 2, token = SyntaxKind.GreaterThanGreaterThanGreaterThanToken; 2159 } 2160 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 2161 return pos += 2, token = SyntaxKind.GreaterThanGreaterThanEqualsToken; 2162 } 2163 pos++; 2164 return token = SyntaxKind.GreaterThanGreaterThanToken; 2165 } 2166 if (text.charCodeAt(pos) === CharacterCodes.equals) { 2167 pos++; 2168 return token = SyntaxKind.GreaterThanEqualsToken; 2169 } 2170 } 2171 return token; 2172 } 2173 2174 function reScanAsteriskEqualsToken(): SyntaxKind { 2175 Debug.assert(token === SyntaxKind.AsteriskEqualsToken, "'reScanAsteriskEqualsToken' should only be called on a '*='"); 2176 pos = tokenPos + 1; 2177 return token = SyntaxKind.EqualsToken; 2178 } 2179 2180 function reScanSlashToken(): SyntaxKind { 2181 if (token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) { 2182 let p = tokenPos + 1; 2183 let inEscape = false; 2184 let inCharacterClass = false; 2185 while (true) { 2186 // If we reach the end of a file, or hit a newline, then this is an unterminated 2187 // regex. Report error and return what we have so far. 2188 if (p >= end) { 2189 tokenFlags |= TokenFlags.Unterminated; 2190 error(Diagnostics.Unterminated_regular_expression_literal); 2191 break; 2192 } 2193 2194 const ch = text.charCodeAt(p); 2195 if (isLineBreak(ch)) { 2196 tokenFlags |= TokenFlags.Unterminated; 2197 error(Diagnostics.Unterminated_regular_expression_literal); 2198 break; 2199 } 2200 2201 if (inEscape) { 2202 // Parsing an escape character; 2203 // reset the flag and just advance to the next char. 2204 inEscape = false; 2205 } 2206 else if (ch === CharacterCodes.slash && !inCharacterClass) { 2207 // A slash within a character class is permissible, 2208 // but in general it signals the end of the regexp literal. 2209 p++; 2210 break; 2211 } 2212 else if (ch === CharacterCodes.openBracket) { 2213 inCharacterClass = true; 2214 } 2215 else if (ch === CharacterCodes.backslash) { 2216 inEscape = true; 2217 } 2218 else if (ch === CharacterCodes.closeBracket) { 2219 inCharacterClass = false; 2220 } 2221 p++; 2222 } 2223 2224 while (p < end && isIdentifierPart(text.charCodeAt(p), languageVersion)) { 2225 p++; 2226 } 2227 pos = p; 2228 tokenValue = text.substring(tokenPos, pos); 2229 token = SyntaxKind.RegularExpressionLiteral; 2230 } 2231 return token; 2232 } 2233 2234 function appendIfCommentDirective( 2235 commentDirectives: CommentDirective[] | undefined, 2236 text: string, 2237 commentDirectiveRegEx: RegExp, 2238 lineStart: number, 2239 ) { 2240 const type = getDirectiveFromComment(trimStringStart(text), commentDirectiveRegEx); 2241 if (type === undefined) { 2242 return commentDirectives; 2243 } 2244 2245 return append( 2246 commentDirectives, 2247 { 2248 range: { pos: lineStart, end: pos }, 2249 type, 2250 }, 2251 ); 2252 } 2253 2254 function getDirectiveFromComment(text: string, commentDirectiveRegEx: RegExp) { 2255 const match = commentDirectiveRegEx.exec(text); 2256 if (!match) { 2257 return undefined; 2258 } 2259 2260 switch (match[1]) { 2261 case "ts-expect-error": 2262 return CommentDirectiveType.ExpectError; 2263 2264 case "ts-ignore": 2265 return CommentDirectiveType.Ignore; 2266 } 2267 2268 return undefined; 2269 } 2270 2271 /** 2272 * Unconditionally back up and scan a template expression portion. 2273 */ 2274 function reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind { 2275 Debug.assert(token === SyntaxKind.CloseBraceToken, "'reScanTemplateToken' should only be called on a '}'"); 2276 pos = tokenPos; 2277 return token = scanTemplateAndSetTokenValue(isTaggedTemplate); 2278 } 2279 2280 function reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind { 2281 pos = tokenPos; 2282 return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ true); 2283 } 2284 2285 function reScanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind { 2286 pos = tokenPos = startPos; 2287 return token = scanJsxToken(allowMultilineJsxText); 2288 } 2289 2290 function reScanLessThanToken(): SyntaxKind { 2291 if (token === SyntaxKind.LessThanLessThanToken) { 2292 pos = tokenPos + 1; 2293 return token = SyntaxKind.LessThanToken; 2294 } 2295 return token; 2296 } 2297 2298 function reScanHashToken(): SyntaxKind { 2299 if (token === SyntaxKind.PrivateIdentifier) { 2300 pos = tokenPos + 1; 2301 return token = SyntaxKind.HashToken; 2302 } 2303 return token; 2304 } 2305 2306 function reScanQuestionToken(): SyntaxKind { 2307 Debug.assert(token === SyntaxKind.QuestionQuestionToken, "'reScanQuestionToken' should only be called on a '??'"); 2308 pos = tokenPos + 1; 2309 return token = SyntaxKind.QuestionToken; 2310 } 2311 2312 function scanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind { 2313 startPos = tokenPos = pos; 2314 2315 if (pos >= end) { 2316 return token = SyntaxKind.EndOfFileToken; 2317 } 2318 2319 let char = text.charCodeAt(pos); 2320 if (char === CharacterCodes.lessThan) { 2321 if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { 2322 pos += 2; 2323 return token = SyntaxKind.LessThanSlashToken; 2324 } 2325 pos++; 2326 return token = SyntaxKind.LessThanToken; 2327 } 2328 2329 if (char === CharacterCodes.openBrace) { 2330 pos++; 2331 return token = SyntaxKind.OpenBraceToken; 2332 } 2333 2334 // First non-whitespace character on this line. 2335 let firstNonWhitespace = 0; 2336 2337 // These initial values are special because the first line is: 2338 // firstNonWhitespace = 0 to indicate that we want leading whitespace, 2339 2340 while (pos < end) { 2341 char = text.charCodeAt(pos); 2342 if (char === CharacterCodes.openBrace) { 2343 break; 2344 } 2345 if (char === CharacterCodes.lessThan) { 2346 if (isConflictMarkerTrivia(text, pos)) { 2347 pos = scanConflictMarkerTrivia(text, pos, error); 2348 return token = SyntaxKind.ConflictMarkerTrivia; 2349 } 2350 break; 2351 } 2352 if (char === CharacterCodes.greaterThan) { 2353 error(Diagnostics.Unexpected_token_Did_you_mean_or_gt, pos, 1); 2354 } 2355 if (char === CharacterCodes.closeBrace) { 2356 error(Diagnostics.Unexpected_token_Did_you_mean_or_rbrace, pos, 1); 2357 } 2358 2359 // FirstNonWhitespace is 0, then we only see whitespaces so far. If we see a linebreak, we want to ignore that whitespaces. 2360 // i.e (- : whitespace) 2361 // <div>---- 2362 // </div> becomes <div></div> 2363 // 2364 // <div>----</div> becomes <div>----</div> 2365 if (isLineBreak(char) && firstNonWhitespace === 0) { 2366 firstNonWhitespace = -1; 2367 } 2368 else if (!allowMultilineJsxText && isLineBreak(char) && firstNonWhitespace > 0) { 2369 // Stop JsxText on each line during formatting. This allows the formatter to 2370 // indent each line correctly. 2371 break; 2372 } 2373 else if (!isWhiteSpaceLike(char)) { 2374 firstNonWhitespace = pos; 2375 } 2376 2377 pos++; 2378 } 2379 2380 tokenValue = text.substring(startPos, pos); 2381 2382 return firstNonWhitespace === -1 ? SyntaxKind.JsxTextAllWhiteSpaces : SyntaxKind.JsxText; 2383 } 2384 2385 // Scans a JSX identifier; these differ from normal identifiers in that 2386 // they allow dashes 2387 function scanJsxIdentifier(): SyntaxKind { 2388 if (tokenIsIdentifierOrKeyword(token)) { 2389 // An identifier or keyword has already been parsed - check for a `-` or a single instance of `:` and then append it and 2390 // everything after it to the token 2391 // Do note that this means that `scanJsxIdentifier` effectively _mutates_ the visible token without advancing to a new token 2392 // Any caller should be expecting this behavior and should only read the pos or token value after calling it. 2393 let namespaceSeparator = false; 2394 while (pos < end) { 2395 const ch = text.charCodeAt(pos); 2396 if (ch === CharacterCodes.minus) { 2397 tokenValue += "-"; 2398 pos++; 2399 continue; 2400 } 2401 else if (ch === CharacterCodes.colon && !namespaceSeparator) { 2402 tokenValue += ":"; 2403 pos++; 2404 namespaceSeparator = true; 2405 token = SyntaxKind.Identifier; // swap from keyword kind to identifier kind 2406 continue; 2407 } 2408 const oldPos = pos; 2409 tokenValue += scanIdentifierParts(); // reuse `scanIdentifierParts` so unicode escapes are handled 2410 if (pos === oldPos) { 2411 break; 2412 } 2413 } 2414 // Do not include a trailing namespace separator in the token, since this is against the spec. 2415 if (tokenValue.slice(-1) === ":") { 2416 tokenValue = tokenValue.slice(0, -1); 2417 pos--; 2418 } 2419 return getIdentifierToken(); 2420 } 2421 return token; 2422 } 2423 2424 function scanJsxAttributeValue(): SyntaxKind { 2425 startPos = pos; 2426 2427 switch (text.charCodeAt(pos)) { 2428 case CharacterCodes.doubleQuote: 2429 case CharacterCodes.singleQuote: 2430 tokenValue = scanString(/*jsxAttributeString*/ true); 2431 return token = SyntaxKind.StringLiteral; 2432 default: 2433 // If this scans anything other than `{`, it's a parse error. 2434 return scan(); 2435 } 2436 } 2437 2438 function reScanJsxAttributeValue(): SyntaxKind { 2439 pos = tokenPos = startPos; 2440 return scanJsxAttributeValue(); 2441 } 2442 2443 function scanJsDocToken(): JSDocSyntaxKind { 2444 startPos = tokenPos = pos; 2445 tokenFlags = TokenFlags.None; 2446 if (pos >= end) { 2447 return token = SyntaxKind.EndOfFileToken; 2448 } 2449 2450 const ch = codePointAt(text, pos); 2451 pos += charSize(ch); 2452 switch (ch) { 2453 case CharacterCodes.tab: 2454 case CharacterCodes.verticalTab: 2455 case CharacterCodes.formFeed: 2456 case CharacterCodes.space: 2457 while (pos < end && isWhiteSpaceSingleLine(text.charCodeAt(pos))) { 2458 pos++; 2459 } 2460 return token = SyntaxKind.WhitespaceTrivia; 2461 case CharacterCodes.at: 2462 return token = SyntaxKind.AtToken; 2463 case CharacterCodes.carriageReturn: 2464 if (text.charCodeAt(pos) === CharacterCodes.lineFeed) { 2465 pos++; 2466 } 2467 // falls through 2468 case CharacterCodes.lineFeed: 2469 tokenFlags |= TokenFlags.PrecedingLineBreak; 2470 return token = SyntaxKind.NewLineTrivia; 2471 case CharacterCodes.asterisk: 2472 return token = SyntaxKind.AsteriskToken; 2473 case CharacterCodes.openBrace: 2474 return token = SyntaxKind.OpenBraceToken; 2475 case CharacterCodes.closeBrace: 2476 return token = SyntaxKind.CloseBraceToken; 2477 case CharacterCodes.openBracket: 2478 return token = SyntaxKind.OpenBracketToken; 2479 case CharacterCodes.closeBracket: 2480 return token = SyntaxKind.CloseBracketToken; 2481 case CharacterCodes.lessThan: 2482 return token = SyntaxKind.LessThanToken; 2483 case CharacterCodes.greaterThan: 2484 return token = SyntaxKind.GreaterThanToken; 2485 case CharacterCodes.equals: 2486 return token = SyntaxKind.EqualsToken; 2487 case CharacterCodes.comma: 2488 return token = SyntaxKind.CommaToken; 2489 case CharacterCodes.dot: 2490 return token = SyntaxKind.DotToken; 2491 case CharacterCodes.backtick: 2492 return token = SyntaxKind.BacktickToken; 2493 case CharacterCodes.hash: 2494 return token = SyntaxKind.HashToken; 2495 case CharacterCodes.backslash: 2496 pos--; 2497 const extendedCookedChar = peekExtendedUnicodeEscape(); 2498 if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) { 2499 pos += 3; 2500 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 2501 tokenValue = scanExtendedUnicodeEscape() + scanIdentifierParts(); 2502 return token = getIdentifierToken(); 2503 } 2504 2505 const cookedChar = peekUnicodeEscape(); 2506 if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) { 2507 pos += 6; 2508 tokenFlags |= TokenFlags.UnicodeEscape; 2509 tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts(); 2510 return token = getIdentifierToken(); 2511 } 2512 pos++; 2513 return token = SyntaxKind.Unknown; 2514 } 2515 2516 if (isIdentifierStart(ch, languageVersion)) { 2517 let char = ch; 2518 while (pos < end && isIdentifierPart(char = codePointAt(text, pos), languageVersion) || text.charCodeAt(pos) === CharacterCodes.minus) pos += charSize(char); 2519 tokenValue = text.substring(tokenPos, pos); 2520 if (char === CharacterCodes.backslash) { 2521 tokenValue += scanIdentifierParts(); 2522 } 2523 return token = getIdentifierToken(); 2524 } 2525 else { 2526 return token = SyntaxKind.Unknown; 2527 } 2528 } 2529 2530 function speculationHelper<T>(callback: () => T, isLookahead: boolean): T { 2531 const savePos = pos; 2532 const saveStartPos = startPos; 2533 const saveTokenPos = tokenPos; 2534 const saveToken = token; 2535 const saveTokenValue = tokenValue; 2536 const saveTokenFlags = tokenFlags; 2537 const result = callback(); 2538 2539 // If our callback returned something 'falsy' or we're just looking ahead, 2540 // then unconditionally restore us to where we were. 2541 if (!result || isLookahead) { 2542 pos = savePos; 2543 startPos = saveStartPos; 2544 tokenPos = saveTokenPos; 2545 token = saveToken; 2546 tokenValue = saveTokenValue; 2547 tokenFlags = saveTokenFlags; 2548 } 2549 return result; 2550 } 2551 2552 function scanRange<T>(start: number, length: number, callback: () => T): T { 2553 const saveEnd = end; 2554 const savePos = pos; 2555 const saveStartPos = startPos; 2556 const saveTokenPos = tokenPos; 2557 const saveToken = token; 2558 const saveTokenValue = tokenValue; 2559 const saveTokenFlags = tokenFlags; 2560 const saveErrorExpectations = commentDirectives; 2561 2562 setText(text, start, length); 2563 const result = callback(); 2564 2565 end = saveEnd; 2566 pos = savePos; 2567 startPos = saveStartPos; 2568 tokenPos = saveTokenPos; 2569 token = saveToken; 2570 tokenValue = saveTokenValue; 2571 tokenFlags = saveTokenFlags; 2572 commentDirectives = saveErrorExpectations; 2573 2574 return result; 2575 } 2576 2577 function lookAhead<T>(callback: () => T): T { 2578 return speculationHelper(callback, /*isLookahead*/ true); 2579 } 2580 2581 function tryScan<T>(callback: () => T): T { 2582 return speculationHelper(callback, /*isLookahead*/ false); 2583 } 2584 2585 function getText(): string { 2586 return text; 2587 } 2588 2589 function clearCommentDirectives() { 2590 commentDirectives = undefined; 2591 } 2592 2593 function setText(newText: string | undefined, start: number | undefined, length: number | undefined) { 2594 text = newText || ""; 2595 end = length === undefined ? text.length : start! + length; 2596 setTextPos(start || 0); 2597 } 2598 2599 function setOnError(errorCallback: ErrorCallback | undefined) { 2600 onError = errorCallback; 2601 } 2602 2603 function setScriptTarget(scriptTarget: ScriptTarget) { 2604 languageVersion = scriptTarget; 2605 } 2606 2607 function setLanguageVariant(variant: LanguageVariant) { 2608 languageVariant = variant; 2609 } 2610 2611 function setTextPos(textPos: number) { 2612 Debug.assert(textPos >= 0); 2613 pos = textPos; 2614 startPos = textPos; 2615 tokenPos = textPos; 2616 token = SyntaxKind.Unknown; 2617 tokenValue = undefined!; 2618 tokenFlags = TokenFlags.None; 2619 } 2620 2621 function setInJSDocType(inType: boolean) { 2622 inJSDocType += inType ? 1 : -1; 2623 } 2624 } 2625 2626 /* @internal */ 2627 const codePointAt: (s: string, i: number) => number = (String.prototype as any).codePointAt ? (s, i) => (s as any).codePointAt(i) : function codePointAt(str, i): number { 2628 // from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/codePointAt 2629 const size = str.length; 2630 // Account for out-of-bounds indices: 2631 if (i < 0 || i >= size) { 2632 return undefined!; // String.codePointAt returns `undefined` for OOB indexes 2633 } 2634 // Get the first code unit 2635 const first = str.charCodeAt(i); 2636 // check if it’s the start of a surrogate pair 2637 if (first >= 0xD800 && first <= 0xDBFF && size > i + 1) { // high surrogate and there is a next code unit 2638 const second = str.charCodeAt(i + 1); 2639 if (second >= 0xDC00 && second <= 0xDFFF) { // low surrogate 2640 // https://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae 2641 return (first - 0xD800) * 0x400 + second - 0xDC00 + 0x10000; 2642 } 2643 } 2644 return first; 2645 }; 2646 2647 /* @internal */ 2648 function charSize(ch: number) { 2649 if (ch >= 0x10000) { 2650 return 2; 2651 } 2652 return 1; 2653 } 2654 2655 // Derived from the 10.1.1 UTF16Encoding of the ES6 Spec. 2656 function utf16EncodeAsStringFallback(codePoint: number) { 2657 Debug.assert(0x0 <= codePoint && codePoint <= 0x10FFFF); 2658 2659 if (codePoint <= 65535) { 2660 return String.fromCharCode(codePoint); 2661 } 2662 2663 const codeUnit1 = Math.floor((codePoint - 65536) / 1024) + 0xD800; 2664 const codeUnit2 = ((codePoint - 65536) % 1024) + 0xDC00; 2665 2666 return String.fromCharCode(codeUnit1, codeUnit2); 2667 } 2668 2669 const utf16EncodeAsStringWorker: (codePoint: number) => string = (String as any).fromCodePoint ? codePoint => (String as any).fromCodePoint(codePoint) : utf16EncodeAsStringFallback; 2670 2671 /* @internal */ 2672 export function utf16EncodeAsString(codePoint: number) { 2673 return utf16EncodeAsStringWorker(codePoint); 2674 } 2675} 2676