1import { 2 append, 3 arraysEqual, 4 binarySearch, 5 CharacterCodes, 6 CommentDirective, 7 CommentDirectiveType, 8 CommentKind, 9 CommentRange, 10 compareValues, 11 Debug, 12 DiagnosticMessage, 13 Diagnostics, 14 ESMap, 15 getEntries, 16 identity, 17 JSDocSyntaxKind, 18 JsxTokenSyntaxKind, 19 KeywordSyntaxKind, 20 LanguageVariant, 21 LineAndCharacter, 22 Map, 23 MapLike, 24 parsePseudoBigInt, 25 positionIsSynthesized, 26 ScriptTarget, 27 SourceFileLike, 28 SyntaxKind, 29 TokenFlags, 30 trimStringStart 31} from "./_namespaces/ts"; 32 33export type ErrorCallback = (message: DiagnosticMessage, length: number) => void; 34 35/** @internal */ 36export function tokenIsIdentifierOrKeyword(token: SyntaxKind): boolean { 37 return token >= SyntaxKind.Identifier; 38} 39 40/** @internal */ 41export function tokenIsIdentifierOrKeywordOrGreaterThan(token: SyntaxKind): boolean { 42 return token === SyntaxKind.GreaterThanToken || tokenIsIdentifierOrKeyword(token); 43} 44 45export interface Scanner { 46 getStartPos(): number; 47 getToken(): SyntaxKind; 48 getTextPos(): number; 49 getTokenPos(): number; 50 getTokenText(): string; 51 getTokenValue(): string; 52 hasUnicodeEscape(): boolean; 53 hasExtendedUnicodeEscape(): boolean; 54 hasPrecedingLineBreak(): boolean; 55 /** @internal */ 56 hasPrecedingJSDocComment(): boolean; 57 isIdentifier(): boolean; 58 isReservedWord(): boolean; 59 isUnterminated(): boolean; 60 /** @internal */ 61 getNumericLiteralFlags(): TokenFlags; 62 /** @internal */ 63 getCommentDirectives(): CommentDirective[] | undefined; 64 /** @internal */ 65 getTokenFlags(): TokenFlags; 66 reScanGreaterToken(): SyntaxKind; 67 reScanSlashToken(): SyntaxKind; 68 reScanAsteriskEqualsToken(): SyntaxKind; 69 reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind; 70 reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind; 71 scanJsxIdentifier(): SyntaxKind; 72 scanJsxAttributeValue(): SyntaxKind; 73 reScanJsxAttributeValue(): SyntaxKind; 74 reScanJsxToken(allowMultilineJsxText?: boolean): JsxTokenSyntaxKind; 75 reScanLessThanToken(): SyntaxKind; 76 reScanHashToken(): SyntaxKind; 77 reScanQuestionToken(): SyntaxKind; 78 reScanInvalidIdentifier(): SyntaxKind; 79 scanJsxToken(): JsxTokenSyntaxKind; 80 scanJsDocToken(): JSDocSyntaxKind; 81 scan(): SyntaxKind; 82 83 getText(): string; 84 /** @internal */ 85 clearCommentDirectives(): void; 86 // Sets the text for the scanner to scan. An optional subrange starting point and length 87 // can be provided to have the scanner only scan a portion of the text. 88 setText(text: string | undefined, start?: number, length?: number): void; 89 setOnError(onError: ErrorCallback | undefined): void; 90 setScriptTarget(scriptTarget: ScriptTarget): void; 91 setLanguageVariant(variant: LanguageVariant): void; 92 setTextPos(textPos: number): void; 93 /** @internal */ 94 setInJSDocType(inType: boolean): void; 95 // Invokes the provided callback then unconditionally restores the scanner to the state it 96 // was in immediately prior to invoking the callback. The result of invoking the callback 97 // is returned from this function. 98 lookAhead<T>(callback: () => T): T; 99 100 // Invokes the callback with the scanner set to scan the specified range. When the callback 101 // returns, the scanner is restored to the state it was in before scanRange was called. 102 scanRange<T>(start: number, length: number, callback: () => T): T; 103 104 // Invokes the provided callback. If the callback returns something falsy, then it restores 105 // the scanner to the state it was in immediately prior to invoking the callback. If the 106 // callback returns something truthy, then the scanner state is not rolled back. The result 107 // of invoking the callback is returned from this function. 108 tryScan<T>(callback: () => T): T; 109 setEtsContext(isEtsContext: boolean): void; 110} 111 112/** @internal */ 113export const textToKeywordObj: MapLike<KeywordSyntaxKind> = { 114 abstract: SyntaxKind.AbstractKeyword, 115 accessor: SyntaxKind.AccessorKeyword, 116 any: SyntaxKind.AnyKeyword, 117 as: SyntaxKind.AsKeyword, 118 asserts: SyntaxKind.AssertsKeyword, 119 assert: SyntaxKind.AssertKeyword, 120 bigint: SyntaxKind.BigIntKeyword, 121 boolean: SyntaxKind.BooleanKeyword, 122 break: SyntaxKind.BreakKeyword, 123 case: SyntaxKind.CaseKeyword, 124 catch: SyntaxKind.CatchKeyword, 125 class: SyntaxKind.ClassKeyword, 126 continue: SyntaxKind.ContinueKeyword, 127 const: SyntaxKind.ConstKeyword, 128 ["" + "constructor"]: SyntaxKind.ConstructorKeyword, 129 debugger: SyntaxKind.DebuggerKeyword, 130 declare: SyntaxKind.DeclareKeyword, 131 default: SyntaxKind.DefaultKeyword, 132 delete: SyntaxKind.DeleteKeyword, 133 do: SyntaxKind.DoKeyword, 134 else: SyntaxKind.ElseKeyword, 135 enum: SyntaxKind.EnumKeyword, 136 export: SyntaxKind.ExportKeyword, 137 extends: SyntaxKind.ExtendsKeyword, 138 false: SyntaxKind.FalseKeyword, 139 finally: SyntaxKind.FinallyKeyword, 140 for: SyntaxKind.ForKeyword, 141 from: SyntaxKind.FromKeyword, 142 function: SyntaxKind.FunctionKeyword, 143 get: SyntaxKind.GetKeyword, 144 if: SyntaxKind.IfKeyword, 145 implements: SyntaxKind.ImplementsKeyword, 146 import: SyntaxKind.ImportKeyword, 147 in: SyntaxKind.InKeyword, 148 infer: SyntaxKind.InferKeyword, 149 instanceof: SyntaxKind.InstanceOfKeyword, 150 interface: SyntaxKind.InterfaceKeyword, 151 intrinsic: SyntaxKind.IntrinsicKeyword, 152 is: SyntaxKind.IsKeyword, 153 keyof: SyntaxKind.KeyOfKeyword, 154 let: SyntaxKind.LetKeyword, 155 module: SyntaxKind.ModuleKeyword, 156 namespace: SyntaxKind.NamespaceKeyword, 157 never: SyntaxKind.NeverKeyword, 158 new: SyntaxKind.NewKeyword, 159 null: SyntaxKind.NullKeyword, 160 number: SyntaxKind.NumberKeyword, 161 object: SyntaxKind.ObjectKeyword, 162 package: SyntaxKind.PackageKeyword, 163 private: SyntaxKind.PrivateKeyword, 164 protected: SyntaxKind.ProtectedKeyword, 165 public: SyntaxKind.PublicKeyword, 166 override: SyntaxKind.OverrideKeyword, 167 out: SyntaxKind.OutKeyword, 168 readonly: SyntaxKind.ReadonlyKeyword, 169 require: SyntaxKind.RequireKeyword, 170 global: SyntaxKind.GlobalKeyword, 171 return: SyntaxKind.ReturnKeyword, 172 satisfies: SyntaxKind.SatisfiesKeyword, 173 set: SyntaxKind.SetKeyword, 174 static: SyntaxKind.StaticKeyword, 175 string: SyntaxKind.StringKeyword, 176 struct: SyntaxKind.StructKeyword, 177 super: SyntaxKind.SuperKeyword, 178 switch: SyntaxKind.SwitchKeyword, 179 symbol: SyntaxKind.SymbolKeyword, 180 this: SyntaxKind.ThisKeyword, 181 throw: SyntaxKind.ThrowKeyword, 182 true: SyntaxKind.TrueKeyword, 183 try: SyntaxKind.TryKeyword, 184 type: SyntaxKind.TypeKeyword, 185 lazy: SyntaxKind.LazyKeyword, 186 typeof: SyntaxKind.TypeOfKeyword, 187 undefined: SyntaxKind.UndefinedKeyword, 188 unique: SyntaxKind.UniqueKeyword, 189 unknown: SyntaxKind.UnknownKeyword, 190 var: SyntaxKind.VarKeyword, 191 void: SyntaxKind.VoidKeyword, 192 while: SyntaxKind.WhileKeyword, 193 with: SyntaxKind.WithKeyword, 194 yield: SyntaxKind.YieldKeyword, 195 async: SyntaxKind.AsyncKeyword, 196 await: SyntaxKind.AwaitKeyword, 197 of: SyntaxKind.OfKeyword, 198}; 199 200const textToKeyword = new Map(getEntries(textToKeywordObj)); 201 202const textToToken = new Map(getEntries({ 203 ...textToKeywordObj, 204 "{": SyntaxKind.OpenBraceToken, 205 "}": SyntaxKind.CloseBraceToken, 206 "(": SyntaxKind.OpenParenToken, 207 ")": SyntaxKind.CloseParenToken, 208 "[": SyntaxKind.OpenBracketToken, 209 "]": SyntaxKind.CloseBracketToken, 210 ".": SyntaxKind.DotToken, 211 "...": SyntaxKind.DotDotDotToken, 212 ";": SyntaxKind.SemicolonToken, 213 ",": SyntaxKind.CommaToken, 214 "<": SyntaxKind.LessThanToken, 215 ">": SyntaxKind.GreaterThanToken, 216 "<=": SyntaxKind.LessThanEqualsToken, 217 ">=": SyntaxKind.GreaterThanEqualsToken, 218 "==": SyntaxKind.EqualsEqualsToken, 219 "!=": SyntaxKind.ExclamationEqualsToken, 220 "===": SyntaxKind.EqualsEqualsEqualsToken, 221 "!==": SyntaxKind.ExclamationEqualsEqualsToken, 222 "=>": SyntaxKind.EqualsGreaterThanToken, 223 "+": SyntaxKind.PlusToken, 224 "-": SyntaxKind.MinusToken, 225 "**": SyntaxKind.AsteriskAsteriskToken, 226 "*": SyntaxKind.AsteriskToken, 227 "/": SyntaxKind.SlashToken, 228 "%": SyntaxKind.PercentToken, 229 "++": SyntaxKind.PlusPlusToken, 230 "--": SyntaxKind.MinusMinusToken, 231 "<<": SyntaxKind.LessThanLessThanToken, 232 "</": SyntaxKind.LessThanSlashToken, 233 ">>": SyntaxKind.GreaterThanGreaterThanToken, 234 ">>>": SyntaxKind.GreaterThanGreaterThanGreaterThanToken, 235 "&": SyntaxKind.AmpersandToken, 236 "|": SyntaxKind.BarToken, 237 "^": SyntaxKind.CaretToken, 238 "!": SyntaxKind.ExclamationToken, 239 "~": SyntaxKind.TildeToken, 240 "&&": SyntaxKind.AmpersandAmpersandToken, 241 "||": SyntaxKind.BarBarToken, 242 "?": SyntaxKind.QuestionToken, 243 "??": SyntaxKind.QuestionQuestionToken, 244 "?.": SyntaxKind.QuestionDotToken, 245 ":": SyntaxKind.ColonToken, 246 "=": SyntaxKind.EqualsToken, 247 "+=": SyntaxKind.PlusEqualsToken, 248 "-=": SyntaxKind.MinusEqualsToken, 249 "*=": SyntaxKind.AsteriskEqualsToken, 250 "**=": SyntaxKind.AsteriskAsteriskEqualsToken, 251 "/=": SyntaxKind.SlashEqualsToken, 252 "%=": SyntaxKind.PercentEqualsToken, 253 "<<=": SyntaxKind.LessThanLessThanEqualsToken, 254 ">>=": SyntaxKind.GreaterThanGreaterThanEqualsToken, 255 ">>>=": SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken, 256 "&=": SyntaxKind.AmpersandEqualsToken, 257 "|=": SyntaxKind.BarEqualsToken, 258 "^=": SyntaxKind.CaretEqualsToken, 259 "||=": SyntaxKind.BarBarEqualsToken, 260 "&&=": SyntaxKind.AmpersandAmpersandEqualsToken, 261 "??=": SyntaxKind.QuestionQuestionEqualsToken, 262 "@": SyntaxKind.AtToken, 263 "#": SyntaxKind.HashToken, 264 "`": SyntaxKind.BacktickToken, 265})); 266 267/* 268 As per ECMAScript Language Specification 3th Edition, Section 7.6: Identifiers 269 IdentifierStart :: 270 Can contain Unicode 3.0.0 categories: 271 Uppercase letter (Lu), 272 Lowercase letter (Ll), 273 Titlecase letter (Lt), 274 Modifier letter (Lm), 275 Other letter (Lo), or 276 Letter number (Nl). 277 IdentifierPart :: = 278 Can contain IdentifierStart + Unicode 3.0.0 categories: 279 Non-spacing mark (Mn), 280 Combining spacing mark (Mc), 281 Decimal number (Nd), or 282 Connector punctuation (Pc). 283 284 Codepoint ranges for ES3 Identifiers are extracted from the Unicode 3.0.0 specification at: 285 http://www.unicode.org/Public/3.0-Update/UnicodeData-3.0.0.txt 286*/ 287const unicodeES3IdentifierStart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 543, 546, 563, 592, 685, 688, 696, 699, 705, 720, 721, 736, 740, 750, 750, 890, 890, 902, 902, 904, 906, 908, 908, 910, 929, 931, 974, 976, 983, 986, 1011, 1024, 1153, 1164, 1220, 1223, 1224, 1227, 1228, 1232, 1269, 1272, 1273, 1329, 1366, 1369, 1369, 1377, 1415, 1488, 1514, 1520, 1522, 1569, 1594, 1600, 1610, 1649, 1747, 1749, 1749, 1765, 1766, 1786, 1788, 1808, 1808, 1810, 1836, 1920, 1957, 2309, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2524, 2525, 2527, 2529, 2544, 2545, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2699, 2701, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2784, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2870, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 2997, 2999, 3001, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3168, 3169, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3294, 3294, 3296, 3297, 3333, 3340, 3342, 3344, 3346, 3368, 3370, 3385, 3424, 3425, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3805, 3840, 3840, 3904, 3911, 3913, 3946, 3976, 3979, 4096, 4129, 4131, 4135, 4137, 4138, 4176, 4181, 4256, 4293, 4304, 4342, 4352, 4441, 4447, 4514, 4520, 4601, 4608, 4614, 4616, 4678, 4680, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4742, 4744, 4744, 4746, 4749, 4752, 4782, 4784, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4814, 4816, 4822, 4824, 4846, 4848, 4878, 4880, 4880, 4882, 4885, 4888, 4894, 4896, 4934, 4936, 4954, 5024, 5108, 5121, 5740, 5743, 5750, 5761, 5786, 5792, 5866, 6016, 6067, 6176, 6263, 6272, 6312, 7680, 7835, 7840, 7929, 7936, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8319, 8319, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8497, 8499, 8505, 8544, 8579, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12346, 12353, 12436, 12445, 12446, 12449, 12538, 12540, 12542, 12549, 12588, 12593, 12686, 12704, 12727, 13312, 19893, 19968, 40869, 40960, 42124, 44032, 55203, 63744, 64045, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65138, 65140, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 288const unicodeES3IdentifierPart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 543, 546, 563, 592, 685, 688, 696, 699, 705, 720, 721, 736, 740, 750, 750, 768, 846, 864, 866, 890, 890, 902, 902, 904, 906, 908, 908, 910, 929, 931, 974, 976, 983, 986, 1011, 1024, 1153, 1155, 1158, 1164, 1220, 1223, 1224, 1227, 1228, 1232, 1269, 1272, 1273, 1329, 1366, 1369, 1369, 1377, 1415, 1425, 1441, 1443, 1465, 1467, 1469, 1471, 1471, 1473, 1474, 1476, 1476, 1488, 1514, 1520, 1522, 1569, 1594, 1600, 1621, 1632, 1641, 1648, 1747, 1749, 1756, 1759, 1768, 1770, 1773, 1776, 1788, 1808, 1836, 1840, 1866, 1920, 1968, 2305, 2307, 2309, 2361, 2364, 2381, 2384, 2388, 2392, 2403, 2406, 2415, 2433, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2562, 2562, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2649, 2652, 2654, 2654, 2662, 2676, 2689, 2691, 2693, 2699, 2701, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2784, 2790, 2799, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2870, 2873, 2876, 2883, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2913, 2918, 2927, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 2997, 2999, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3031, 3031, 3047, 3055, 3073, 3075, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3168, 3169, 3174, 3183, 3202, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3262, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3297, 3302, 3311, 3330, 3331, 3333, 3340, 3342, 3344, 3346, 3368, 3370, 3385, 3390, 3395, 3398, 3400, 3402, 3405, 3415, 3415, 3424, 3425, 3430, 3439, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3769, 3771, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3805, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3946, 3953, 3972, 3974, 3979, 3984, 3991, 3993, 4028, 4038, 4038, 4096, 4129, 4131, 4135, 4137, 4138, 4140, 4146, 4150, 4153, 4160, 4169, 4176, 4185, 4256, 4293, 4304, 4342, 4352, 4441, 4447, 4514, 4520, 4601, 4608, 4614, 4616, 4678, 4680, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4742, 4744, 4744, 4746, 4749, 4752, 4782, 4784, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4814, 4816, 4822, 4824, 4846, 4848, 4878, 4880, 4880, 4882, 4885, 4888, 4894, 4896, 4934, 4936, 4954, 4969, 4977, 5024, 5108, 5121, 5740, 5743, 5750, 5761, 5786, 5792, 5866, 6016, 6099, 6112, 6121, 6160, 6169, 6176, 6263, 6272, 6313, 7680, 7835, 7840, 7929, 7936, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8255, 8256, 8319, 8319, 8400, 8412, 8417, 8417, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8497, 8499, 8505, 8544, 8579, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12346, 12353, 12436, 12441, 12442, 12445, 12446, 12449, 12542, 12549, 12588, 12593, 12686, 12704, 12727, 13312, 19893, 19968, 40869, 40960, 42124, 44032, 55203, 63744, 64045, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65056, 65059, 65075, 65076, 65101, 65103, 65136, 65138, 65140, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65381, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 289 290/* 291 As per ECMAScript Language Specification 5th Edition, Section 7.6: ISyntaxToken Names and Identifiers 292 IdentifierStart :: 293 Can contain Unicode 6.2 categories: 294 Uppercase letter (Lu), 295 Lowercase letter (Ll), 296 Titlecase letter (Lt), 297 Modifier letter (Lm), 298 Other letter (Lo), or 299 Letter number (Nl). 300 IdentifierPart :: 301 Can contain IdentifierStart + Unicode 6.2 categories: 302 Non-spacing mark (Mn), 303 Combining spacing mark (Mc), 304 Decimal number (Nd), 305 Connector punctuation (Pc), 306 <ZWNJ>, or 307 <ZWJ>. 308 309 Codepoint ranges for ES5 Identifiers are extracted from the Unicode 6.2 specification at: 310 http://www.unicode.org/Public/6.2.0/ucd/UnicodeData.txt 311*/ 312const unicodeES5IdentifierStart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1319, 1329, 1366, 1369, 1369, 1377, 1415, 1488, 1514, 1520, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2208, 2208, 2210, 2220, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2423, 2425, 2431, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3133, 3133, 3160, 3161, 3168, 3169, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3294, 3294, 3296, 3297, 3313, 3314, 3333, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3424, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5108, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5872, 5888, 5900, 5902, 5905, 5920, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6263, 6272, 6312, 6314, 6314, 6320, 6389, 6400, 6428, 6480, 6509, 6512, 6516, 6528, 6571, 6593, 6599, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6987, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7401, 7404, 7406, 7409, 7413, 7414, 7424, 7615, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11823, 11823, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12589, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40908, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42647, 42656, 42735, 42775, 42783, 42786, 42888, 42891, 42894, 42896, 42899, 42912, 42922, 43000, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642, 43648, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43968, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 313const unicodeES5IdentifierPart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 768, 884, 886, 887, 890, 893, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1155, 1159, 1162, 1319, 1329, 1366, 1369, 1369, 1377, 1415, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1488, 1514, 1520, 1522, 1552, 1562, 1568, 1641, 1646, 1747, 1749, 1756, 1759, 1768, 1770, 1788, 1791, 1791, 1808, 1866, 1869, 1969, 1984, 2037, 2042, 2042, 2048, 2093, 2112, 2139, 2208, 2208, 2210, 2220, 2276, 2302, 2304, 2403, 2406, 2415, 2417, 2423, 2425, 2431, 2433, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504, 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2561, 2563, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2649, 2652, 2654, 2654, 2662, 2677, 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2799, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2876, 2884, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2915, 2918, 2927, 2929, 2929, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024, 3031, 3031, 3046, 3055, 3073, 3075, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3133, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3160, 3161, 3168, 3171, 3174, 3183, 3202, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3299, 3302, 3311, 3313, 3314, 3330, 3331, 3333, 3340, 3342, 3344, 3346, 3386, 3389, 3396, 3398, 3400, 3402, 3406, 3415, 3415, 3424, 3427, 3430, 3439, 3450, 3455, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3769, 3771, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3807, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3948, 3953, 3972, 3974, 3991, 3993, 4028, 4038, 4038, 4096, 4169, 4176, 4253, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4957, 4959, 4992, 5007, 5024, 5108, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5872, 5888, 5900, 5902, 5908, 5920, 5940, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6099, 6103, 6103, 6108, 6109, 6112, 6121, 6155, 6157, 6160, 6169, 6176, 6263, 6272, 6314, 6320, 6389, 6400, 6428, 6432, 6443, 6448, 6459, 6470, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6617, 6656, 6683, 6688, 6750, 6752, 6780, 6783, 6793, 6800, 6809, 6823, 6823, 6912, 6987, 6992, 7001, 7019, 7027, 7040, 7155, 7168, 7223, 7232, 7241, 7245, 7293, 7376, 7378, 7380, 7414, 7424, 7654, 7676, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8204, 8205, 8255, 8256, 8276, 8276, 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8412, 8417, 8417, 8421, 8432, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11647, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11744, 11775, 11823, 11823, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12348, 12353, 12438, 12441, 12442, 12445, 12447, 12449, 12538, 12540, 12543, 12549, 12589, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40908, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42539, 42560, 42607, 42612, 42621, 42623, 42647, 42655, 42737, 42775, 42783, 42786, 42888, 42891, 42894, 42896, 42899, 42912, 42922, 43000, 43047, 43072, 43123, 43136, 43204, 43216, 43225, 43232, 43255, 43259, 43259, 43264, 43309, 43312, 43347, 43360, 43388, 43392, 43456, 43471, 43481, 43520, 43574, 43584, 43597, 43600, 43609, 43616, 43638, 43642, 43643, 43648, 43714, 43739, 43741, 43744, 43759, 43762, 43766, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43968, 44010, 44012, 44013, 44016, 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65024, 65039, 65056, 65062, 65075, 65076, 65101, 65103, 65136, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ]; 314 315/** 316 * Generated by scripts/regenerate-unicode-identifier-parts.js on node v12.4.0 with unicode 12.1 317 * based on http://www.unicode.org/reports/tr31/ and https://www.ecma-international.org/ecma-262/6.0/#sec-names-and-keywords 318 * unicodeESNextIdentifierStart corresponds to the ID_Start and Other_ID_Start property, and 319 * unicodeESNextIdentifierPart corresponds to ID_Continue, Other_ID_Continue, plus ID_Start and Other_ID_Start 320 */ 321const unicodeESNextIdentifierStart = [65, 90, 97, 122, 170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 880, 884, 886, 887, 890, 893, 895, 895, 902, 902, 904, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1488, 1514, 1519, 1522, 1568, 1610, 1646, 1647, 1649, 1747, 1749, 1749, 1765, 1766, 1774, 1775, 1786, 1788, 1791, 1791, 1808, 1808, 1810, 1839, 1869, 1957, 1969, 1969, 1994, 2026, 2036, 2037, 2042, 2042, 2048, 2069, 2074, 2074, 2084, 2084, 2088, 2088, 2112, 2136, 2144, 2154, 2208, 2228, 2230, 2237, 2308, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2417, 2432, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2493, 2493, 2510, 2510, 2524, 2525, 2527, 2529, 2544, 2545, 2556, 2556, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2785, 2809, 2809, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2929, 2929, 2947, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3024, 3024, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3133, 3160, 3162, 3168, 3169, 3200, 3200, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3261, 3261, 3294, 3294, 3296, 3297, 3313, 3314, 3333, 3340, 3342, 3344, 3346, 3386, 3389, 3389, 3406, 3406, 3412, 3414, 3423, 3425, 3450, 3455, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3807, 3840, 3840, 3904, 3911, 3913, 3948, 3976, 3980, 4096, 4138, 4159, 4159, 4176, 4181, 4186, 4189, 4193, 4193, 4197, 4198, 4206, 4208, 4213, 4225, 4238, 4238, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5880, 5888, 5900, 5902, 5905, 5920, 5937, 5952, 5969, 5984, 5996, 5998, 6000, 6016, 6067, 6103, 6103, 6108, 6108, 6176, 6264, 6272, 6312, 6314, 6314, 6320, 6389, 6400, 6430, 6480, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6656, 6678, 6688, 6740, 6823, 6823, 6917, 6963, 6981, 6987, 7043, 7072, 7086, 7087, 7098, 7141, 7168, 7203, 7245, 7247, 7258, 7293, 7296, 7304, 7312, 7354, 7357, 7359, 7401, 7404, 7406, 7411, 7413, 7414, 7418, 7418, 7424, 7615, 7680, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8305, 8305, 8319, 8319, 8336, 8348, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8472, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11502, 11506, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11648, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12348, 12353, 12438, 12443, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40943, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42527, 42538, 42539, 42560, 42606, 42623, 42653, 42656, 42735, 42775, 42783, 42786, 42888, 42891, 42943, 42946, 42950, 42999, 43009, 43011, 43013, 43015, 43018, 43020, 43042, 43072, 43123, 43138, 43187, 43250, 43255, 43259, 43259, 43261, 43262, 43274, 43301, 43312, 43334, 43360, 43388, 43396, 43442, 43471, 43471, 43488, 43492, 43494, 43503, 43514, 43518, 43520, 43560, 43584, 43586, 43588, 43595, 43616, 43638, 43642, 43642, 43646, 43695, 43697, 43697, 43701, 43702, 43705, 43709, 43712, 43712, 43714, 43714, 43739, 43741, 43744, 43754, 43762, 43764, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43879, 43888, 44002, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65856, 65908, 66176, 66204, 66208, 66256, 66304, 66335, 66349, 66378, 66384, 66421, 66432, 66461, 66464, 66499, 66504, 66511, 66513, 66517, 66560, 66717, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 67072, 67382, 67392, 67413, 67424, 67431, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68096, 68112, 68115, 68117, 68119, 68121, 68149, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68324, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, 68800, 68850, 68864, 68899, 69376, 69404, 69415, 69415, 69424, 69445, 69600, 69622, 69635, 69687, 69763, 69807, 69840, 69864, 69891, 69926, 69956, 69956, 69968, 70002, 70006, 70006, 70019, 70066, 70081, 70084, 70106, 70106, 70108, 70108, 70144, 70161, 70163, 70187, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70366, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70461, 70461, 70480, 70480, 70493, 70497, 70656, 70708, 70727, 70730, 70751, 70751, 70784, 70831, 70852, 70853, 70855, 70855, 71040, 71086, 71128, 71131, 71168, 71215, 71236, 71236, 71296, 71338, 71352, 71352, 71424, 71450, 71680, 71723, 71840, 71903, 71935, 71935, 72096, 72103, 72106, 72144, 72161, 72161, 72163, 72163, 72192, 72192, 72203, 72242, 72250, 72250, 72272, 72272, 72284, 72329, 72349, 72349, 72384, 72440, 72704, 72712, 72714, 72750, 72768, 72768, 72818, 72847, 72960, 72966, 72968, 72969, 72971, 73008, 73030, 73030, 73056, 73061, 73063, 73064, 73066, 73097, 73112, 73112, 73440, 73458, 73728, 74649, 74752, 74862, 74880, 75075, 77824, 78894, 82944, 83526, 92160, 92728, 92736, 92766, 92880, 92909, 92928, 92975, 92992, 92995, 93027, 93047, 93053, 93071, 93760, 93823, 93952, 94026, 94032, 94032, 94099, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101106, 110592, 110878, 110928, 110930, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, 123136, 123180, 123191, 123197, 123214, 123214, 123584, 123627, 124928, 125124, 125184, 125251, 125259, 125259, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173782, 173824, 177972, 177984, 178205, 178208, 183969, 183984, 191456, 194560, 195101]; 322const unicodeESNextIdentifierPart = [48, 57, 65, 90, 95, 95, 97, 122, 170, 170, 181, 181, 183, 183, 186, 186, 192, 214, 216, 246, 248, 705, 710, 721, 736, 740, 748, 748, 750, 750, 768, 884, 886, 887, 890, 893, 895, 895, 902, 906, 908, 908, 910, 929, 931, 1013, 1015, 1153, 1155, 1159, 1162, 1327, 1329, 1366, 1369, 1369, 1376, 1416, 1425, 1469, 1471, 1471, 1473, 1474, 1476, 1477, 1479, 1479, 1488, 1514, 1519, 1522, 1552, 1562, 1568, 1641, 1646, 1747, 1749, 1756, 1759, 1768, 1770, 1788, 1791, 1791, 1808, 1866, 1869, 1969, 1984, 2037, 2042, 2042, 2045, 2045, 2048, 2093, 2112, 2139, 2144, 2154, 2208, 2228, 2230, 2237, 2259, 2273, 2275, 2403, 2406, 2415, 2417, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2500, 2503, 2504, 2507, 2510, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2556, 2556, 2558, 2558, 2561, 2563, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2641, 2641, 2649, 2652, 2654, 2654, 2662, 2677, 2689, 2691, 2693, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2787, 2790, 2799, 2809, 2815, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2869, 2873, 2876, 2884, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2915, 2918, 2927, 2929, 2929, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3024, 3024, 3031, 3031, 3046, 3055, 3072, 3084, 3086, 3088, 3090, 3112, 3114, 3129, 3133, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3160, 3162, 3168, 3171, 3174, 3183, 3200, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3260, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3299, 3302, 3311, 3313, 3314, 3328, 3331, 3333, 3340, 3342, 3344, 3346, 3396, 3398, 3400, 3402, 3406, 3412, 3415, 3423, 3427, 3430, 3439, 3450, 3455, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3558, 3567, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3718, 3722, 3724, 3747, 3749, 3749, 3751, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3807, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3948, 3953, 3972, 3974, 3991, 3993, 4028, 4038, 4038, 4096, 4169, 4176, 4253, 4256, 4293, 4295, 4295, 4301, 4301, 4304, 4346, 4348, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4744, 4746, 4749, 4752, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4822, 4824, 4880, 4882, 4885, 4888, 4954, 4957, 4959, 4969, 4977, 4992, 5007, 5024, 5109, 5112, 5117, 5121, 5740, 5743, 5759, 5761, 5786, 5792, 5866, 5870, 5880, 5888, 5900, 5902, 5908, 5920, 5940, 5952, 5971, 5984, 5996, 5998, 6000, 6002, 6003, 6016, 6099, 6103, 6103, 6108, 6109, 6112, 6121, 6155, 6157, 6160, 6169, 6176, 6264, 6272, 6314, 6320, 6389, 6400, 6430, 6432, 6443, 6448, 6459, 6470, 6509, 6512, 6516, 6528, 6571, 6576, 6601, 6608, 6618, 6656, 6683, 6688, 6750, 6752, 6780, 6783, 6793, 6800, 6809, 6823, 6823, 6832, 6845, 6912, 6987, 6992, 7001, 7019, 7027, 7040, 7155, 7168, 7223, 7232, 7241, 7245, 7293, 7296, 7304, 7312, 7354, 7357, 7359, 7376, 7378, 7380, 7418, 7424, 7673, 7675, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8255, 8256, 8276, 8276, 8305, 8305, 8319, 8319, 8336, 8348, 8400, 8412, 8417, 8417, 8421, 8432, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8472, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8505, 8508, 8511, 8517, 8521, 8526, 8526, 8544, 8584, 11264, 11310, 11312, 11358, 11360, 11492, 11499, 11507, 11520, 11557, 11559, 11559, 11565, 11565, 11568, 11623, 11631, 11631, 11647, 11670, 11680, 11686, 11688, 11694, 11696, 11702, 11704, 11710, 11712, 11718, 11720, 11726, 11728, 11734, 11736, 11742, 11744, 11775, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12348, 12353, 12438, 12441, 12447, 12449, 12538, 12540, 12543, 12549, 12591, 12593, 12686, 12704, 12730, 12784, 12799, 13312, 19893, 19968, 40943, 40960, 42124, 42192, 42237, 42240, 42508, 42512, 42539, 42560, 42607, 42612, 42621, 42623, 42737, 42775, 42783, 42786, 42888, 42891, 42943, 42946, 42950, 42999, 43047, 43072, 43123, 43136, 43205, 43216, 43225, 43232, 43255, 43259, 43259, 43261, 43309, 43312, 43347, 43360, 43388, 43392, 43456, 43471, 43481, 43488, 43518, 43520, 43574, 43584, 43597, 43600, 43609, 43616, 43638, 43642, 43714, 43739, 43741, 43744, 43759, 43762, 43766, 43777, 43782, 43785, 43790, 43793, 43798, 43808, 43814, 43816, 43822, 43824, 43866, 43868, 43879, 43888, 44010, 44012, 44013, 44016, 44025, 44032, 55203, 55216, 55238, 55243, 55291, 63744, 64109, 64112, 64217, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65024, 65039, 65056, 65071, 65075, 65076, 65101, 65103, 65136, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, 65536, 65547, 65549, 65574, 65576, 65594, 65596, 65597, 65599, 65613, 65616, 65629, 65664, 65786, 65856, 65908, 66045, 66045, 66176, 66204, 66208, 66256, 66272, 66272, 66304, 66335, 66349, 66378, 66384, 66426, 66432, 66461, 66464, 66499, 66504, 66511, 66513, 66517, 66560, 66717, 66720, 66729, 66736, 66771, 66776, 66811, 66816, 66855, 66864, 66915, 67072, 67382, 67392, 67413, 67424, 67431, 67584, 67589, 67592, 67592, 67594, 67637, 67639, 67640, 67644, 67644, 67647, 67669, 67680, 67702, 67712, 67742, 67808, 67826, 67828, 67829, 67840, 67861, 67872, 67897, 67968, 68023, 68030, 68031, 68096, 68099, 68101, 68102, 68108, 68115, 68117, 68119, 68121, 68149, 68152, 68154, 68159, 68159, 68192, 68220, 68224, 68252, 68288, 68295, 68297, 68326, 68352, 68405, 68416, 68437, 68448, 68466, 68480, 68497, 68608, 68680, 68736, 68786, 68800, 68850, 68864, 68903, 68912, 68921, 69376, 69404, 69415, 69415, 69424, 69456, 69600, 69622, 69632, 69702, 69734, 69743, 69759, 69818, 69840, 69864, 69872, 69881, 69888, 69940, 69942, 69951, 69956, 69958, 69968, 70003, 70006, 70006, 70016, 70084, 70089, 70092, 70096, 70106, 70108, 70108, 70144, 70161, 70163, 70199, 70206, 70206, 70272, 70278, 70280, 70280, 70282, 70285, 70287, 70301, 70303, 70312, 70320, 70378, 70384, 70393, 70400, 70403, 70405, 70412, 70415, 70416, 70419, 70440, 70442, 70448, 70450, 70451, 70453, 70457, 70459, 70468, 70471, 70472, 70475, 70477, 70480, 70480, 70487, 70487, 70493, 70499, 70502, 70508, 70512, 70516, 70656, 70730, 70736, 70745, 70750, 70751, 70784, 70853, 70855, 70855, 70864, 70873, 71040, 71093, 71096, 71104, 71128, 71133, 71168, 71232, 71236, 71236, 71248, 71257, 71296, 71352, 71360, 71369, 71424, 71450, 71453, 71467, 71472, 71481, 71680, 71738, 71840, 71913, 71935, 71935, 72096, 72103, 72106, 72151, 72154, 72161, 72163, 72164, 72192, 72254, 72263, 72263, 72272, 72345, 72349, 72349, 72384, 72440, 72704, 72712, 72714, 72758, 72760, 72768, 72784, 72793, 72818, 72847, 72850, 72871, 72873, 72886, 72960, 72966, 72968, 72969, 72971, 73014, 73018, 73018, 73020, 73021, 73023, 73031, 73040, 73049, 73056, 73061, 73063, 73064, 73066, 73102, 73104, 73105, 73107, 73112, 73120, 73129, 73440, 73462, 73728, 74649, 74752, 74862, 74880, 75075, 77824, 78894, 82944, 83526, 92160, 92728, 92736, 92766, 92768, 92777, 92880, 92909, 92912, 92916, 92928, 92982, 92992, 92995, 93008, 93017, 93027, 93047, 93053, 93071, 93760, 93823, 93952, 94026, 94031, 94087, 94095, 94111, 94176, 94177, 94179, 94179, 94208, 100343, 100352, 101106, 110592, 110878, 110928, 110930, 110948, 110951, 110960, 111355, 113664, 113770, 113776, 113788, 113792, 113800, 113808, 113817, 113821, 113822, 119141, 119145, 119149, 119154, 119163, 119170, 119173, 119179, 119210, 119213, 119362, 119364, 119808, 119892, 119894, 119964, 119966, 119967, 119970, 119970, 119973, 119974, 119977, 119980, 119982, 119993, 119995, 119995, 119997, 120003, 120005, 120069, 120071, 120074, 120077, 120084, 120086, 120092, 120094, 120121, 120123, 120126, 120128, 120132, 120134, 120134, 120138, 120144, 120146, 120485, 120488, 120512, 120514, 120538, 120540, 120570, 120572, 120596, 120598, 120628, 120630, 120654, 120656, 120686, 120688, 120712, 120714, 120744, 120746, 120770, 120772, 120779, 120782, 120831, 121344, 121398, 121403, 121452, 121461, 121461, 121476, 121476, 121499, 121503, 121505, 121519, 122880, 122886, 122888, 122904, 122907, 122913, 122915, 122916, 122918, 122922, 123136, 123180, 123184, 123197, 123200, 123209, 123214, 123214, 123584, 123641, 124928, 125124, 125136, 125142, 125184, 125259, 125264, 125273, 126464, 126467, 126469, 126495, 126497, 126498, 126500, 126500, 126503, 126503, 126505, 126514, 126516, 126519, 126521, 126521, 126523, 126523, 126530, 126530, 126535, 126535, 126537, 126537, 126539, 126539, 126541, 126543, 126545, 126546, 126548, 126548, 126551, 126551, 126553, 126553, 126555, 126555, 126557, 126557, 126559, 126559, 126561, 126562, 126564, 126564, 126567, 126570, 126572, 126578, 126580, 126583, 126585, 126588, 126590, 126590, 126592, 126601, 126603, 126619, 126625, 126627, 126629, 126633, 126635, 126651, 131072, 173782, 173824, 177972, 177984, 178205, 178208, 183969, 183984, 191456, 194560, 195101, 917760, 917999]; 323 324/** 325 * Test for whether a single line comment with leading whitespace trimmed's text contains a directive. 326 */ 327const commentDirectiveRegExSingleLine = /^\/\/\/?\s*@(ts-expect-error|ts-ignore)/; 328 329/** 330 * Test for whether a multi-line comment with leading whitespace trimmed's last line contains a directive. 331 */ 332const commentDirectiveRegExMultiLine = /^(?:\/|\*)*\s*@(ts-expect-error|ts-ignore)/; 333 334function lookupInUnicodeMap(code: number, map: readonly number[]): boolean { 335 // Bail out quickly if it couldn't possibly be in the map. 336 if (code < map[0]) { 337 return false; 338 } 339 340 // Perform binary search in one of the Unicode range maps 341 let lo = 0; 342 let hi: number = map.length; 343 let mid: number; 344 345 while (lo + 1 < hi) { 346 mid = lo + (hi - lo) / 2; 347 // mid has to be even to catch a range's beginning 348 mid -= mid % 2; 349 if (map[mid] <= code && code <= map[mid + 1]) { 350 return true; 351 } 352 353 if (code < map[mid]) { 354 hi = mid; 355 } 356 else { 357 lo = mid + 2; 358 } 359 } 360 361 return false; 362} 363 364/** @internal */ export function isUnicodeIdentifierStart(code: number, languageVersion: ScriptTarget | undefined) { 365 return languageVersion! >= ScriptTarget.ES2015 ? 366 lookupInUnicodeMap(code, unicodeESNextIdentifierStart) : 367 languageVersion === ScriptTarget.ES5 ? lookupInUnicodeMap(code, unicodeES5IdentifierStart) : 368 lookupInUnicodeMap(code, unicodeES3IdentifierStart); 369} 370 371function isUnicodeIdentifierPart(code: number, languageVersion: ScriptTarget | undefined) { 372 return languageVersion! >= ScriptTarget.ES2015 ? 373 lookupInUnicodeMap(code, unicodeESNextIdentifierPart) : 374 languageVersion === ScriptTarget.ES5 ? lookupInUnicodeMap(code, unicodeES5IdentifierPart) : 375 lookupInUnicodeMap(code, unicodeES3IdentifierPart); 376} 377 378function makeReverseMap(source: ESMap<string, number>): string[] { 379 const result: string[] = []; 380 source.forEach((value, name) => { 381 result[value] = name; 382 }); 383 return result; 384} 385 386const tokenStrings = makeReverseMap(textToToken); 387export function tokenToString(t: SyntaxKind): string | undefined { 388 return tokenStrings[t]; 389} 390 391/** @internal */ 392export function stringToToken(s: string): SyntaxKind | undefined { 393 return textToToken.get(s); 394} 395 396/** @internal */ 397export function computeLineStarts(text: string): number[] { 398 const result: number[] = []; 399 let pos = 0; 400 let lineStart = 0; 401 while (pos < text.length) { 402 const ch = text.charCodeAt(pos); 403 pos++; 404 switch (ch) { 405 case CharacterCodes.carriageReturn: 406 if (text.charCodeAt(pos) === CharacterCodes.lineFeed) { 407 pos++; 408 } 409 // falls through 410 case CharacterCodes.lineFeed: 411 result.push(lineStart); 412 lineStart = pos; 413 break; 414 default: 415 if (ch > CharacterCodes.maxAsciiCharacter && isLineBreak(ch)) { 416 result.push(lineStart); 417 lineStart = pos; 418 } 419 break; 420 } 421 } 422 result.push(lineStart); 423 return result; 424} 425 426export function getPositionOfLineAndCharacter(sourceFile: SourceFileLike, line: number, character: number): number; 427/** @internal */ 428export function getPositionOfLineAndCharacter(sourceFile: SourceFileLike, line: number, character: number, allowEdits?: true): number; // eslint-disable-line @typescript-eslint/unified-signatures 429export function getPositionOfLineAndCharacter(sourceFile: SourceFileLike, line: number, character: number, allowEdits?: true): number { 430 return sourceFile.getPositionOfLineAndCharacter ? 431 sourceFile.getPositionOfLineAndCharacter(line, character, allowEdits) : 432 computePositionOfLineAndCharacter(getLineStarts(sourceFile), line, character, sourceFile.text, allowEdits); 433} 434 435/** @internal */ 436export function computePositionOfLineAndCharacter(lineStarts: readonly number[], line: number, character: number, debugText?: string, allowEdits?: true): number { 437 if (line < 0 || line >= lineStarts.length) { 438 if (allowEdits) { 439 // Clamp line to nearest allowable value 440 line = line < 0 ? 0 : line >= lineStarts.length ? lineStarts.length - 1 : line; 441 } 442 else { 443 Debug.fail(`Bad line number. Line: ${line}, lineStarts.length: ${lineStarts.length} , line map is correct? ${debugText !== undefined ? arraysEqual(lineStarts, computeLineStarts(debugText)) : "unknown"}`); 444 } 445 } 446 447 const res = lineStarts[line] + character; 448 if (allowEdits) { 449 // Clamp to nearest allowable values to allow the underlying to be edited without crashing (accuracy is lost, instead) 450 // TODO: Somehow track edits between file as it was during the creation of sourcemap we have and the current file and 451 // apply them to the computed position to improve accuracy 452 return res > lineStarts[line + 1] ? lineStarts[line + 1] : typeof debugText === "string" && res > debugText.length ? debugText.length : res; 453 } 454 if (line < lineStarts.length - 1) { 455 Debug.assert(res < lineStarts[line + 1]); 456 } 457 else if (debugText !== undefined) { 458 Debug.assert(res <= debugText.length); // Allow single character overflow for trailing newline 459 } 460 return res; 461} 462 463/** @internal */ 464export function getLineStarts(sourceFile: SourceFileLike): readonly number[] { 465 return sourceFile.lineMap || (sourceFile.lineMap = computeLineStarts(sourceFile.text)); 466} 467 468/** @internal */ 469export function computeLineAndCharacterOfPosition(lineStarts: readonly number[], position: number): LineAndCharacter { 470 const lineNumber = computeLineOfPosition(lineStarts, position); 471 return { 472 line: lineNumber, 473 character: position - lineStarts[lineNumber] 474 }; 475} 476 477/** 478 * @internal 479 * We assume the first line starts at position 0 and 'position' is non-negative. 480 */ 481export function computeLineOfPosition(lineStarts: readonly number[], position: number, lowerBound?: number) { 482 let lineNumber = binarySearch(lineStarts, position, identity, compareValues, lowerBound); 483 if (lineNumber < 0) { 484 // If the actual position was not found, 485 // the binary search returns the 2's-complement of the next line start 486 // e.g. if the line starts at [5, 10, 23, 80] and the position requested was 20 487 // then the search will return -2. 488 // 489 // We want the index of the previous line start, so we subtract 1. 490 // Review 2's-complement if this is confusing. 491 lineNumber = ~lineNumber - 1; 492 Debug.assert(lineNumber !== -1, "position cannot precede the beginning of the file"); 493 } 494 return lineNumber; 495} 496 497/** @internal */ 498export function getLinesBetweenPositions(sourceFile: SourceFileLike, pos1: number, pos2: number) { 499 if (pos1 === pos2) return 0; 500 const lineStarts = getLineStarts(sourceFile); 501 const lower = Math.min(pos1, pos2); 502 const isNegative = lower === pos2; 503 const upper = isNegative ? pos1 : pos2; 504 const lowerLine = computeLineOfPosition(lineStarts, lower); 505 const upperLine = computeLineOfPosition(lineStarts, upper, lowerLine); 506 return isNegative ? lowerLine - upperLine : upperLine - lowerLine; 507} 508 509export function getLineAndCharacterOfPosition(sourceFile: SourceFileLike, position: number): LineAndCharacter { 510 return computeLineAndCharacterOfPosition(getLineStarts(sourceFile), position); 511} 512 513export function isWhiteSpaceLike(ch: number): boolean { 514 return isWhiteSpaceSingleLine(ch) || isLineBreak(ch); 515} 516 517/** Does not include line breaks. For that, see isWhiteSpaceLike. */ 518export function isWhiteSpaceSingleLine(ch: number): boolean { 519 // Note: nextLine is in the Zs space, and should be considered to be a whitespace. 520 // It is explicitly not a line-break as it isn't in the exact set specified by EcmaScript. 521 return ch === CharacterCodes.space || 522 ch === CharacterCodes.tab || 523 ch === CharacterCodes.verticalTab || 524 ch === CharacterCodes.formFeed || 525 ch === CharacterCodes.nonBreakingSpace || 526 ch === CharacterCodes.nextLine || 527 ch === CharacterCodes.ogham || 528 ch >= CharacterCodes.enQuad && ch <= CharacterCodes.zeroWidthSpace || 529 ch === CharacterCodes.narrowNoBreakSpace || 530 ch === CharacterCodes.mathematicalSpace || 531 ch === CharacterCodes.ideographicSpace || 532 ch === CharacterCodes.byteOrderMark; 533} 534 535export function isLineBreak(ch: number): boolean { 536 // ES5 7.3: 537 // The ECMAScript line terminator characters are listed in Table 3. 538 // Table 3: Line Terminator Characters 539 // Code Unit Value Name Formal Name 540 // \u000A Line Feed <LF> 541 // \u000D Carriage Return <CR> 542 // \u2028 Line separator <LS> 543 // \u2029 Paragraph separator <PS> 544 // Only the characters in Table 3 are treated as line terminators. Other new line or line 545 // breaking characters are treated as white space but not as line terminators. 546 547 return ch === CharacterCodes.lineFeed || 548 ch === CharacterCodes.carriageReturn || 549 ch === CharacterCodes.lineSeparator || 550 ch === CharacterCodes.paragraphSeparator; 551} 552 553function isDigit(ch: number): boolean { 554 return ch >= CharacterCodes._0 && ch <= CharacterCodes._9; 555} 556 557function isHexDigit(ch: number): boolean { 558 return isDigit(ch) || ch >= CharacterCodes.A && ch <= CharacterCodes.F || ch >= CharacterCodes.a && ch <= CharacterCodes.f; 559} 560 561function isCodePoint(code: number): boolean { 562 return code <= 0x10FFFF; 563} 564 565/** @internal */ 566export function isOctalDigit(ch: number): boolean { 567 return ch >= CharacterCodes._0 && ch <= CharacterCodes._7; 568} 569 570export function couldStartTrivia(text: string, pos: number): boolean { 571 // Keep in sync with skipTrivia 572 const ch = text.charCodeAt(pos); 573 switch (ch) { 574 case CharacterCodes.carriageReturn: 575 case CharacterCodes.lineFeed: 576 case CharacterCodes.tab: 577 case CharacterCodes.verticalTab: 578 case CharacterCodes.formFeed: 579 case CharacterCodes.space: 580 case CharacterCodes.slash: 581 // starts of normal trivia 582 // falls through 583 case CharacterCodes.lessThan: 584 case CharacterCodes.bar: 585 case CharacterCodes.equals: 586 case CharacterCodes.greaterThan: 587 // Starts of conflict marker trivia 588 return true; 589 case CharacterCodes.hash: 590 // Only if its the beginning can we have #! trivia 591 return pos === 0; 592 default: 593 return ch > CharacterCodes.maxAsciiCharacter; 594 } 595} 596 597/** @internal */ 598export function skipTrivia(text: string, pos: number, stopAfterLineBreak?: boolean, stopAtComments?: boolean, inJSDoc?: boolean): number { 599 if (positionIsSynthesized(pos)) { 600 return pos; 601 } 602 603 let canConsumeStar = false; 604 // Keep in sync with couldStartTrivia 605 while (true) { 606 const ch = text.charCodeAt(pos); 607 switch (ch) { 608 case CharacterCodes.carriageReturn: 609 if (text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { 610 pos++; 611 } 612 // falls through 613 case CharacterCodes.lineFeed: 614 pos++; 615 if (stopAfterLineBreak) { 616 return pos; 617 } 618 canConsumeStar = !!inJSDoc; 619 continue; 620 case CharacterCodes.tab: 621 case CharacterCodes.verticalTab: 622 case CharacterCodes.formFeed: 623 case CharacterCodes.space: 624 pos++; 625 continue; 626 case CharacterCodes.slash: 627 if (stopAtComments) { 628 break; 629 } 630 if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { 631 pos += 2; 632 while (pos < text.length) { 633 if (isLineBreak(text.charCodeAt(pos))) { 634 break; 635 } 636 pos++; 637 } 638 canConsumeStar = false; 639 continue; 640 } 641 if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { 642 pos += 2; 643 while (pos < text.length) { 644 if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { 645 pos += 2; 646 break; 647 } 648 pos++; 649 } 650 canConsumeStar = false; 651 continue; 652 } 653 break; 654 655 case CharacterCodes.lessThan: 656 case CharacterCodes.bar: 657 case CharacterCodes.equals: 658 case CharacterCodes.greaterThan: 659 if (isConflictMarkerTrivia(text, pos)) { 660 pos = scanConflictMarkerTrivia(text, pos); 661 canConsumeStar = false; 662 continue; 663 } 664 break; 665 666 case CharacterCodes.hash: 667 if (pos === 0 && isShebangTrivia(text, pos)) { 668 pos = scanShebangTrivia(text, pos); 669 canConsumeStar = false; 670 continue; 671 } 672 break; 673 674 case CharacterCodes.asterisk: 675 if (canConsumeStar) { 676 pos++; 677 canConsumeStar = false; 678 continue; 679 } 680 break; 681 682 default: 683 if (ch > CharacterCodes.maxAsciiCharacter && (isWhiteSpaceLike(ch))) { 684 pos++; 685 continue; 686 } 687 break; 688 } 689 return pos; 690 } 691} 692 693// All conflict markers consist of the same character repeated seven times. If it is 694// a <<<<<<< or >>>>>>> marker then it is also followed by a space. 695const mergeConflictMarkerLength = "<<<<<<<".length; 696 697function isConflictMarkerTrivia(text: string, pos: number) { 698 Debug.assert(pos >= 0); 699 700 // Conflict markers must be at the start of a line. 701 if (pos === 0 || isLineBreak(text.charCodeAt(pos - 1))) { 702 const ch = text.charCodeAt(pos); 703 704 if ((pos + mergeConflictMarkerLength) < text.length) { 705 for (let i = 0; i < mergeConflictMarkerLength; i++) { 706 if (text.charCodeAt(pos + i) !== ch) { 707 return false; 708 } 709 } 710 711 return ch === CharacterCodes.equals || 712 text.charCodeAt(pos + mergeConflictMarkerLength) === CharacterCodes.space; 713 } 714 } 715 716 return false; 717} 718 719function scanConflictMarkerTrivia(text: string, pos: number, error?: (diag: DiagnosticMessage, pos?: number, len?: number) => void) { 720 if (error) { 721 error(Diagnostics.Merge_conflict_marker_encountered, pos, mergeConflictMarkerLength); 722 } 723 724 const ch = text.charCodeAt(pos); 725 const len = text.length; 726 727 if (ch === CharacterCodes.lessThan || ch === CharacterCodes.greaterThan) { 728 while (pos < len && !isLineBreak(text.charCodeAt(pos))) { 729 pos++; 730 } 731 } 732 else { 733 Debug.assert(ch === CharacterCodes.bar || ch === CharacterCodes.equals); 734 // Consume everything from the start of a ||||||| or ======= marker to the start 735 // of the next ======= or >>>>>>> marker. 736 while (pos < len) { 737 const currentChar = text.charCodeAt(pos); 738 if ((currentChar === CharacterCodes.equals || currentChar === CharacterCodes.greaterThan) && currentChar !== ch && isConflictMarkerTrivia(text, pos)) { 739 break; 740 } 741 742 pos++; 743 } 744 } 745 746 return pos; 747} 748 749const shebangTriviaRegex = /^#!.*/; 750 751/** @internal */ 752export function isShebangTrivia(text: string, pos: number) { 753 // Shebangs check must only be done at the start of the file 754 Debug.assert(pos === 0); 755 return shebangTriviaRegex.test(text); 756} 757 758/** @internal */ 759export function scanShebangTrivia(text: string, pos: number) { 760 const shebang = shebangTriviaRegex.exec(text)![0]; 761 pos = pos + shebang.length; 762 return pos; 763} 764 765/** 766 * Invokes a callback for each comment range following the provided position. 767 * 768 * Single-line comment ranges include the leading double-slash characters but not the ending 769 * line break. Multi-line comment ranges include the leading slash-asterisk and trailing 770 * asterisk-slash characters. 771 * 772 * @param reduce If true, accumulates the result of calling the callback in a fashion similar 773 * to reduceLeft. If false, iteration stops when the callback returns a truthy value. 774 * @param text The source text to scan. 775 * @param pos The position at which to start scanning. 776 * @param trailing If false, whitespace is skipped until the first line break and comments 777 * between that location and the next token are returned. If true, comments occurring 778 * between the given position and the next line break are returned. 779 * @param cb The callback to execute as each comment range is encountered. 780 * @param state A state value to pass to each iteration of the callback. 781 * @param initial An initial value to pass when accumulating results (when "reduce" is true). 782 * @returns If "reduce" is true, the accumulated value. If "reduce" is false, the first truthy 783 * return value of the callback. 784 */ 785function iterateCommentRanges<T, U>(reduce: boolean, text: string, pos: number, trailing: boolean, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U | undefined) => U, state: T, initial?: U): U | undefined { 786 let pendingPos!: number; 787 let pendingEnd!: number; 788 let pendingKind!: CommentKind; 789 let pendingHasTrailingNewLine!: boolean; 790 let hasPendingCommentRange = false; 791 let collecting = trailing; 792 let accumulator = initial; 793 if (pos === 0) { 794 collecting = true; 795 const shebang = getShebang(text); 796 if (shebang) { 797 pos = shebang.length; 798 } 799 } 800 scan: while (pos >= 0 && pos < text.length) { 801 const ch = text.charCodeAt(pos); 802 switch (ch) { 803 case CharacterCodes.carriageReturn: 804 if (text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { 805 pos++; 806 } 807 // falls through 808 case CharacterCodes.lineFeed: 809 pos++; 810 if (trailing) { 811 break scan; 812 } 813 814 collecting = true; 815 if (hasPendingCommentRange) { 816 pendingHasTrailingNewLine = true; 817 } 818 819 continue; 820 case CharacterCodes.tab: 821 case CharacterCodes.verticalTab: 822 case CharacterCodes.formFeed: 823 case CharacterCodes.space: 824 pos++; 825 continue; 826 case CharacterCodes.slash: 827 const nextChar = text.charCodeAt(pos + 1); 828 let hasTrailingNewLine = false; 829 if (nextChar === CharacterCodes.slash || nextChar === CharacterCodes.asterisk) { 830 const kind = nextChar === CharacterCodes.slash ? SyntaxKind.SingleLineCommentTrivia : SyntaxKind.MultiLineCommentTrivia; 831 const startPos = pos; 832 pos += 2; 833 if (nextChar === CharacterCodes.slash) { 834 while (pos < text.length) { 835 if (isLineBreak(text.charCodeAt(pos))) { 836 hasTrailingNewLine = true; 837 break; 838 } 839 pos++; 840 } 841 } 842 else { 843 while (pos < text.length) { 844 if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { 845 pos += 2; 846 break; 847 } 848 pos++; 849 } 850 } 851 852 if (collecting) { 853 if (hasPendingCommentRange) { 854 accumulator = cb(pendingPos, pendingEnd, pendingKind, pendingHasTrailingNewLine, state, accumulator); 855 if (!reduce && accumulator) { 856 // If we are not reducing and we have a truthy result, return it. 857 return accumulator; 858 } 859 } 860 861 pendingPos = startPos; 862 pendingEnd = pos; 863 pendingKind = kind; 864 pendingHasTrailingNewLine = hasTrailingNewLine; 865 hasPendingCommentRange = true; 866 } 867 868 continue; 869 } 870 break scan; 871 default: 872 if (ch > CharacterCodes.maxAsciiCharacter && (isWhiteSpaceLike(ch))) { 873 if (hasPendingCommentRange && isLineBreak(ch)) { 874 pendingHasTrailingNewLine = true; 875 } 876 pos++; 877 continue; 878 } 879 break scan; 880 } 881 } 882 883 if (hasPendingCommentRange) { 884 accumulator = cb(pendingPos, pendingEnd, pendingKind, pendingHasTrailingNewLine, state, accumulator); 885 } 886 887 return accumulator; 888} 889 890export function forEachLeadingCommentRange<U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean) => U): U | undefined; 891export function forEachLeadingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state: T): U | undefined; 892export function forEachLeadingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state?: T): U | undefined { 893 return iterateCommentRanges(/*reduce*/ false, text, pos, /*trailing*/ false, cb, state); 894} 895 896export function forEachTrailingCommentRange<U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean) => U): U | undefined; 897export function forEachTrailingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state: T): U | undefined; 898export function forEachTrailingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state?: T): U | undefined { 899 return iterateCommentRanges(/*reduce*/ false, text, pos, /*trailing*/ true, cb, state); 900} 901 902export function reduceEachLeadingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { 903 return iterateCommentRanges(/*reduce*/ true, text, pos, /*trailing*/ false, cb, state, initial); 904} 905 906export function reduceEachTrailingCommentRange<T, U>(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { 907 return iterateCommentRanges(/*reduce*/ true, text, pos, /*trailing*/ true, cb, state, initial); 908} 909 910function appendCommentRange(pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, _state: any, comments: CommentRange[]) { 911 if (!comments) { 912 comments = []; 913 } 914 915 comments.push({ kind, pos, end, hasTrailingNewLine }); 916 return comments; 917} 918 919export function getLeadingCommentRanges(text: string, pos: number): CommentRange[] | undefined { 920 return reduceEachLeadingCommentRange(text, pos, appendCommentRange, /*state*/ undefined, /*initial*/ undefined); 921} 922 923export function getTrailingCommentRanges(text: string, pos: number): CommentRange[] | undefined { 924 return reduceEachTrailingCommentRange(text, pos, appendCommentRange, /*state*/ undefined, /*initial*/ undefined); 925} 926 927/** Optionally, get the shebang */ 928export function getShebang(text: string): string | undefined { 929 const match = shebangTriviaRegex.exec(text); 930 if (match) { 931 return match[0]; 932 } 933} 934 935export function isIdentifierStart(ch: number, languageVersion: ScriptTarget | undefined): boolean { 936 return ch >= CharacterCodes.A && ch <= CharacterCodes.Z || ch >= CharacterCodes.a && ch <= CharacterCodes.z || 937 ch === CharacterCodes.$ || ch === CharacterCodes._ || 938 ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierStart(ch, languageVersion); 939} 940 941export function isIdentifierPart(ch: number, languageVersion: ScriptTarget | undefined, identifierVariant?: LanguageVariant): boolean { 942 return ch >= CharacterCodes.A && ch <= CharacterCodes.Z || ch >= CharacterCodes.a && ch <= CharacterCodes.z || 943 ch >= CharacterCodes._0 && ch <= CharacterCodes._9 || ch === CharacterCodes.$ || ch === CharacterCodes._ || 944 // "-" and ":" are valid in JSX Identifiers 945 (identifierVariant === LanguageVariant.JSX ? (ch === CharacterCodes.minus || ch === CharacterCodes.colon) : false) || 946 ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion); 947} 948 949/** @internal */ 950export function isIdentifierText(name: string, languageVersion: ScriptTarget | undefined, identifierVariant?: LanguageVariant): boolean { 951 let ch = codePointAt(name, 0); 952 if (!isIdentifierStart(ch, languageVersion)) { 953 return false; 954 } 955 956 for (let i = charSize(ch); i < name.length; i += charSize(ch)) { 957 if (!isIdentifierPart(ch = codePointAt(name, i), languageVersion, identifierVariant)) { 958 return false; 959 } 960 } 961 962 return true; 963} 964 965// Creates a scanner over a (possibly unspecified) range of a piece of text. 966export function createScanner(languageVersion: ScriptTarget, 967 skipTrivia: boolean, 968 languageVariant = LanguageVariant.Standard, 969 textInitial?: string, 970 onError?: ErrorCallback, 971 start?: number, 972 length?: number): Scanner { 973 974 /* eslint-disable no-var */ 975 var text = textInitial!; 976 977 // Current position (end position of text of current token) 978 var pos: number; 979 980 981 // end of text 982 var end: number; 983 984 // Start position of whitespace before current token 985 var startPos: number; 986 987 // Start position of text of current token 988 var tokenPos: number; 989 990 var token: SyntaxKind; 991 var tokenValue!: string; 992 var tokenFlags: TokenFlags; 993 994 var commentDirectives: CommentDirective[] | undefined; 995 var inJSDocType = 0; 996 var inEtsContext: boolean = false; 997 998 setText(text, start, length); 999 1000 var scanner: Scanner = { 1001 getStartPos: () => startPos, 1002 getTextPos: () => pos, 1003 getToken: () => token, 1004 getTokenPos: () => tokenPos, 1005 getTokenText: () => text.substring(tokenPos, pos), 1006 getTokenValue: () => tokenValue, 1007 hasUnicodeEscape: () => (tokenFlags & TokenFlags.UnicodeEscape) !== 0, 1008 hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0, 1009 hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0, 1010 hasPrecedingJSDocComment: () => (tokenFlags & TokenFlags.PrecedingJSDocComment) !== 0, 1011 isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord, 1012 isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord, 1013 isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0, 1014 getCommentDirectives: () => commentDirectives, 1015 getNumericLiteralFlags: () => tokenFlags & TokenFlags.NumericLiteralFlags, 1016 getTokenFlags: () => tokenFlags, 1017 reScanGreaterToken, 1018 reScanAsteriskEqualsToken, 1019 reScanSlashToken, 1020 reScanTemplateToken, 1021 reScanTemplateHeadOrNoSubstitutionTemplate, 1022 scanJsxIdentifier, 1023 scanJsxAttributeValue, 1024 reScanJsxAttributeValue, 1025 reScanJsxToken, 1026 reScanLessThanToken, 1027 reScanHashToken, 1028 reScanQuestionToken, 1029 reScanInvalidIdentifier, 1030 scanJsxToken, 1031 scanJsDocToken, 1032 scan, 1033 getText, 1034 clearCommentDirectives, 1035 setText, 1036 setScriptTarget, 1037 setLanguageVariant, 1038 setOnError, 1039 setTextPos, 1040 setInJSDocType, 1041 tryScan, 1042 lookAhead, 1043 scanRange, 1044 setEtsContext 1045 }; 1046 /* eslint-enable no-var */ 1047 1048 if (Debug.isDebugging) { 1049 Object.defineProperty(scanner, "__debugShowCurrentPositionInText", { 1050 get: () => { 1051 const text = scanner.getText(); 1052 return text.slice(0, scanner.getStartPos()) + "║" + text.slice(scanner.getStartPos()); 1053 }, 1054 }); 1055 } 1056 1057 return scanner; 1058 1059 function setEtsContext(isEtsContext: boolean): void { 1060 inEtsContext = isEtsContext; 1061 } 1062 1063 function error(message: DiagnosticMessage): void; 1064 function error(message: DiagnosticMessage, errPos: number, length: number): void; 1065 function error(message: DiagnosticMessage, errPos: number = pos, length?: number): void { 1066 if (onError) { 1067 const oldPos = pos; 1068 pos = errPos; 1069 onError(message, length || 0); 1070 pos = oldPos; 1071 } 1072 } 1073 1074 function scanNumberFragment(): string { 1075 let start = pos; 1076 let allowSeparator = false; 1077 let isPreviousTokenSeparator = false; 1078 let result = ""; 1079 while (true) { 1080 const ch = text.charCodeAt(pos); 1081 if (ch === CharacterCodes._) { 1082 tokenFlags |= TokenFlags.ContainsSeparator; 1083 if (allowSeparator) { 1084 allowSeparator = false; 1085 isPreviousTokenSeparator = true; 1086 result += text.substring(start, pos); 1087 } 1088 else if (isPreviousTokenSeparator) { 1089 error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1); 1090 } 1091 else { 1092 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1); 1093 } 1094 pos++; 1095 start = pos; 1096 continue; 1097 } 1098 if (isDigit(ch)) { 1099 allowSeparator = true; 1100 isPreviousTokenSeparator = false; 1101 pos++; 1102 continue; 1103 } 1104 break; 1105 } 1106 if (text.charCodeAt(pos - 1) === CharacterCodes._) { 1107 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1); 1108 } 1109 return result + text.substring(start, pos); 1110 } 1111 1112 function scanNumber(): { type: SyntaxKind, value: string } { 1113 const start = pos; 1114 const mainFragment = scanNumberFragment(); 1115 let decimalFragment: string | undefined; 1116 let scientificFragment: string | undefined; 1117 if (text.charCodeAt(pos) === CharacterCodes.dot) { 1118 pos++; 1119 decimalFragment = scanNumberFragment(); 1120 } 1121 let end = pos; 1122 if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) { 1123 pos++; 1124 tokenFlags |= TokenFlags.Scientific; 1125 if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++; 1126 const preNumericPart = pos; 1127 const finalFragment = scanNumberFragment(); 1128 if (!finalFragment) { 1129 error(Diagnostics.Digit_expected); 1130 } 1131 else { 1132 scientificFragment = text.substring(end, preNumericPart) + finalFragment; 1133 end = pos; 1134 } 1135 } 1136 let result: string; 1137 if (tokenFlags & TokenFlags.ContainsSeparator) { 1138 result = mainFragment; 1139 if (decimalFragment) { 1140 result += "." + decimalFragment; 1141 } 1142 if (scientificFragment) { 1143 result += scientificFragment; 1144 } 1145 } 1146 else { 1147 result = text.substring(start, end); // No need to use all the fragments; no _ removal needed 1148 } 1149 1150 if (decimalFragment !== undefined || tokenFlags & TokenFlags.Scientific) { 1151 checkForIdentifierStartAfterNumericLiteral(start, decimalFragment === undefined && !!(tokenFlags & TokenFlags.Scientific)); 1152 return { 1153 type: SyntaxKind.NumericLiteral, 1154 value: "" + +result // if value is not an integer, it can be safely coerced to a number 1155 }; 1156 } 1157 else { 1158 tokenValue = result; 1159 const type = checkBigIntSuffix(); // if value is an integer, check whether it is a bigint 1160 checkForIdentifierStartAfterNumericLiteral(start); 1161 return { type, value: tokenValue }; 1162 } 1163 } 1164 1165 function checkForIdentifierStartAfterNumericLiteral(numericStart: number, isScientific?: boolean) { 1166 if (!isIdentifierStart(codePointAt(text, pos), languageVersion)) { 1167 return; 1168 } 1169 1170 const identifierStart = pos; 1171 const { length } = scanIdentifierParts(); 1172 1173 if (length === 1 && text[identifierStart] === "n") { 1174 if (isScientific) { 1175 error(Diagnostics.A_bigint_literal_cannot_use_exponential_notation, numericStart, identifierStart - numericStart + 1); 1176 } 1177 else { 1178 error(Diagnostics.A_bigint_literal_must_be_an_integer, numericStart, identifierStart - numericStart + 1); 1179 } 1180 } 1181 else { 1182 error(Diagnostics.An_identifier_or_keyword_cannot_immediately_follow_a_numeric_literal, identifierStart, length); 1183 pos = identifierStart; 1184 } 1185 } 1186 1187 function scanOctalDigits(): number { 1188 const start = pos; 1189 while (isOctalDigit(text.charCodeAt(pos))) { 1190 pos++; 1191 } 1192 return +(text.substring(start, pos)); 1193 } 1194 1195 /** 1196 * Scans the given number of hexadecimal digits in the text, 1197 * returning -1 if the given number is unavailable. 1198 */ 1199 function scanExactNumberOfHexDigits(count: number, canHaveSeparators: boolean): number { 1200 const valueString = scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ false, canHaveSeparators); 1201 return valueString ? parseInt(valueString, 16) : -1; 1202 } 1203 1204 /** 1205 * Scans as many hexadecimal digits as are available in the text, 1206 * returning "" if the given number of digits was unavailable. 1207 */ 1208 function scanMinimumNumberOfHexDigits(count: number, canHaveSeparators: boolean): string { 1209 return scanHexDigits(/*minCount*/ count, /*scanAsManyAsPossible*/ true, canHaveSeparators); 1210 } 1211 1212 function scanHexDigits(minCount: number, scanAsManyAsPossible: boolean, canHaveSeparators: boolean): string { 1213 let valueChars: number[] = []; 1214 let allowSeparator = false; 1215 let isPreviousTokenSeparator = false; 1216 while (valueChars.length < minCount || scanAsManyAsPossible) { 1217 let ch = text.charCodeAt(pos); 1218 if (canHaveSeparators && ch === CharacterCodes._) { 1219 tokenFlags |= TokenFlags.ContainsSeparator; 1220 if (allowSeparator) { 1221 allowSeparator = false; 1222 isPreviousTokenSeparator = true; 1223 } 1224 else if (isPreviousTokenSeparator) { 1225 error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1); 1226 } 1227 else { 1228 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1); 1229 } 1230 pos++; 1231 continue; 1232 } 1233 allowSeparator = canHaveSeparators; 1234 if (ch >= CharacterCodes.A && ch <= CharacterCodes.F) { 1235 ch += CharacterCodes.a - CharacterCodes.A; // standardize hex literals to lowercase 1236 } 1237 else if (!((ch >= CharacterCodes._0 && ch <= CharacterCodes._9) || 1238 (ch >= CharacterCodes.a && ch <= CharacterCodes.f) 1239 )) { 1240 break; 1241 } 1242 valueChars.push(ch); 1243 pos++; 1244 isPreviousTokenSeparator = false; 1245 } 1246 if (valueChars.length < minCount) { 1247 valueChars = []; 1248 } 1249 if (text.charCodeAt(pos - 1) === CharacterCodes._) { 1250 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1); 1251 } 1252 return String.fromCharCode(...valueChars); 1253 } 1254 1255 function scanString(jsxAttributeString = false): string { 1256 const quote = text.charCodeAt(pos); 1257 pos++; 1258 let result = ""; 1259 let start = pos; 1260 while (true) { 1261 if (pos >= end) { 1262 result += text.substring(start, pos); 1263 tokenFlags |= TokenFlags.Unterminated; 1264 error(Diagnostics.Unterminated_string_literal); 1265 break; 1266 } 1267 const ch = text.charCodeAt(pos); 1268 if (ch === quote) { 1269 result += text.substring(start, pos); 1270 pos++; 1271 break; 1272 } 1273 if (ch === CharacterCodes.backslash && !jsxAttributeString) { 1274 result += text.substring(start, pos); 1275 result += scanEscapeSequence(); 1276 start = pos; 1277 continue; 1278 } 1279 if (isLineBreak(ch) && !jsxAttributeString) { 1280 result += text.substring(start, pos); 1281 tokenFlags |= TokenFlags.Unterminated; 1282 error(Diagnostics.Unterminated_string_literal); 1283 break; 1284 } 1285 pos++; 1286 } 1287 return result; 1288 } 1289 1290 /** 1291 * Sets the current 'tokenValue' and returns a NoSubstitutionTemplateLiteral or 1292 * a literal component of a TemplateExpression. 1293 */ 1294 function scanTemplateAndSetTokenValue(isTaggedTemplate: boolean): SyntaxKind { 1295 const startedWithBacktick = text.charCodeAt(pos) === CharacterCodes.backtick; 1296 1297 pos++; 1298 let start = pos; 1299 let contents = ""; 1300 let resultingToken: SyntaxKind; 1301 1302 while (true) { 1303 if (pos >= end) { 1304 contents += text.substring(start, pos); 1305 tokenFlags |= TokenFlags.Unterminated; 1306 error(Diagnostics.Unterminated_template_literal); 1307 resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail; 1308 break; 1309 } 1310 1311 const currChar = text.charCodeAt(pos); 1312 1313 // '`' 1314 if (currChar === CharacterCodes.backtick) { 1315 contents += text.substring(start, pos); 1316 pos++; 1317 resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail; 1318 break; 1319 } 1320 1321 // '${' 1322 if (currChar === CharacterCodes.$ && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.openBrace) { 1323 contents += text.substring(start, pos); 1324 pos += 2; 1325 resultingToken = startedWithBacktick ? SyntaxKind.TemplateHead : SyntaxKind.TemplateMiddle; 1326 break; 1327 } 1328 1329 // Escape character 1330 if (currChar === CharacterCodes.backslash) { 1331 contents += text.substring(start, pos); 1332 contents += scanEscapeSequence(isTaggedTemplate); 1333 start = pos; 1334 continue; 1335 } 1336 1337 // Speculated ECMAScript 6 Spec 11.8.6.1: 1338 // <CR><LF> and <CR> LineTerminatorSequences are normalized to <LF> for Template Values 1339 if (currChar === CharacterCodes.carriageReturn) { 1340 contents += text.substring(start, pos); 1341 pos++; 1342 1343 if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) { 1344 pos++; 1345 } 1346 1347 contents += "\n"; 1348 start = pos; 1349 continue; 1350 } 1351 1352 pos++; 1353 } 1354 1355 Debug.assert(resultingToken !== undefined); 1356 1357 tokenValue = contents; 1358 return resultingToken; 1359 } 1360 1361 function scanEscapeSequence(isTaggedTemplate?: boolean): string { 1362 const start = pos; 1363 pos++; 1364 if (pos >= end) { 1365 error(Diagnostics.Unexpected_end_of_text); 1366 return ""; 1367 } 1368 const ch = text.charCodeAt(pos); 1369 pos++; 1370 switch (ch) { 1371 case CharacterCodes._0: 1372 // '\01' 1373 if (isTaggedTemplate && pos < end && isDigit(text.charCodeAt(pos))) { 1374 pos++; 1375 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1376 return text.substring(start, pos); 1377 } 1378 return "\0"; 1379 case CharacterCodes.b: 1380 return "\b"; 1381 case CharacterCodes.t: 1382 return "\t"; 1383 case CharacterCodes.n: 1384 return "\n"; 1385 case CharacterCodes.v: 1386 return "\v"; 1387 case CharacterCodes.f: 1388 return "\f"; 1389 case CharacterCodes.r: 1390 return "\r"; 1391 case CharacterCodes.singleQuote: 1392 return "\'"; 1393 case CharacterCodes.doubleQuote: 1394 return "\""; 1395 case CharacterCodes.u: 1396 if (isTaggedTemplate) { 1397 // '\u' or '\u0' or '\u00' or '\u000' 1398 for (let escapePos = pos; escapePos < pos + 4; escapePos++) { 1399 if (escapePos < end && !isHexDigit(text.charCodeAt(escapePos)) && text.charCodeAt(escapePos) !== CharacterCodes.openBrace) { 1400 pos = escapePos; 1401 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1402 return text.substring(start, pos); 1403 } 1404 } 1405 } 1406 // '\u{DDDDDDDD}' 1407 if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) { 1408 pos++; 1409 1410 // '\u{' 1411 if (isTaggedTemplate && !isHexDigit(text.charCodeAt(pos))) { 1412 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1413 return text.substring(start, pos); 1414 } 1415 1416 if (isTaggedTemplate) { 1417 const savePos = pos; 1418 const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); 1419 const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1; 1420 1421 // '\u{Not Code Point' or '\u{CodePoint' 1422 if (!isCodePoint(escapedValue) || text.charCodeAt(pos) !== CharacterCodes.closeBrace) { 1423 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1424 return text.substring(start, pos); 1425 } 1426 else { 1427 pos = savePos; 1428 } 1429 } 1430 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 1431 return scanExtendedUnicodeEscape(); 1432 } 1433 1434 tokenFlags |= TokenFlags.UnicodeEscape; 1435 // '\uDDDD' 1436 return scanHexadecimalEscape(/*numDigits*/ 4); 1437 1438 case CharacterCodes.x: 1439 if (isTaggedTemplate) { 1440 if (!isHexDigit(text.charCodeAt(pos))) { 1441 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1442 return text.substring(start, pos); 1443 } 1444 else if (!isHexDigit(text.charCodeAt(pos + 1))) { 1445 pos++; 1446 tokenFlags |= TokenFlags.ContainsInvalidEscape; 1447 return text.substring(start, pos); 1448 } 1449 } 1450 // '\xDD' 1451 return scanHexadecimalEscape(/*numDigits*/ 2); 1452 1453 // when encountering a LineContinuation (i.e. a backslash and a line terminator sequence), 1454 // the line terminator is interpreted to be "the empty code unit sequence". 1455 case CharacterCodes.carriageReturn: 1456 if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) { 1457 pos++; 1458 } 1459 // falls through 1460 case CharacterCodes.lineFeed: 1461 case CharacterCodes.lineSeparator: 1462 case CharacterCodes.paragraphSeparator: 1463 return ""; 1464 default: 1465 return String.fromCharCode(ch); 1466 } 1467 } 1468 1469 function scanHexadecimalEscape(numDigits: number): string { 1470 const escapedValue = scanExactNumberOfHexDigits(numDigits, /*canHaveSeparators*/ false); 1471 1472 if (escapedValue >= 0) { 1473 return String.fromCharCode(escapedValue); 1474 } 1475 else { 1476 error(Diagnostics.Hexadecimal_digit_expected); 1477 return ""; 1478 } 1479 } 1480 1481 function scanExtendedUnicodeEscape(): string { 1482 const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); 1483 const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1; 1484 let isInvalidExtendedEscape = false; 1485 1486 // Validate the value of the digit 1487 if (escapedValue < 0) { 1488 error(Diagnostics.Hexadecimal_digit_expected); 1489 isInvalidExtendedEscape = true; 1490 } 1491 else if (escapedValue > 0x10FFFF) { 1492 error(Diagnostics.An_extended_Unicode_escape_value_must_be_between_0x0_and_0x10FFFF_inclusive); 1493 isInvalidExtendedEscape = true; 1494 } 1495 1496 if (pos >= end) { 1497 error(Diagnostics.Unexpected_end_of_text); 1498 isInvalidExtendedEscape = true; 1499 } 1500 else if (text.charCodeAt(pos) === CharacterCodes.closeBrace) { 1501 // Only swallow the following character up if it's a '}'. 1502 pos++; 1503 } 1504 else { 1505 error(Diagnostics.Unterminated_Unicode_escape_sequence); 1506 isInvalidExtendedEscape = true; 1507 } 1508 1509 if (isInvalidExtendedEscape) { 1510 return ""; 1511 } 1512 1513 return utf16EncodeAsString(escapedValue); 1514 } 1515 1516 // Current character is known to be a backslash. Check for Unicode escape of the form '\uXXXX' 1517 // and return code point value if valid Unicode escape is found. Otherwise return -1. 1518 function peekUnicodeEscape(): number { 1519 if (pos + 5 < end && text.charCodeAt(pos + 1) === CharacterCodes.u) { 1520 const start = pos; 1521 pos += 2; 1522 const value = scanExactNumberOfHexDigits(4, /*canHaveSeparators*/ false); 1523 pos = start; 1524 return value; 1525 } 1526 return -1; 1527 } 1528 1529 1530 function peekExtendedUnicodeEscape(): number { 1531 if (codePointAt(text, pos + 1) === CharacterCodes.u && codePointAt(text, pos + 2) === CharacterCodes.openBrace) { 1532 const start = pos; 1533 pos += 3; 1534 const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false); 1535 const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1; 1536 pos = start; 1537 return escapedValue; 1538 } 1539 return -1; 1540 } 1541 1542 function scanIdentifierParts(): string { 1543 let result = ""; 1544 let start = pos; 1545 while (pos < end) { 1546 let ch = codePointAt(text, pos); 1547 if (isIdentifierPart(ch, languageVersion)) { 1548 pos += charSize(ch); 1549 } 1550 else if (ch === CharacterCodes.backslash) { 1551 ch = peekExtendedUnicodeEscape(); 1552 if (ch >= 0 && isIdentifierPart(ch, languageVersion)) { 1553 pos += 3; 1554 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 1555 result += scanExtendedUnicodeEscape(); 1556 start = pos; 1557 continue; 1558 } 1559 ch = peekUnicodeEscape(); 1560 if (!(ch >= 0 && isIdentifierPart(ch, languageVersion))) { 1561 break; 1562 } 1563 tokenFlags |= TokenFlags.UnicodeEscape; 1564 result += text.substring(start, pos); 1565 result += utf16EncodeAsString(ch); 1566 // Valid Unicode escape is always six characters 1567 pos += 6; 1568 start = pos; 1569 } 1570 else { 1571 break; 1572 } 1573 } 1574 result += text.substring(start, pos); 1575 return result; 1576 } 1577 1578 function getIdentifierToken(): SyntaxKind.Identifier | KeywordSyntaxKind { 1579 // Reserved words are between 2 and 12 characters long and start with a lowercase letter 1580 const len = tokenValue.length; 1581 if (len >= 2 && len <= 12) { 1582 const ch = tokenValue.charCodeAt(0); 1583 if (ch >= CharacterCodes.a && ch <= CharacterCodes.z) { 1584 const keyword = textToKeyword.get(tokenValue); 1585 if (keyword !== undefined) { 1586 token = keyword; 1587 if (keyword === SyntaxKind.StructKeyword && !inEtsContext) { 1588 token = SyntaxKind.Identifier; 1589 } 1590 return token; 1591 } 1592 } 1593 } 1594 return token = SyntaxKind.Identifier; 1595 } 1596 1597 function scanBinaryOrOctalDigits(base: 2 | 8): string { 1598 let value = ""; 1599 // For counting number of digits; Valid binaryIntegerLiteral must have at least one binary digit following B or b. 1600 // Similarly valid octalIntegerLiteral must have at least one octal digit following o or O. 1601 let separatorAllowed = false; 1602 let isPreviousTokenSeparator = false; 1603 while (true) { 1604 const ch = text.charCodeAt(pos); 1605 // Numeric separators are allowed anywhere within a numeric literal, except not at the beginning, or following another separator 1606 if (ch === CharacterCodes._) { 1607 tokenFlags |= TokenFlags.ContainsSeparator; 1608 if (separatorAllowed) { 1609 separatorAllowed = false; 1610 isPreviousTokenSeparator = true; 1611 } 1612 else if (isPreviousTokenSeparator) { 1613 error(Diagnostics.Multiple_consecutive_numeric_separators_are_not_permitted, pos, 1); 1614 } 1615 else { 1616 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos, 1); 1617 } 1618 pos++; 1619 continue; 1620 } 1621 separatorAllowed = true; 1622 if (!isDigit(ch) || ch - CharacterCodes._0 >= base) { 1623 break; 1624 } 1625 value += text[pos]; 1626 pos++; 1627 isPreviousTokenSeparator = false; 1628 } 1629 if (text.charCodeAt(pos - 1) === CharacterCodes._) { 1630 // Literal ends with underscore - not allowed 1631 error(Diagnostics.Numeric_separators_are_not_allowed_here, pos - 1, 1); 1632 } 1633 return value; 1634 } 1635 1636 function checkBigIntSuffix(): SyntaxKind { 1637 if (text.charCodeAt(pos) === CharacterCodes.n) { 1638 tokenValue += "n"; 1639 // Use base 10 instead of base 2 or base 8 for shorter literals 1640 if (tokenFlags & TokenFlags.BinaryOrOctalSpecifier) { 1641 tokenValue = parsePseudoBigInt(tokenValue) + "n"; 1642 } 1643 pos++; 1644 return SyntaxKind.BigIntLiteral; 1645 } 1646 else { // not a bigint, so can convert to number in simplified form 1647 // Number() may not support 0b or 0o, so use parseInt() instead 1648 const numericValue = tokenFlags & TokenFlags.BinarySpecifier 1649 ? parseInt(tokenValue.slice(2), 2) // skip "0b" 1650 : tokenFlags & TokenFlags.OctalSpecifier 1651 ? parseInt(tokenValue.slice(2), 8) // skip "0o" 1652 : +tokenValue; 1653 tokenValue = "" + numericValue; 1654 return SyntaxKind.NumericLiteral; 1655 } 1656 } 1657 1658 function scan(): SyntaxKind { 1659 startPos = pos; 1660 tokenFlags = TokenFlags.None; 1661 let asteriskSeen = false; 1662 while (true) { 1663 tokenPos = pos; 1664 if (pos >= end) { 1665 return token = SyntaxKind.EndOfFileToken; 1666 } 1667 const ch = codePointAt(text, pos); 1668 1669 // Special handling for shebang 1670 if (ch === CharacterCodes.hash && pos === 0 && isShebangTrivia(text, pos)) { 1671 pos = scanShebangTrivia(text, pos); 1672 if (skipTrivia) { 1673 continue; 1674 } 1675 else { 1676 return token = SyntaxKind.ShebangTrivia; 1677 } 1678 } 1679 1680 switch (ch) { 1681 case CharacterCodes.lineFeed: 1682 case CharacterCodes.carriageReturn: 1683 tokenFlags |= TokenFlags.PrecedingLineBreak; 1684 if (skipTrivia) { 1685 pos++; 1686 continue; 1687 } 1688 else { 1689 if (ch === CharacterCodes.carriageReturn && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { 1690 // consume both CR and LF 1691 pos += 2; 1692 } 1693 else { 1694 pos++; 1695 } 1696 return token = SyntaxKind.NewLineTrivia; 1697 } 1698 case CharacterCodes.tab: 1699 case CharacterCodes.verticalTab: 1700 case CharacterCodes.formFeed: 1701 case CharacterCodes.space: 1702 case CharacterCodes.nonBreakingSpace: 1703 case CharacterCodes.ogham: 1704 case CharacterCodes.enQuad: 1705 case CharacterCodes.emQuad: 1706 case CharacterCodes.enSpace: 1707 case CharacterCodes.emSpace: 1708 case CharacterCodes.threePerEmSpace: 1709 case CharacterCodes.fourPerEmSpace: 1710 case CharacterCodes.sixPerEmSpace: 1711 case CharacterCodes.figureSpace: 1712 case CharacterCodes.punctuationSpace: 1713 case CharacterCodes.thinSpace: 1714 case CharacterCodes.hairSpace: 1715 case CharacterCodes.zeroWidthSpace: 1716 case CharacterCodes.narrowNoBreakSpace: 1717 case CharacterCodes.mathematicalSpace: 1718 case CharacterCodes.ideographicSpace: 1719 case CharacterCodes.byteOrderMark: 1720 if (skipTrivia) { 1721 pos++; 1722 continue; 1723 } 1724 else { 1725 while (pos < end && isWhiteSpaceSingleLine(text.charCodeAt(pos))) { 1726 pos++; 1727 } 1728 return token = SyntaxKind.WhitespaceTrivia; 1729 } 1730 case CharacterCodes.exclamation: 1731 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1732 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1733 return pos += 3, token = SyntaxKind.ExclamationEqualsEqualsToken; 1734 } 1735 return pos += 2, token = SyntaxKind.ExclamationEqualsToken; 1736 } 1737 pos++; 1738 return token = SyntaxKind.ExclamationToken; 1739 case CharacterCodes.doubleQuote: 1740 case CharacterCodes.singleQuote: 1741 tokenValue = scanString(); 1742 return token = SyntaxKind.StringLiteral; 1743 case CharacterCodes.backtick: 1744 return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ false); 1745 case CharacterCodes.percent: 1746 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1747 return pos += 2, token = SyntaxKind.PercentEqualsToken; 1748 } 1749 pos++; 1750 return token = SyntaxKind.PercentToken; 1751 case CharacterCodes.ampersand: 1752 if (text.charCodeAt(pos + 1) === CharacterCodes.ampersand) { 1753 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1754 return pos += 3, token = SyntaxKind.AmpersandAmpersandEqualsToken; 1755 } 1756 return pos += 2, token = SyntaxKind.AmpersandAmpersandToken; 1757 } 1758 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1759 return pos += 2, token = SyntaxKind.AmpersandEqualsToken; 1760 } 1761 pos++; 1762 return token = SyntaxKind.AmpersandToken; 1763 case CharacterCodes.openParen: 1764 pos++; 1765 return token = SyntaxKind.OpenParenToken; 1766 case CharacterCodes.closeParen: 1767 pos++; 1768 return token = SyntaxKind.CloseParenToken; 1769 case CharacterCodes.asterisk: 1770 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1771 return pos += 2, token = SyntaxKind.AsteriskEqualsToken; 1772 } 1773 if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { 1774 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1775 return pos += 3, token = SyntaxKind.AsteriskAsteriskEqualsToken; 1776 } 1777 return pos += 2, token = SyntaxKind.AsteriskAsteriskToken; 1778 } 1779 pos++; 1780 if (inJSDocType && !asteriskSeen && (tokenFlags & TokenFlags.PrecedingLineBreak)) { 1781 // decoration at the start of a JSDoc comment line 1782 asteriskSeen = true; 1783 continue; 1784 } 1785 return token = SyntaxKind.AsteriskToken; 1786 case CharacterCodes.plus: 1787 if (text.charCodeAt(pos + 1) === CharacterCodes.plus) { 1788 return pos += 2, token = SyntaxKind.PlusPlusToken; 1789 } 1790 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1791 return pos += 2, token = SyntaxKind.PlusEqualsToken; 1792 } 1793 pos++; 1794 return token = SyntaxKind.PlusToken; 1795 case CharacterCodes.comma: 1796 pos++; 1797 return token = SyntaxKind.CommaToken; 1798 case CharacterCodes.minus: 1799 if (text.charCodeAt(pos + 1) === CharacterCodes.minus) { 1800 return pos += 2, token = SyntaxKind.MinusMinusToken; 1801 } 1802 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1803 return pos += 2, token = SyntaxKind.MinusEqualsToken; 1804 } 1805 pos++; 1806 return token = SyntaxKind.MinusToken; 1807 case CharacterCodes.dot: 1808 if (isDigit(text.charCodeAt(pos + 1))) { 1809 tokenValue = scanNumber().value; 1810 return token = SyntaxKind.NumericLiteral; 1811 } 1812 if (text.charCodeAt(pos + 1) === CharacterCodes.dot && text.charCodeAt(pos + 2) === CharacterCodes.dot) { 1813 return pos += 3, token = SyntaxKind.DotDotDotToken; 1814 } 1815 pos++; 1816 return token = SyntaxKind.DotToken; 1817 case CharacterCodes.slash: 1818 // Single-line comment 1819 if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { 1820 pos += 2; 1821 1822 while (pos < end) { 1823 if (isLineBreak(text.charCodeAt(pos))) { 1824 break; 1825 } 1826 pos++; 1827 } 1828 1829 commentDirectives = appendIfCommentDirective( 1830 commentDirectives, 1831 text.slice(tokenPos, pos), 1832 commentDirectiveRegExSingleLine, 1833 tokenPos, 1834 ); 1835 1836 if (skipTrivia) { 1837 continue; 1838 } 1839 else { 1840 return token = SyntaxKind.SingleLineCommentTrivia; 1841 } 1842 } 1843 // Multi-line comment 1844 if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { 1845 pos += 2; 1846 if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) !== CharacterCodes.slash) { 1847 tokenFlags |= TokenFlags.PrecedingJSDocComment; 1848 } 1849 1850 let commentClosed = false; 1851 let lastLineStart = tokenPos; 1852 while (pos < end) { 1853 const ch = text.charCodeAt(pos); 1854 1855 if (ch === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { 1856 pos += 2; 1857 commentClosed = true; 1858 break; 1859 } 1860 1861 pos++; 1862 1863 if (isLineBreak(ch)) { 1864 lastLineStart = pos; 1865 tokenFlags |= TokenFlags.PrecedingLineBreak; 1866 } 1867 } 1868 1869 commentDirectives = appendIfCommentDirective(commentDirectives, text.slice(lastLineStart, pos), commentDirectiveRegExMultiLine, lastLineStart); 1870 1871 if (!commentClosed) { 1872 error(Diagnostics.Asterisk_Slash_expected); 1873 } 1874 1875 if (skipTrivia) { 1876 continue; 1877 } 1878 else { 1879 if (!commentClosed) { 1880 tokenFlags |= TokenFlags.Unterminated; 1881 } 1882 return token = SyntaxKind.MultiLineCommentTrivia; 1883 } 1884 } 1885 1886 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1887 return pos += 2, token = SyntaxKind.SlashEqualsToken; 1888 } 1889 1890 pos++; 1891 return token = SyntaxKind.SlashToken; 1892 1893 case CharacterCodes._0: 1894 if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) { 1895 pos += 2; 1896 tokenValue = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ true); 1897 if (!tokenValue) { 1898 error(Diagnostics.Hexadecimal_digit_expected); 1899 tokenValue = "0"; 1900 } 1901 tokenValue = "0x" + tokenValue; 1902 tokenFlags |= TokenFlags.HexSpecifier; 1903 return token = checkBigIntSuffix(); 1904 } 1905 else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) { 1906 pos += 2; 1907 tokenValue = scanBinaryOrOctalDigits(/* base */ 2); 1908 if (!tokenValue) { 1909 error(Diagnostics.Binary_digit_expected); 1910 tokenValue = "0"; 1911 } 1912 tokenValue = "0b" + tokenValue; 1913 tokenFlags |= TokenFlags.BinarySpecifier; 1914 return token = checkBigIntSuffix(); 1915 } 1916 else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) { 1917 pos += 2; 1918 tokenValue = scanBinaryOrOctalDigits(/* base */ 8); 1919 if (!tokenValue) { 1920 error(Diagnostics.Octal_digit_expected); 1921 tokenValue = "0"; 1922 } 1923 tokenValue = "0o" + tokenValue; 1924 tokenFlags |= TokenFlags.OctalSpecifier; 1925 return token = checkBigIntSuffix(); 1926 } 1927 // Try to parse as an octal 1928 if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) { 1929 tokenValue = "" + scanOctalDigits(); 1930 tokenFlags |= TokenFlags.Octal; 1931 return token = SyntaxKind.NumericLiteral; 1932 } 1933 // This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero 1934 // can only be followed by an octal digit, a dot, or the end of the number literal. However, we are being 1935 // permissive and allowing decimal digits of the form 08* and 09* (which many browsers also do). 1936 // falls through 1937 case CharacterCodes._1: 1938 case CharacterCodes._2: 1939 case CharacterCodes._3: 1940 case CharacterCodes._4: 1941 case CharacterCodes._5: 1942 case CharacterCodes._6: 1943 case CharacterCodes._7: 1944 case CharacterCodes._8: 1945 case CharacterCodes._9: 1946 ({ type: token, value: tokenValue } = scanNumber()); 1947 return token; 1948 case CharacterCodes.colon: 1949 pos++; 1950 return token = SyntaxKind.ColonToken; 1951 case CharacterCodes.semicolon: 1952 pos++; 1953 return token = SyntaxKind.SemicolonToken; 1954 case CharacterCodes.lessThan: 1955 if (isConflictMarkerTrivia(text, pos)) { 1956 pos = scanConflictMarkerTrivia(text, pos, error); 1957 if (skipTrivia) { 1958 continue; 1959 } 1960 else { 1961 return token = SyntaxKind.ConflictMarkerTrivia; 1962 } 1963 } 1964 1965 if (text.charCodeAt(pos + 1) === CharacterCodes.lessThan) { 1966 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1967 return pos += 3, token = SyntaxKind.LessThanLessThanEqualsToken; 1968 } 1969 return pos += 2, token = SyntaxKind.LessThanLessThanToken; 1970 } 1971 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1972 return pos += 2, token = SyntaxKind.LessThanEqualsToken; 1973 } 1974 if (languageVariant === LanguageVariant.JSX && 1975 text.charCodeAt(pos + 1) === CharacterCodes.slash && 1976 text.charCodeAt(pos + 2) !== CharacterCodes.asterisk) { 1977 return pos += 2, token = SyntaxKind.LessThanSlashToken; 1978 } 1979 pos++; 1980 return token = SyntaxKind.LessThanToken; 1981 case CharacterCodes.equals: 1982 if (isConflictMarkerTrivia(text, pos)) { 1983 pos = scanConflictMarkerTrivia(text, pos, error); 1984 if (skipTrivia) { 1985 continue; 1986 } 1987 else { 1988 return token = SyntaxKind.ConflictMarkerTrivia; 1989 } 1990 } 1991 1992 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 1993 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 1994 return pos += 3, token = SyntaxKind.EqualsEqualsEqualsToken; 1995 } 1996 return pos += 2, token = SyntaxKind.EqualsEqualsToken; 1997 } 1998 if (text.charCodeAt(pos + 1) === CharacterCodes.greaterThan) { 1999 return pos += 2, token = SyntaxKind.EqualsGreaterThanToken; 2000 } 2001 pos++; 2002 return token = SyntaxKind.EqualsToken; 2003 case CharacterCodes.greaterThan: 2004 if (isConflictMarkerTrivia(text, pos)) { 2005 pos = scanConflictMarkerTrivia(text, pos, error); 2006 if (skipTrivia) { 2007 continue; 2008 } 2009 else { 2010 return token = SyntaxKind.ConflictMarkerTrivia; 2011 } 2012 } 2013 2014 pos++; 2015 return token = SyntaxKind.GreaterThanToken; 2016 case CharacterCodes.question: 2017 if (text.charCodeAt(pos + 1) === CharacterCodes.dot && !isDigit(text.charCodeAt(pos + 2))) { 2018 return pos += 2, token = SyntaxKind.QuestionDotToken; 2019 } 2020 if (text.charCodeAt(pos + 1) === CharacterCodes.question) { 2021 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 2022 return pos += 3, token = SyntaxKind.QuestionQuestionEqualsToken; 2023 } 2024 return pos += 2, token = SyntaxKind.QuestionQuestionToken; 2025 } 2026 pos++; 2027 return token = SyntaxKind.QuestionToken; 2028 case CharacterCodes.openBracket: 2029 pos++; 2030 return token = SyntaxKind.OpenBracketToken; 2031 case CharacterCodes.closeBracket: 2032 pos++; 2033 return token = SyntaxKind.CloseBracketToken; 2034 case CharacterCodes.caret: 2035 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 2036 return pos += 2, token = SyntaxKind.CaretEqualsToken; 2037 } 2038 pos++; 2039 return token = SyntaxKind.CaretToken; 2040 case CharacterCodes.openBrace: 2041 pos++; 2042 return token = SyntaxKind.OpenBraceToken; 2043 case CharacterCodes.bar: 2044 if (isConflictMarkerTrivia(text, pos)) { 2045 pos = scanConflictMarkerTrivia(text, pos, error); 2046 if (skipTrivia) { 2047 continue; 2048 } 2049 else { 2050 return token = SyntaxKind.ConflictMarkerTrivia; 2051 } 2052 } 2053 2054 if (text.charCodeAt(pos + 1) === CharacterCodes.bar) { 2055 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 2056 return pos += 3, token = SyntaxKind.BarBarEqualsToken; 2057 } 2058 return pos += 2, token = SyntaxKind.BarBarToken; 2059 } 2060 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 2061 return pos += 2, token = SyntaxKind.BarEqualsToken; 2062 } 2063 pos++; 2064 return token = SyntaxKind.BarToken; 2065 case CharacterCodes.closeBrace: 2066 pos++; 2067 return token = SyntaxKind.CloseBraceToken; 2068 case CharacterCodes.tilde: 2069 pos++; 2070 return token = SyntaxKind.TildeToken; 2071 case CharacterCodes.at: 2072 pos++; 2073 return token = SyntaxKind.AtToken; 2074 case CharacterCodes.backslash: 2075 const extendedCookedChar = peekExtendedUnicodeEscape(); 2076 if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) { 2077 pos += 3; 2078 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 2079 tokenValue = scanExtendedUnicodeEscape() + scanIdentifierParts(); 2080 return token = getIdentifierToken(); 2081 } 2082 2083 const cookedChar = peekUnicodeEscape(); 2084 if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) { 2085 pos += 6; 2086 tokenFlags |= TokenFlags.UnicodeEscape; 2087 tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts(); 2088 return token = getIdentifierToken(); 2089 } 2090 2091 error(Diagnostics.Invalid_character); 2092 pos++; 2093 return token = SyntaxKind.Unknown; 2094 case CharacterCodes.hash: 2095 if (pos !== 0 && text[pos + 1] === "!") { 2096 error(Diagnostics.can_only_be_used_at_the_start_of_a_file); 2097 pos++; 2098 return token = SyntaxKind.Unknown; 2099 } 2100 2101 const charAfterHash = codePointAt(text, pos + 1); 2102 if (charAfterHash === CharacterCodes.backslash) { 2103 pos++; 2104 const extendedCookedChar = peekExtendedUnicodeEscape(); 2105 if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) { 2106 pos += 3; 2107 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 2108 tokenValue = "#" + scanExtendedUnicodeEscape() + scanIdentifierParts(); 2109 return token = SyntaxKind.PrivateIdentifier; 2110 } 2111 2112 const cookedChar = peekUnicodeEscape(); 2113 if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) { 2114 pos += 6; 2115 tokenFlags |= TokenFlags.UnicodeEscape; 2116 tokenValue = "#" + String.fromCharCode(cookedChar) + scanIdentifierParts(); 2117 return token = SyntaxKind.PrivateIdentifier; 2118 } 2119 pos--; 2120 } 2121 2122 if (isIdentifierStart(charAfterHash, languageVersion)) { 2123 pos++; 2124 // We're relying on scanIdentifier's behavior and adjusting the token kind after the fact. 2125 // Notably absent from this block is the fact that calling a function named "scanIdentifier", 2126 // but identifiers don't include '#', and that function doesn't deal with it at all. 2127 // This works because 'scanIdentifier' tries to reuse source characters and builds up substrings; 2128 // however, it starts at the 'tokenPos' which includes the '#', and will "accidentally" prepend the '#' for us. 2129 scanIdentifier(charAfterHash, languageVersion); 2130 } 2131 else { 2132 tokenValue = "#"; 2133 error(Diagnostics.Invalid_character, pos++, charSize(ch)); 2134 } 2135 return token = SyntaxKind.PrivateIdentifier; 2136 default: 2137 const identifierKind = scanIdentifier(ch, languageVersion); 2138 if (identifierKind) { 2139 return token = identifierKind; 2140 } 2141 else if (isWhiteSpaceSingleLine(ch)) { 2142 pos += charSize(ch); 2143 continue; 2144 } 2145 else if (isLineBreak(ch)) { 2146 tokenFlags |= TokenFlags.PrecedingLineBreak; 2147 pos += charSize(ch); 2148 continue; 2149 } 2150 const size = charSize(ch); 2151 error(Diagnostics.Invalid_character, pos, size); 2152 pos += size; 2153 return token = SyntaxKind.Unknown; 2154 } 2155 } 2156 } 2157 2158 function reScanInvalidIdentifier(): SyntaxKind { 2159 Debug.assert(token === SyntaxKind.Unknown, "'reScanInvalidIdentifier' should only be called when the current token is 'SyntaxKind.Unknown'."); 2160 pos = tokenPos = startPos; 2161 tokenFlags = 0; 2162 const ch = codePointAt(text, pos); 2163 const identifierKind = scanIdentifier(ch, ScriptTarget.ESNext); 2164 if (identifierKind) { 2165 return token = identifierKind; 2166 } 2167 pos += charSize(ch); 2168 return token; // Still `SyntaKind.Unknown` 2169 } 2170 2171 function scanIdentifier(startCharacter: number, languageVersion: ScriptTarget) { 2172 let ch = startCharacter; 2173 if (isIdentifierStart(ch, languageVersion)) { 2174 pos += charSize(ch); 2175 while (pos < end && isIdentifierPart(ch = codePointAt(text, pos), languageVersion)) pos += charSize(ch); 2176 tokenValue = text.substring(tokenPos, pos); 2177 if (ch === CharacterCodes.backslash) { 2178 tokenValue += scanIdentifierParts(); 2179 } 2180 return getIdentifierToken(); 2181 } 2182 } 2183 2184 function reScanGreaterToken(): SyntaxKind { 2185 if (token === SyntaxKind.GreaterThanToken) { 2186 if (text.charCodeAt(pos) === CharacterCodes.greaterThan) { 2187 if (text.charCodeAt(pos + 1) === CharacterCodes.greaterThan) { 2188 if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { 2189 return pos += 3, token = SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken; 2190 } 2191 return pos += 2, token = SyntaxKind.GreaterThanGreaterThanGreaterThanToken; 2192 } 2193 if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { 2194 return pos += 2, token = SyntaxKind.GreaterThanGreaterThanEqualsToken; 2195 } 2196 pos++; 2197 return token = SyntaxKind.GreaterThanGreaterThanToken; 2198 } 2199 if (text.charCodeAt(pos) === CharacterCodes.equals) { 2200 pos++; 2201 return token = SyntaxKind.GreaterThanEqualsToken; 2202 } 2203 } 2204 return token; 2205 } 2206 2207 function reScanAsteriskEqualsToken(): SyntaxKind { 2208 Debug.assert(token === SyntaxKind.AsteriskEqualsToken, "'reScanAsteriskEqualsToken' should only be called on a '*='"); 2209 pos = tokenPos + 1; 2210 return token = SyntaxKind.EqualsToken; 2211 } 2212 2213 function reScanSlashToken(): SyntaxKind { 2214 if (token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) { 2215 let p = tokenPos + 1; 2216 let inEscape = false; 2217 let inCharacterClass = false; 2218 while (true) { 2219 // If we reach the end of a file, or hit a newline, then this is an unterminated 2220 // regex. Report error and return what we have so far. 2221 if (p >= end) { 2222 tokenFlags |= TokenFlags.Unterminated; 2223 error(Diagnostics.Unterminated_regular_expression_literal); 2224 break; 2225 } 2226 2227 const ch = text.charCodeAt(p); 2228 if (isLineBreak(ch)) { 2229 tokenFlags |= TokenFlags.Unterminated; 2230 error(Diagnostics.Unterminated_regular_expression_literal); 2231 break; 2232 } 2233 2234 if (inEscape) { 2235 // Parsing an escape character; 2236 // reset the flag and just advance to the next char. 2237 inEscape = false; 2238 } 2239 else if (ch === CharacterCodes.slash && !inCharacterClass) { 2240 // A slash within a character class is permissible, 2241 // but in general it signals the end of the regexp literal. 2242 p++; 2243 break; 2244 } 2245 else if (ch === CharacterCodes.openBracket) { 2246 inCharacterClass = true; 2247 } 2248 else if (ch === CharacterCodes.backslash) { 2249 inEscape = true; 2250 } 2251 else if (ch === CharacterCodes.closeBracket) { 2252 inCharacterClass = false; 2253 } 2254 p++; 2255 } 2256 2257 while (p < end && isIdentifierPart(text.charCodeAt(p), languageVersion)) { 2258 p++; 2259 } 2260 pos = p; 2261 tokenValue = text.substring(tokenPos, pos); 2262 token = SyntaxKind.RegularExpressionLiteral; 2263 } 2264 return token; 2265 } 2266 2267 function appendIfCommentDirective( 2268 commentDirectives: CommentDirective[] | undefined, 2269 text: string, 2270 commentDirectiveRegEx: RegExp, 2271 lineStart: number, 2272 ) { 2273 const type = getDirectiveFromComment(trimStringStart(text), commentDirectiveRegEx); 2274 if (type === undefined) { 2275 return commentDirectives; 2276 } 2277 2278 return append( 2279 commentDirectives, 2280 { 2281 range: { pos: lineStart, end: pos }, 2282 type, 2283 }, 2284 ); 2285 } 2286 2287 function getDirectiveFromComment(text: string, commentDirectiveRegEx: RegExp) { 2288 const match = commentDirectiveRegEx.exec(text); 2289 if (!match) { 2290 return undefined; 2291 } 2292 2293 switch (match[1]) { 2294 case "ts-expect-error": 2295 return CommentDirectiveType.ExpectError; 2296 2297 case "ts-ignore": 2298 return CommentDirectiveType.Ignore; 2299 } 2300 2301 return undefined; 2302 } 2303 2304 /** 2305 * Unconditionally back up and scan a template expression portion. 2306 */ 2307 function reScanTemplateToken(isTaggedTemplate: boolean): SyntaxKind { 2308 Debug.assert(token === SyntaxKind.CloseBraceToken, "'reScanTemplateToken' should only be called on a '}'"); 2309 pos = tokenPos; 2310 return token = scanTemplateAndSetTokenValue(isTaggedTemplate); 2311 } 2312 2313 function reScanTemplateHeadOrNoSubstitutionTemplate(): SyntaxKind { 2314 pos = tokenPos; 2315 return token = scanTemplateAndSetTokenValue(/* isTaggedTemplate */ true); 2316 } 2317 2318 function reScanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind { 2319 pos = tokenPos = startPos; 2320 return token = scanJsxToken(allowMultilineJsxText); 2321 } 2322 2323 function reScanLessThanToken(): SyntaxKind { 2324 if (token === SyntaxKind.LessThanLessThanToken) { 2325 pos = tokenPos + 1; 2326 return token = SyntaxKind.LessThanToken; 2327 } 2328 return token; 2329 } 2330 2331 function reScanHashToken(): SyntaxKind { 2332 if (token === SyntaxKind.PrivateIdentifier) { 2333 pos = tokenPos + 1; 2334 return token = SyntaxKind.HashToken; 2335 } 2336 return token; 2337 } 2338 2339 function reScanQuestionToken(): SyntaxKind { 2340 Debug.assert(token === SyntaxKind.QuestionQuestionToken, "'reScanQuestionToken' should only be called on a '??'"); 2341 pos = tokenPos + 1; 2342 return token = SyntaxKind.QuestionToken; 2343 } 2344 2345 function scanJsxToken(allowMultilineJsxText = true): JsxTokenSyntaxKind { 2346 startPos = tokenPos = pos; 2347 2348 if (pos >= end) { 2349 return token = SyntaxKind.EndOfFileToken; 2350 } 2351 2352 let char = text.charCodeAt(pos); 2353 if (char === CharacterCodes.lessThan) { 2354 if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { 2355 pos += 2; 2356 return token = SyntaxKind.LessThanSlashToken; 2357 } 2358 pos++; 2359 return token = SyntaxKind.LessThanToken; 2360 } 2361 2362 if (char === CharacterCodes.openBrace) { 2363 pos++; 2364 return token = SyntaxKind.OpenBraceToken; 2365 } 2366 2367 // First non-whitespace character on this line. 2368 let firstNonWhitespace = 0; 2369 2370 // These initial values are special because the first line is: 2371 // firstNonWhitespace = 0 to indicate that we want leading whitespace, 2372 2373 while (pos < end) { 2374 char = text.charCodeAt(pos); 2375 if (char === CharacterCodes.openBrace) { 2376 break; 2377 } 2378 if (char === CharacterCodes.lessThan) { 2379 if (isConflictMarkerTrivia(text, pos)) { 2380 pos = scanConflictMarkerTrivia(text, pos, error); 2381 return token = SyntaxKind.ConflictMarkerTrivia; 2382 } 2383 break; 2384 } 2385 if (char === CharacterCodes.greaterThan) { 2386 error(Diagnostics.Unexpected_token_Did_you_mean_or_gt, pos, 1); 2387 } 2388 if (char === CharacterCodes.closeBrace) { 2389 error(Diagnostics.Unexpected_token_Did_you_mean_or_rbrace, pos, 1); 2390 } 2391 2392 // FirstNonWhitespace is 0, then we only see whitespaces so far. If we see a linebreak, we want to ignore that whitespaces. 2393 // i.e (- : whitespace) 2394 // <div>---- 2395 // </div> becomes <div></div> 2396 // 2397 // <div>----</div> becomes <div>----</div> 2398 if (isLineBreak(char) && firstNonWhitespace === 0) { 2399 firstNonWhitespace = -1; 2400 } 2401 else if (!allowMultilineJsxText && isLineBreak(char) && firstNonWhitespace > 0) { 2402 // Stop JsxText on each line during formatting. This allows the formatter to 2403 // indent each line correctly. 2404 break; 2405 } 2406 else if (!isWhiteSpaceLike(char)) { 2407 firstNonWhitespace = pos; 2408 } 2409 2410 pos++; 2411 } 2412 2413 tokenValue = text.substring(startPos, pos); 2414 2415 return firstNonWhitespace === -1 ? SyntaxKind.JsxTextAllWhiteSpaces : SyntaxKind.JsxText; 2416 } 2417 2418 // Scans a JSX identifier; these differ from normal identifiers in that 2419 // they allow dashes 2420 function scanJsxIdentifier(): SyntaxKind { 2421 if (tokenIsIdentifierOrKeyword(token)) { 2422 // An identifier or keyword has already been parsed - check for a `-` or a single instance of `:` and then append it and 2423 // everything after it to the token 2424 // Do note that this means that `scanJsxIdentifier` effectively _mutates_ the visible token without advancing to a new token 2425 // Any caller should be expecting this behavior and should only read the pos or token value after calling it. 2426 let namespaceSeparator = false; 2427 while (pos < end) { 2428 const ch = text.charCodeAt(pos); 2429 if (ch === CharacterCodes.minus) { 2430 tokenValue += "-"; 2431 pos++; 2432 continue; 2433 } 2434 else if (ch === CharacterCodes.colon && !namespaceSeparator) { 2435 tokenValue += ":"; 2436 pos++; 2437 namespaceSeparator = true; 2438 token = SyntaxKind.Identifier; // swap from keyword kind to identifier kind 2439 continue; 2440 } 2441 const oldPos = pos; 2442 tokenValue += scanIdentifierParts(); // reuse `scanIdentifierParts` so unicode escapes are handled 2443 if (pos === oldPos) { 2444 break; 2445 } 2446 } 2447 // Do not include a trailing namespace separator in the token, since this is against the spec. 2448 if (tokenValue.slice(-1) === ":") { 2449 tokenValue = tokenValue.slice(0, -1); 2450 pos--; 2451 } 2452 return getIdentifierToken(); 2453 } 2454 return token; 2455 } 2456 2457 function scanJsxAttributeValue(): SyntaxKind { 2458 startPos = pos; 2459 2460 switch (text.charCodeAt(pos)) { 2461 case CharacterCodes.doubleQuote: 2462 case CharacterCodes.singleQuote: 2463 tokenValue = scanString(/*jsxAttributeString*/ true); 2464 return token = SyntaxKind.StringLiteral; 2465 default: 2466 // If this scans anything other than `{`, it's a parse error. 2467 return scan(); 2468 } 2469 } 2470 2471 function reScanJsxAttributeValue(): SyntaxKind { 2472 pos = tokenPos = startPos; 2473 return scanJsxAttributeValue(); 2474 } 2475 2476 function scanJsDocToken(): JSDocSyntaxKind { 2477 startPos = tokenPos = pos; 2478 tokenFlags = TokenFlags.None; 2479 if (pos >= end) { 2480 return token = SyntaxKind.EndOfFileToken; 2481 } 2482 2483 const ch = codePointAt(text, pos); 2484 pos += charSize(ch); 2485 switch (ch) { 2486 case CharacterCodes.tab: 2487 case CharacterCodes.verticalTab: 2488 case CharacterCodes.formFeed: 2489 case CharacterCodes.space: 2490 while (pos < end && isWhiteSpaceSingleLine(text.charCodeAt(pos))) { 2491 pos++; 2492 } 2493 return token = SyntaxKind.WhitespaceTrivia; 2494 case CharacterCodes.at: 2495 return token = SyntaxKind.AtToken; 2496 case CharacterCodes.carriageReturn: 2497 if (text.charCodeAt(pos) === CharacterCodes.lineFeed) { 2498 pos++; 2499 } 2500 // falls through 2501 case CharacterCodes.lineFeed: 2502 tokenFlags |= TokenFlags.PrecedingLineBreak; 2503 return token = SyntaxKind.NewLineTrivia; 2504 case CharacterCodes.asterisk: 2505 return token = SyntaxKind.AsteriskToken; 2506 case CharacterCodes.openBrace: 2507 return token = SyntaxKind.OpenBraceToken; 2508 case CharacterCodes.closeBrace: 2509 return token = SyntaxKind.CloseBraceToken; 2510 case CharacterCodes.openBracket: 2511 return token = SyntaxKind.OpenBracketToken; 2512 case CharacterCodes.closeBracket: 2513 return token = SyntaxKind.CloseBracketToken; 2514 case CharacterCodes.lessThan: 2515 return token = SyntaxKind.LessThanToken; 2516 case CharacterCodes.greaterThan: 2517 return token = SyntaxKind.GreaterThanToken; 2518 case CharacterCodes.equals: 2519 return token = SyntaxKind.EqualsToken; 2520 case CharacterCodes.comma: 2521 return token = SyntaxKind.CommaToken; 2522 case CharacterCodes.dot: 2523 return token = SyntaxKind.DotToken; 2524 case CharacterCodes.backtick: 2525 return token = SyntaxKind.BacktickToken; 2526 case CharacterCodes.hash: 2527 return token = SyntaxKind.HashToken; 2528 case CharacterCodes.backslash: 2529 pos--; 2530 const extendedCookedChar = peekExtendedUnicodeEscape(); 2531 if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) { 2532 pos += 3; 2533 tokenFlags |= TokenFlags.ExtendedUnicodeEscape; 2534 tokenValue = scanExtendedUnicodeEscape() + scanIdentifierParts(); 2535 return token = getIdentifierToken(); 2536 } 2537 2538 const cookedChar = peekUnicodeEscape(); 2539 if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) { 2540 pos += 6; 2541 tokenFlags |= TokenFlags.UnicodeEscape; 2542 tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts(); 2543 return token = getIdentifierToken(); 2544 } 2545 pos++; 2546 return token = SyntaxKind.Unknown; 2547 } 2548 2549 if (isIdentifierStart(ch, languageVersion)) { 2550 let char = ch; 2551 while (pos < end && isIdentifierPart(char = codePointAt(text, pos), languageVersion) || text.charCodeAt(pos) === CharacterCodes.minus) pos += charSize(char); 2552 tokenValue = text.substring(tokenPos, pos); 2553 if (char === CharacterCodes.backslash) { 2554 tokenValue += scanIdentifierParts(); 2555 } 2556 return token = getIdentifierToken(); 2557 } 2558 else { 2559 return token = SyntaxKind.Unknown; 2560 } 2561 } 2562 2563 function speculationHelper<T>(callback: () => T, isLookahead: boolean): T { 2564 const savePos = pos; 2565 const saveStartPos = startPos; 2566 const saveTokenPos = tokenPos; 2567 const saveToken = token; 2568 const saveTokenValue = tokenValue; 2569 const saveTokenFlags = tokenFlags; 2570 const result = callback(); 2571 2572 // If our callback returned something 'falsy' or we're just looking ahead, 2573 // then unconditionally restore us to where we were. 2574 if (!result || isLookahead) { 2575 pos = savePos; 2576 startPos = saveStartPos; 2577 tokenPos = saveTokenPos; 2578 token = saveToken; 2579 tokenValue = saveTokenValue; 2580 tokenFlags = saveTokenFlags; 2581 } 2582 return result; 2583 } 2584 2585 function scanRange<T>(start: number, length: number, callback: () => T): T { 2586 const saveEnd = end; 2587 const savePos = pos; 2588 const saveStartPos = startPos; 2589 const saveTokenPos = tokenPos; 2590 const saveToken = token; 2591 const saveTokenValue = tokenValue; 2592 const saveTokenFlags = tokenFlags; 2593 const saveErrorExpectations = commentDirectives; 2594 2595 setText(text, start, length); 2596 const result = callback(); 2597 2598 end = saveEnd; 2599 pos = savePos; 2600 startPos = saveStartPos; 2601 tokenPos = saveTokenPos; 2602 token = saveToken; 2603 tokenValue = saveTokenValue; 2604 tokenFlags = saveTokenFlags; 2605 commentDirectives = saveErrorExpectations; 2606 2607 return result; 2608 } 2609 2610 function lookAhead<T>(callback: () => T): T { 2611 return speculationHelper(callback, /*isLookahead*/ true); 2612 } 2613 2614 function tryScan<T>(callback: () => T): T { 2615 return speculationHelper(callback, /*isLookahead*/ false); 2616 } 2617 2618 function getText(): string { 2619 return text; 2620 } 2621 2622 function clearCommentDirectives() { 2623 commentDirectives = undefined; 2624 } 2625 2626 function setText(newText: string | undefined, start: number | undefined, length: number | undefined) { 2627 text = newText || ""; 2628 end = length === undefined ? text.length : start! + length; 2629 setTextPos(start || 0); 2630 } 2631 2632 function setOnError(errorCallback: ErrorCallback | undefined) { 2633 onError = errorCallback; 2634 } 2635 2636 function setScriptTarget(scriptTarget: ScriptTarget) { 2637 languageVersion = scriptTarget; 2638 } 2639 2640 function setLanguageVariant(variant: LanguageVariant) { 2641 languageVariant = variant; 2642 } 2643 2644 function setTextPos(textPos: number) { 2645 Debug.assert(textPos >= 0); 2646 pos = textPos; 2647 startPos = textPos; 2648 tokenPos = textPos; 2649 token = SyntaxKind.Unknown; 2650 tokenValue = undefined!; 2651 tokenFlags = TokenFlags.None; 2652 } 2653 2654 function setInJSDocType(inType: boolean) { 2655 inJSDocType += inType ? 1 : -1; 2656 } 2657} 2658 2659/** @internal */ 2660const codePointAt: (s: string, i: number) => number = (String.prototype as any).codePointAt ? (s, i) => (s as any).codePointAt(i) : function codePointAt(str, i): number { 2661 // from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/codePointAt 2662 const size = str.length; 2663 // Account for out-of-bounds indices: 2664 if (i < 0 || i >= size) { 2665 return undefined!; // String.codePointAt returns `undefined` for OOB indexes 2666 } 2667 // Get the first code unit 2668 const first = str.charCodeAt(i); 2669 // check if it’s the start of a surrogate pair 2670 if (first >= 0xD800 && first <= 0xDBFF && size > i + 1) { // high surrogate and there is a next code unit 2671 const second = str.charCodeAt(i + 1); 2672 if (second >= 0xDC00 && second <= 0xDFFF) { // low surrogate 2673 // https://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae 2674 return (first - 0xD800) * 0x400 + second - 0xDC00 + 0x10000; 2675 } 2676 } 2677 return first; 2678}; 2679 2680/** @internal */ 2681function charSize(ch: number) { 2682 if (ch >= 0x10000) { 2683 return 2; 2684 } 2685 return 1; 2686} 2687 2688// Derived from the 10.1.1 UTF16Encoding of the ES6 Spec. 2689function utf16EncodeAsStringFallback(codePoint: number) { 2690 Debug.assert(0x0 <= codePoint && codePoint <= 0x10FFFF); 2691 2692 if (codePoint <= 65535) { 2693 return String.fromCharCode(codePoint); 2694 } 2695 2696 const codeUnit1 = Math.floor((codePoint - 65536) / 1024) + 0xD800; 2697 const codeUnit2 = ((codePoint - 65536) % 1024) + 0xDC00; 2698 2699 return String.fromCharCode(codeUnit1, codeUnit2); 2700} 2701 2702const utf16EncodeAsStringWorker: (codePoint: number) => string = (String as any).fromCodePoint ? codePoint => (String as any).fromCodePoint(codePoint) : utf16EncodeAsStringFallback; 2703 2704/** @internal */ 2705export function utf16EncodeAsString(codePoint: number) { 2706 return utf16EncodeAsStringWorker(codePoint); 2707} 2708