1// lots of tests use quoted code 2/* eslint-disable no-template-curly-in-string */ 3 4interface ClassificationEntry { 5 value: any; 6 classification: ts.TokenClass; 7 position?: number; 8} 9 10describe("unittests:: services:: Colorization", () => { 11 // Use the shim adapter to ensure test coverage of the shim layer for the classifier 12 const languageServiceAdapter = new Harness.LanguageService.ShimLanguageServiceAdapter(/*preprocessToResolve*/ false); 13 const classifier = languageServiceAdapter.getClassifier(); 14 15 function getEntryAtPosition(result: ts.ClassificationResult, position: number) { 16 let entryPosition = 0; 17 for (const entry of result.entries) { 18 if (entryPosition === position) { 19 return entry; 20 } 21 entryPosition += entry.length; 22 } 23 return undefined; 24 } 25 26 function punctuation(text: string, position?: number) { return createClassification(text, ts.TokenClass.Punctuation, position); } 27 function keyword(text: string, position?: number) { return createClassification(text, ts.TokenClass.Keyword, position); } 28 function operator(text: string, position?: number) { return createClassification(text, ts.TokenClass.Operator, position); } 29 function comment(text: string, position?: number) { return createClassification(text, ts.TokenClass.Comment, position); } 30 function whitespace(text: string, position?: number) { return createClassification(text, ts.TokenClass.Whitespace, position); } 31 function identifier(text: string, position?: number) { return createClassification(text, ts.TokenClass.Identifier, position); } 32 function numberLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.NumberLiteral, position); } 33 function stringLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.StringLiteral, position); } 34 function finalEndOfLineState(value: number): ClassificationEntry { return { value, classification: undefined!, position: 0 }; } // TODO: GH#18217 35 function createClassification(value: string, classification: ts.TokenClass, position?: number): ClassificationEntry { 36 return { value, classification, position }; 37 } 38 39 function testLexicalClassification(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void { 40 const result = classifier.getClassificationsForLine(text, initialEndOfLineState, /*syntacticClassifierAbsent*/ false); 41 42 for (const expectedEntry of expectedEntries) { 43 if (expectedEntry.classification === undefined) { 44 assert.equal(result.finalLexState, expectedEntry.value, "final endOfLineState does not match expected."); 45 } 46 else { 47 const actualEntryPosition = expectedEntry.position !== undefined ? expectedEntry.position : text.indexOf(expectedEntry.value); 48 assert(actualEntryPosition >= 0, "token: '" + expectedEntry.value + "' does not exit in text: '" + text + "'."); 49 50 const actualEntry = getEntryAtPosition(result, actualEntryPosition)!; 51 52 assert(actualEntry, "Could not find classification entry for '" + expectedEntry.value + "' at position: " + actualEntryPosition); 53 assert.equal(actualEntry.classification, expectedEntry.classification, "Classification class does not match expected. Expected: " + ts.TokenClass[expectedEntry.classification] + ", Actual: " + ts.TokenClass[actualEntry.classification]); 54 assert.equal(actualEntry.length, expectedEntry.value.length, "Classification length does not match expected. Expected: " + ts.TokenClass[expectedEntry.value.length] + ", Actual: " + ts.TokenClass[actualEntry.length]); 55 } 56 } 57 } 58 59 describe("test getClassifications", () => { 60 it("returns correct token classes", () => { 61 testLexicalClassification("var x: string = \"foo\" ?? \"bar\"; //Hello", 62 ts.EndOfLineState.None, 63 keyword("var"), 64 whitespace(" "), 65 identifier("x"), 66 punctuation(":"), 67 keyword("string"), 68 operator("="), 69 stringLiteral("\"foo\""), 70 whitespace(" "), 71 operator("??"), 72 stringLiteral("\"foo\""), 73 comment("//Hello"), 74 punctuation(";")); 75 }); 76 77 it("correctly classifies a comment after a divide operator", () => { 78 testLexicalClassification("1 / 2 // comment", 79 ts.EndOfLineState.None, 80 numberLiteral("1"), 81 whitespace(" "), 82 operator("/"), 83 numberLiteral("2"), 84 comment("// comment")); 85 }); 86 87 it("correctly classifies a literal after a divide operator", () => { 88 testLexicalClassification("1 / 2, 3 / 4", 89 ts.EndOfLineState.None, 90 numberLiteral("1"), 91 whitespace(" "), 92 operator("/"), 93 numberLiteral("2"), 94 numberLiteral("3"), 95 numberLiteral("4"), 96 operator(",")); 97 }); 98 99 it("correctly classifies a multiline string with one backslash", () => { 100 testLexicalClassification("'line1\\", 101 ts.EndOfLineState.None, 102 stringLiteral("'line1\\"), 103 finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); 104 }); 105 106 it("correctly classifies a multiline string with three backslashes", () => { 107 testLexicalClassification("'line1\\\\\\", 108 ts.EndOfLineState.None, 109 stringLiteral("'line1\\\\\\"), 110 finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); 111 }); 112 113 it("correctly classifies an unterminated single-line string with no backslashes", () => { 114 testLexicalClassification("'line1", 115 ts.EndOfLineState.None, 116 stringLiteral("'line1"), 117 finalEndOfLineState(ts.EndOfLineState.None)); 118 }); 119 120 it("correctly classifies an unterminated single-line string with two backslashes", () => { 121 testLexicalClassification("'line1\\\\", 122 ts.EndOfLineState.None, 123 stringLiteral("'line1\\\\"), 124 finalEndOfLineState(ts.EndOfLineState.None)); 125 }); 126 127 it("correctly classifies an unterminated single-line string with four backslashes", () => { 128 testLexicalClassification("'line1\\\\\\\\", 129 ts.EndOfLineState.None, 130 stringLiteral("'line1\\\\\\\\"), 131 finalEndOfLineState(ts.EndOfLineState.None)); 132 }); 133 134 it("correctly classifies the continuing line of a multiline string ending in one backslash", () => { 135 testLexicalClassification("\\", 136 ts.EndOfLineState.InDoubleQuoteStringLiteral, 137 stringLiteral("\\"), 138 finalEndOfLineState(ts.EndOfLineState.InDoubleQuoteStringLiteral)); 139 }); 140 141 it("correctly classifies the continuing line of a multiline string ending in three backslashes", () => { 142 testLexicalClassification("\\", 143 ts.EndOfLineState.InDoubleQuoteStringLiteral, 144 stringLiteral("\\"), 145 finalEndOfLineState(ts.EndOfLineState.InDoubleQuoteStringLiteral)); 146 }); 147 148 it("correctly classifies the last line of an unterminated multiline string ending in no backslashes", () => { 149 testLexicalClassification(" ", 150 ts.EndOfLineState.InDoubleQuoteStringLiteral, 151 stringLiteral(" "), 152 finalEndOfLineState(ts.EndOfLineState.None)); 153 }); 154 155 it("correctly classifies the last line of an unterminated multiline string ending in two backslashes", () => { 156 testLexicalClassification("\\\\", 157 ts.EndOfLineState.InDoubleQuoteStringLiteral, 158 stringLiteral("\\\\"), 159 finalEndOfLineState(ts.EndOfLineState.None)); 160 }); 161 162 it("correctly classifies the last line of an unterminated multiline string ending in four backslashes", () => { 163 testLexicalClassification("\\\\\\\\", 164 ts.EndOfLineState.InDoubleQuoteStringLiteral, 165 stringLiteral("\\\\\\\\"), 166 finalEndOfLineState(ts.EndOfLineState.None)); 167 }); 168 169 it("correctly classifies the last line of a multiline string", () => { 170 testLexicalClassification("'", 171 ts.EndOfLineState.InSingleQuoteStringLiteral, 172 stringLiteral("'"), 173 finalEndOfLineState(ts.EndOfLineState.None)); 174 }); 175 176 it("correctly classifies an unterminated multiline comment", () => { 177 testLexicalClassification("/*", 178 ts.EndOfLineState.None, 179 comment("/*"), 180 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); 181 }); 182 183 it("correctly classifies the termination of a multiline comment", () => { 184 testLexicalClassification(" */ ", 185 ts.EndOfLineState.InMultiLineCommentTrivia, 186 comment(" */"), 187 finalEndOfLineState(ts.EndOfLineState.None)); 188 }); 189 190 it("correctly classifies the continuation of a multiline comment", () => { 191 testLexicalClassification("LOREM IPSUM DOLOR ", 192 ts.EndOfLineState.InMultiLineCommentTrivia, 193 comment("LOREM IPSUM DOLOR "), 194 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); 195 }); 196 197 it("correctly classifies an unterminated multiline comment on a line ending in '/*/'", () => { 198 testLexicalClassification(" /*/", 199 ts.EndOfLineState.None, 200 comment("/*/"), 201 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); 202 }); 203 204 it("correctly classifies an unterminated multiline comment with trailing space", () => { 205 testLexicalClassification("/* ", 206 ts.EndOfLineState.None, 207 comment("/* "), 208 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); 209 }); 210 211 it("correctly classifies a keyword after a dot", () => { 212 testLexicalClassification("a.var", 213 ts.EndOfLineState.None, 214 identifier("var")); 215 }); 216 217 it("correctly classifies a string literal after a dot", () => { 218 testLexicalClassification("a.\"var\"", 219 ts.EndOfLineState.None, 220 stringLiteral("\"var\"")); 221 }); 222 223 it("correctly classifies a keyword after a dot separated by comment trivia", () => { 224 testLexicalClassification("a./*hello world*/ var", 225 ts.EndOfLineState.None, 226 identifier("a"), 227 punctuation("."), 228 comment("/*hello world*/"), 229 identifier("var")); 230 }); 231 232 it("classifies a property access with whitespace around the dot", () => { 233 testLexicalClassification(" x .\tfoo ()", 234 ts.EndOfLineState.None, 235 identifier("x"), 236 identifier("foo")); 237 }); 238 239 it("classifies a keyword after a dot on previous line", () => { 240 testLexicalClassification("var", 241 ts.EndOfLineState.None, 242 keyword("var"), 243 finalEndOfLineState(ts.EndOfLineState.None)); 244 }); 245 246 it("classifies multiple keywords properly", () => { 247 testLexicalClassification("public static", 248 ts.EndOfLineState.None, 249 keyword("public"), 250 keyword("static"), 251 finalEndOfLineState(ts.EndOfLineState.None)); 252 253 testLexicalClassification("public var", 254 ts.EndOfLineState.None, 255 keyword("public"), 256 identifier("var"), 257 finalEndOfLineState(ts.EndOfLineState.None)); 258 }); 259 260 it("classifies a single line no substitution template string correctly", () => { 261 testLexicalClassification("`number number public string`", 262 ts.EndOfLineState.None, 263 stringLiteral("`number number public string`"), 264 finalEndOfLineState(ts.EndOfLineState.None)); 265 }); 266 it("classifies substitution parts of a template string correctly", () => { 267 testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`", 268 ts.EndOfLineState.None, 269 stringLiteral("`number '${"), 270 numberLiteral("1"), 271 operator("+"), 272 numberLiteral("1"), 273 stringLiteral("}' string '${"), 274 stringLiteral("'hello'"), 275 stringLiteral("}'`"), 276 finalEndOfLineState(ts.EndOfLineState.None)); 277 }); 278 it("classifies an unterminated no substitution template string correctly", () => { 279 testLexicalClassification("`hello world", 280 ts.EndOfLineState.None, 281 stringLiteral("`hello world"), 282 finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); 283 }); 284 it("classifies the entire line of an unterminated multiline no-substitution/head template", () => { 285 testLexicalClassification("...", 286 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 287 stringLiteral("..."), 288 finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); 289 }); 290 it("classifies the entire line of an unterminated multiline template middle/end", () => { 291 testLexicalClassification("...", 292 ts.EndOfLineState.InTemplateMiddleOrTail, 293 stringLiteral("..."), 294 finalEndOfLineState(ts.EndOfLineState.InTemplateMiddleOrTail)); 295 }); 296 it("classifies a termination of a multiline template head", () => { 297 testLexicalClassification("...${", 298 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 299 stringLiteral("...${"), 300 finalEndOfLineState(ts.EndOfLineState.InTemplateSubstitutionPosition)); 301 }); 302 it("classifies the termination of a multiline no substitution template", () => { 303 testLexicalClassification("...`", 304 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 305 stringLiteral("...`"), 306 finalEndOfLineState(ts.EndOfLineState.None)); 307 }); 308 it("classifies the substitution parts and middle/tail of a multiline template string", () => { 309 testLexicalClassification("${ 1 + 1 }...`", 310 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 311 stringLiteral("${"), 312 numberLiteral("1"), 313 operator("+"), 314 numberLiteral("1"), 315 stringLiteral("}...`"), 316 finalEndOfLineState(ts.EndOfLineState.None)); 317 }); 318 it("classifies a template middle and propagates the end of line state", () => { 319 testLexicalClassification("${ 1 + 1 }...`", 320 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 321 stringLiteral("${"), 322 numberLiteral("1"), 323 operator("+"), 324 numberLiteral("1"), 325 stringLiteral("}...`"), 326 finalEndOfLineState(ts.EndOfLineState.None)); 327 }); 328 it("classifies substitution expressions with curly braces appropriately", () => { 329 let pos = 0; 330 let lastLength = 0; 331 332 testLexicalClassification("...${ () => { } } ${ { x: `1` } }...`", 333 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 334 stringLiteral(track("...${"), pos), 335 punctuation(track(" ", "("), pos), 336 punctuation(track(")"), pos), 337 punctuation(track(" ", "=>"), pos), 338 punctuation(track(" ", "{"), pos), 339 punctuation(track(" ", "}"), pos), 340 stringLiteral(track(" ", "} ${"), pos), 341 punctuation(track(" ", "{"), pos), 342 identifier(track(" ", "x"), pos), 343 punctuation(track(":"), pos), 344 stringLiteral(track(" ", "`1`"), pos), 345 punctuation(track(" ", "}"), pos), 346 stringLiteral(track(" ", "}...`"), pos), 347 finalEndOfLineState(ts.EndOfLineState.None)); 348 349 // Adjusts 'pos' by accounting for the length of each portion of the string, 350 // but only return the last given string 351 function track(...vals: string[]): string { 352 for (const val of vals) { 353 pos += lastLength; 354 lastLength = val.length; 355 } 356 return ts.last(vals); 357 } 358 }); 359 360 it("classifies partially written generics correctly.", () => { 361 testLexicalClassification("Foo<number", 362 ts.EndOfLineState.None, 363 identifier("Foo"), 364 operator("<"), 365 identifier("number"), 366 finalEndOfLineState(ts.EndOfLineState.None)); 367 368 // Looks like a cast, should get classified as a keyword. 369 testLexicalClassification("<number", 370 ts.EndOfLineState.None, 371 operator("<"), 372 keyword("number"), 373 finalEndOfLineState(ts.EndOfLineState.None)); 374 375 // handle nesting properly. 376 testLexicalClassification("Foo<Foo,Foo<number", 377 ts.EndOfLineState.None, 378 identifier("Foo"), 379 operator("<"), 380 identifier("Foo"), 381 operator(","), 382 identifier("Foo"), 383 operator("<"), 384 identifier("number"), 385 finalEndOfLineState(ts.EndOfLineState.None)); 386 }); 387 388 it("LexicallyClassifiesConflictTokens", () => { 389 // Test conflict markers. 390 testLexicalClassification( 391 "class C {\r\n\ 392<<<<<<< HEAD\r\n\ 393 v = 1;\r\n\ 394=======\r\n\ 395 v = 2;\r\n\ 396>>>>>>> Branch - a\r\n\ 397}", 398 ts.EndOfLineState.None, 399 keyword("class"), 400 identifier("C"), 401 punctuation("{"), 402 comment("<<<<<<< HEAD"), 403 identifier("v"), 404 operator("="), 405 numberLiteral("1"), 406 punctuation(";"), 407 comment("=======\r\n v = 2;\r\n"), 408 comment(">>>>>>> Branch - a"), 409 punctuation("}"), 410 finalEndOfLineState(ts.EndOfLineState.None)); 411 412 testLexicalClassification( 413 "<<<<<<< HEAD\r\n\ 414class C { }\r\n\ 415=======\r\n\ 416class D { }\r\n\ 417>>>>>>> Branch - a\r\n", 418 ts.EndOfLineState.None, 419 comment("<<<<<<< HEAD"), 420 keyword("class"), 421 identifier("C"), 422 punctuation("{"), 423 punctuation("}"), 424 comment("=======\r\nclass D { }\r\n"), 425 comment(">>>>>>> Branch - a"), 426 finalEndOfLineState(ts.EndOfLineState.None)); 427 428 testLexicalClassification( 429 "class C {\r\n\ 430<<<<<<< HEAD\r\n\ 431 v = 1;\r\n\ 432||||||| merged common ancestors\r\n\ 433 v = 3;\r\n\ 434=======\r\n\ 435 v = 2;\r\n\ 436>>>>>>> Branch - a\r\n\ 437}", 438 ts.EndOfLineState.None, 439 keyword("class"), 440 identifier("C"), 441 punctuation("{"), 442 comment("<<<<<<< HEAD"), 443 identifier("v"), 444 operator("="), 445 numberLiteral("1"), 446 punctuation(";"), 447 comment("||||||| merged common ancestors\r\n v = 3;\r\n"), 448 comment("=======\r\n v = 2;\r\n"), 449 comment(">>>>>>> Branch - a"), 450 punctuation("}"), 451 finalEndOfLineState(ts.EndOfLineState.None)); 452 453 testLexicalClassification( 454 "<<<<<<< HEAD\r\n\ 455class C { }\r\n\ 456||||||| merged common ancestors\r\n\ 457class E { }\r\n\ 458=======\r\n\ 459class D { }\r\n\ 460>>>>>>> Branch - a\r\n", 461 ts.EndOfLineState.None, 462 comment("<<<<<<< HEAD"), 463 keyword("class"), 464 identifier("C"), 465 punctuation("{"), 466 punctuation("}"), 467 comment("||||||| merged common ancestors\r\nclass E { }\r\n"), 468 comment("=======\r\nclass D { }\r\n"), 469 comment(">>>>>>> Branch - a"), 470 finalEndOfLineState(ts.EndOfLineState.None)); 471 }); 472 473 it("'of' keyword", () => { 474 testLexicalClassification("for (var of of of) { }", 475 ts.EndOfLineState.None, 476 keyword("for"), 477 punctuation("("), 478 keyword("var"), 479 keyword("of"), 480 keyword("of"), 481 keyword("of"), 482 punctuation(")"), 483 punctuation("{"), 484 punctuation("}"), 485 finalEndOfLineState(ts.EndOfLineState.None)); 486 }); 487 }); 488}); 489