1// lots of tests use quoted code 2/* eslint-disable no-template-curly-in-string */ 3 4interface ClassificationEntry { 5 value: any; 6 classification: ts.TokenClass; 7 position?: number; 8} 9 10describe("unittests:: services:: Colorization", () => { 11 // Use the shim adapter to ensure test coverage of the shim layer for the classifier 12 const languageServiceAdapter = new Harness.LanguageService.ShimLanguageServiceAdapter(/*preprocessToResolve*/ false); 13 const classifier = languageServiceAdapter.getClassifier(); 14 15 function getEntryAtPosition(result: ts.ClassificationResult, position: number) { 16 let entryPosition = 0; 17 for (const entry of result.entries) { 18 if (entryPosition === position) { 19 return entry; 20 } 21 entryPosition += entry.length; 22 } 23 return undefined; 24 } 25 26 function punctuation(text: string, position?: number) { 27 return createClassification(text, ts.TokenClass.Punctuation, position); 28 } 29 function keyword(text: string, position?: number) { 30 return createClassification(text, ts.TokenClass.Keyword, position); 31 } 32 function operator(text: string, position?: number) { 33 return createClassification(text, ts.TokenClass.Operator, position); 34 } 35 function comment(text: string, position?: number) { 36 return createClassification(text, ts.TokenClass.Comment, position); 37 } 38 function whitespace(text: string, position?: number) { 39 return createClassification(text, ts.TokenClass.Whitespace, position); 40 } 41 function identifier(text: string, position?: number) { 42 return createClassification(text, ts.TokenClass.Identifier, position); 43 } 44 function numberLiteral(text: string, position?: number) { 45 return createClassification(text, ts.TokenClass.NumberLiteral, position); 46 } 47 function stringLiteral(text: string, position?: number) { 48 return createClassification(text, ts.TokenClass.StringLiteral, position); 49 } 50 function finalEndOfLineState(value: number): ClassificationEntry { 51 // TODO: GH#18217 52 return { value, classification: undefined!, position: 0 }; 53 } 54 function createClassification(value: string, classification: ts.TokenClass, position?: number): ClassificationEntry { 55 return { value, classification, position }; 56 } 57 58 function testLexicalClassification(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void { 59 const result = classifier.getClassificationsForLine(text, initialEndOfLineState, /*syntacticClassifierAbsent*/ false); 60 61 for (const expectedEntry of expectedEntries) { 62 if (expectedEntry.classification === undefined) { 63 assert.equal(result.finalLexState, expectedEntry.value, "final endOfLineState does not match expected."); 64 } 65 else { 66 const actualEntryPosition = expectedEntry.position !== undefined ? expectedEntry.position : text.indexOf(expectedEntry.value); 67 assert(actualEntryPosition >= 0, "token: '" + expectedEntry.value + "' does not exit in text: '" + text + "'."); 68 69 const actualEntry = getEntryAtPosition(result, actualEntryPosition)!; 70 71 assert(actualEntry, "Could not find classification entry for '" + expectedEntry.value + "' at position: " + actualEntryPosition); 72 assert.equal(actualEntry.classification, expectedEntry.classification, "Classification class does not match expected. Expected: " + ts.TokenClass[expectedEntry.classification] + ", Actual: " + ts.TokenClass[actualEntry.classification]); 73 assert.equal(actualEntry.length, expectedEntry.value.length, "Classification length does not match expected. Expected: " + ts.TokenClass[expectedEntry.value.length] + ", Actual: " + ts.TokenClass[actualEntry.length]); 74 } 75 } 76 } 77 78 describe("test getClassifications", () => { 79 it("returns correct token classes", () => { 80 testLexicalClassification("var x: string = \"foo\" ?? \"bar\"; //Hello", 81 ts.EndOfLineState.None, 82 keyword("var"), 83 whitespace(" "), 84 identifier("x"), 85 punctuation(":"), 86 keyword("string"), 87 operator("="), 88 stringLiteral("\"foo\""), 89 whitespace(" "), 90 operator("??"), 91 stringLiteral("\"foo\""), 92 comment("//Hello"), 93 punctuation(";")); 94 }); 95 96 it("correctly classifies a comment after a divide operator", () => { 97 testLexicalClassification("1 / 2 // comment", 98 ts.EndOfLineState.None, 99 numberLiteral("1"), 100 whitespace(" "), 101 operator("/"), 102 numberLiteral("2"), 103 comment("// comment")); 104 }); 105 106 it("correctly classifies a literal after a divide operator", () => { 107 testLexicalClassification("1 / 2, 3 / 4", 108 ts.EndOfLineState.None, 109 numberLiteral("1"), 110 whitespace(" "), 111 operator("/"), 112 numberLiteral("2"), 113 numberLiteral("3"), 114 numberLiteral("4"), 115 operator(",")); 116 }); 117 118 it("correctly classifies a multiline string with one backslash", () => { 119 testLexicalClassification("'line1\\", 120 ts.EndOfLineState.None, 121 stringLiteral("'line1\\"), 122 finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); 123 }); 124 125 it("correctly classifies a multiline string with three backslashes", () => { 126 testLexicalClassification("'line1\\\\\\", 127 ts.EndOfLineState.None, 128 stringLiteral("'line1\\\\\\"), 129 finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); 130 }); 131 132 it("correctly classifies an unterminated single-line string with no backslashes", () => { 133 testLexicalClassification("'line1", 134 ts.EndOfLineState.None, 135 stringLiteral("'line1"), 136 finalEndOfLineState(ts.EndOfLineState.None)); 137 }); 138 139 it("correctly classifies an unterminated single-line string with two backslashes", () => { 140 testLexicalClassification("'line1\\\\", 141 ts.EndOfLineState.None, 142 stringLiteral("'line1\\\\"), 143 finalEndOfLineState(ts.EndOfLineState.None)); 144 }); 145 146 it("correctly classifies an unterminated single-line string with four backslashes", () => { 147 testLexicalClassification("'line1\\\\\\\\", 148 ts.EndOfLineState.None, 149 stringLiteral("'line1\\\\\\\\"), 150 finalEndOfLineState(ts.EndOfLineState.None)); 151 }); 152 153 it("correctly classifies the continuing line of a multiline string ending in one backslash", () => { 154 testLexicalClassification("\\", 155 ts.EndOfLineState.InDoubleQuoteStringLiteral, 156 stringLiteral("\\"), 157 finalEndOfLineState(ts.EndOfLineState.InDoubleQuoteStringLiteral)); 158 }); 159 160 it("correctly classifies the continuing line of a multiline string ending in three backslashes", () => { 161 testLexicalClassification("\\", 162 ts.EndOfLineState.InDoubleQuoteStringLiteral, 163 stringLiteral("\\"), 164 finalEndOfLineState(ts.EndOfLineState.InDoubleQuoteStringLiteral)); 165 }); 166 167 it("correctly classifies the last line of an unterminated multiline string ending in no backslashes", () => { 168 testLexicalClassification(" ", 169 ts.EndOfLineState.InDoubleQuoteStringLiteral, 170 stringLiteral(" "), 171 finalEndOfLineState(ts.EndOfLineState.None)); 172 }); 173 174 it("correctly classifies the last line of an unterminated multiline string ending in two backslashes", () => { 175 testLexicalClassification("\\\\", 176 ts.EndOfLineState.InDoubleQuoteStringLiteral, 177 stringLiteral("\\\\"), 178 finalEndOfLineState(ts.EndOfLineState.None)); 179 }); 180 181 it("correctly classifies the last line of an unterminated multiline string ending in four backslashes", () => { 182 testLexicalClassification("\\\\\\\\", 183 ts.EndOfLineState.InDoubleQuoteStringLiteral, 184 stringLiteral("\\\\\\\\"), 185 finalEndOfLineState(ts.EndOfLineState.None)); 186 }); 187 188 it("correctly classifies the last line of a multiline string", () => { 189 testLexicalClassification("'", 190 ts.EndOfLineState.InSingleQuoteStringLiteral, 191 stringLiteral("'"), 192 finalEndOfLineState(ts.EndOfLineState.None)); 193 }); 194 195 it("correctly classifies an unterminated multiline comment", () => { 196 testLexicalClassification("/*", 197 ts.EndOfLineState.None, 198 comment("/*"), 199 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); 200 }); 201 202 it("correctly classifies the termination of a multiline comment", () => { 203 testLexicalClassification(" */ ", 204 ts.EndOfLineState.InMultiLineCommentTrivia, 205 comment(" */"), 206 finalEndOfLineState(ts.EndOfLineState.None)); 207 }); 208 209 it("correctly classifies the continuation of a multiline comment", () => { 210 testLexicalClassification("LOREM IPSUM DOLOR ", 211 ts.EndOfLineState.InMultiLineCommentTrivia, 212 comment("LOREM IPSUM DOLOR "), 213 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); 214 }); 215 216 it("correctly classifies an unterminated multiline comment on a line ending in '/*/'", () => { 217 testLexicalClassification(" /*/", 218 ts.EndOfLineState.None, 219 comment("/*/"), 220 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); 221 }); 222 223 it("correctly classifies an unterminated multiline comment with trailing space", () => { 224 testLexicalClassification("/* ", 225 ts.EndOfLineState.None, 226 comment("/* "), 227 finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); 228 }); 229 230 it("correctly classifies a keyword after a dot", () => { 231 testLexicalClassification("a.var", 232 ts.EndOfLineState.None, 233 identifier("var")); 234 }); 235 236 it("correctly classifies a string literal after a dot", () => { 237 testLexicalClassification("a.\"var\"", 238 ts.EndOfLineState.None, 239 stringLiteral("\"var\"")); 240 }); 241 242 it("correctly classifies a keyword after a dot separated by comment trivia", () => { 243 testLexicalClassification("a./*hello world*/ var", 244 ts.EndOfLineState.None, 245 identifier("a"), 246 punctuation("."), 247 comment("/*hello world*/"), 248 identifier("var")); 249 }); 250 251 it("classifies a property access with whitespace around the dot", () => { 252 testLexicalClassification(" x .\tfoo ()", 253 ts.EndOfLineState.None, 254 identifier("x"), 255 identifier("foo")); 256 }); 257 258 it("classifies a keyword after a dot on previous line", () => { 259 testLexicalClassification("var", 260 ts.EndOfLineState.None, 261 keyword("var"), 262 finalEndOfLineState(ts.EndOfLineState.None)); 263 }); 264 265 it("classifies multiple keywords properly", () => { 266 testLexicalClassification("public static", 267 ts.EndOfLineState.None, 268 keyword("public"), 269 keyword("static"), 270 finalEndOfLineState(ts.EndOfLineState.None)); 271 272 testLexicalClassification("public var", 273 ts.EndOfLineState.None, 274 keyword("public"), 275 identifier("var"), 276 finalEndOfLineState(ts.EndOfLineState.None)); 277 }); 278 279 it("classifies a single line no substitution template string correctly", () => { 280 testLexicalClassification("`number number public string`", 281 ts.EndOfLineState.None, 282 stringLiteral("`number number public string`"), 283 finalEndOfLineState(ts.EndOfLineState.None)); 284 }); 285 it("classifies substitution parts of a template string correctly", () => { 286 testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`", 287 ts.EndOfLineState.None, 288 stringLiteral("`number '${"), 289 numberLiteral("1"), 290 operator("+"), 291 numberLiteral("1"), 292 stringLiteral("}' string '${"), 293 stringLiteral("'hello'"), 294 stringLiteral("}'`"), 295 finalEndOfLineState(ts.EndOfLineState.None)); 296 }); 297 it("classifies an unterminated no substitution template string correctly", () => { 298 testLexicalClassification("`hello world", 299 ts.EndOfLineState.None, 300 stringLiteral("`hello world"), 301 finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); 302 }); 303 it("classifies the entire line of an unterminated multiline no-substitution/head template", () => { 304 testLexicalClassification("...", 305 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 306 stringLiteral("..."), 307 finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); 308 }); 309 it("classifies the entire line of an unterminated multiline template middle/end", () => { 310 testLexicalClassification("...", 311 ts.EndOfLineState.InTemplateMiddleOrTail, 312 stringLiteral("..."), 313 finalEndOfLineState(ts.EndOfLineState.InTemplateMiddleOrTail)); 314 }); 315 it("classifies a termination of a multiline template head", () => { 316 testLexicalClassification("...${", 317 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 318 stringLiteral("...${"), 319 finalEndOfLineState(ts.EndOfLineState.InTemplateSubstitutionPosition)); 320 }); 321 it("classifies the termination of a multiline no substitution template", () => { 322 testLexicalClassification("...`", 323 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 324 stringLiteral("...`"), 325 finalEndOfLineState(ts.EndOfLineState.None)); 326 }); 327 it("classifies the substitution parts and middle/tail of a multiline template string", () => { 328 testLexicalClassification("${ 1 + 1 }...`", 329 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 330 stringLiteral("${"), 331 numberLiteral("1"), 332 operator("+"), 333 numberLiteral("1"), 334 stringLiteral("}...`"), 335 finalEndOfLineState(ts.EndOfLineState.None)); 336 }); 337 it("classifies a template middle and propagates the end of line state", () => { 338 testLexicalClassification("${ 1 + 1 }...`", 339 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 340 stringLiteral("${"), 341 numberLiteral("1"), 342 operator("+"), 343 numberLiteral("1"), 344 stringLiteral("}...`"), 345 finalEndOfLineState(ts.EndOfLineState.None)); 346 }); 347 it("classifies substitution expressions with curly braces appropriately", () => { 348 let pos = 0; 349 let lastLength = 0; 350 351 testLexicalClassification("...${ () => { } } ${ { x: `1` } }...`", 352 ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, 353 stringLiteral(track("...${"), pos), 354 punctuation(track(" ", "("), pos), 355 punctuation(track(")"), pos), 356 punctuation(track(" ", "=>"), pos), 357 punctuation(track(" ", "{"), pos), 358 punctuation(track(" ", "}"), pos), 359 stringLiteral(track(" ", "} ${"), pos), 360 punctuation(track(" ", "{"), pos), 361 identifier(track(" ", "x"), pos), 362 punctuation(track(":"), pos), 363 stringLiteral(track(" ", "`1`"), pos), 364 punctuation(track(" ", "}"), pos), 365 stringLiteral(track(" ", "}...`"), pos), 366 finalEndOfLineState(ts.EndOfLineState.None)); 367 368 // Adjusts 'pos' by accounting for the length of each portion of the string, 369 // but only return the last given string 370 function track(...vals: string[]): string { 371 for (const val of vals) { 372 pos += lastLength; 373 lastLength = val.length; 374 } 375 return ts.last(vals); 376 } 377 }); 378 379 it("classifies partially written generics correctly.", () => { 380 testLexicalClassification("Foo<number", 381 ts.EndOfLineState.None, 382 identifier("Foo"), 383 operator("<"), 384 identifier("number"), 385 finalEndOfLineState(ts.EndOfLineState.None)); 386 387 // Looks like a cast, should get classified as a keyword. 388 testLexicalClassification("<number", 389 ts.EndOfLineState.None, 390 operator("<"), 391 keyword("number"), 392 finalEndOfLineState(ts.EndOfLineState.None)); 393 394 // handle nesting properly. 395 testLexicalClassification("Foo<Foo,Foo<number", 396 ts.EndOfLineState.None, 397 identifier("Foo"), 398 operator("<"), 399 identifier("Foo"), 400 operator(","), 401 identifier("Foo"), 402 operator("<"), 403 identifier("number"), 404 finalEndOfLineState(ts.EndOfLineState.None)); 405 }); 406 407 it("LexicallyClassifiesConflictTokens", () => { 408 // Test conflict markers. 409 testLexicalClassification( 410 "class C {\r\n\ 411<<<<<<< HEAD\r\n\ 412 v = 1;\r\n\ 413=======\r\n\ 414 v = 2;\r\n\ 415>>>>>>> Branch - a\r\n\ 416}", 417 ts.EndOfLineState.None, 418 keyword("class"), 419 identifier("C"), 420 punctuation("{"), 421 comment("<<<<<<< HEAD"), 422 identifier("v"), 423 operator("="), 424 numberLiteral("1"), 425 punctuation(";"), 426 comment("=======\r\n v = 2;\r\n"), 427 comment(">>>>>>> Branch - a"), 428 punctuation("}"), 429 finalEndOfLineState(ts.EndOfLineState.None)); 430 431 testLexicalClassification( 432 "<<<<<<< HEAD\r\n\ 433class C { }\r\n\ 434=======\r\n\ 435class D { }\r\n\ 436>>>>>>> Branch - a\r\n", 437 ts.EndOfLineState.None, 438 comment("<<<<<<< HEAD"), 439 keyword("class"), 440 identifier("C"), 441 punctuation("{"), 442 punctuation("}"), 443 comment("=======\r\nclass D { }\r\n"), 444 comment(">>>>>>> Branch - a"), 445 finalEndOfLineState(ts.EndOfLineState.None)); 446 447 testLexicalClassification( 448 "class C {\r\n\ 449<<<<<<< HEAD\r\n\ 450 v = 1;\r\n\ 451||||||| merged common ancestors\r\n\ 452 v = 3;\r\n\ 453=======\r\n\ 454 v = 2;\r\n\ 455>>>>>>> Branch - a\r\n\ 456}", 457 ts.EndOfLineState.None, 458 keyword("class"), 459 identifier("C"), 460 punctuation("{"), 461 comment("<<<<<<< HEAD"), 462 identifier("v"), 463 operator("="), 464 numberLiteral("1"), 465 punctuation(";"), 466 comment("||||||| merged common ancestors\r\n v = 3;\r\n"), 467 comment("=======\r\n v = 2;\r\n"), 468 comment(">>>>>>> Branch - a"), 469 punctuation("}"), 470 finalEndOfLineState(ts.EndOfLineState.None)); 471 472 testLexicalClassification( 473 "<<<<<<< HEAD\r\n\ 474class C { }\r\n\ 475||||||| merged common ancestors\r\n\ 476class E { }\r\n\ 477=======\r\n\ 478class D { }\r\n\ 479>>>>>>> Branch - a\r\n", 480 ts.EndOfLineState.None, 481 comment("<<<<<<< HEAD"), 482 keyword("class"), 483 identifier("C"), 484 punctuation("{"), 485 punctuation("}"), 486 comment("||||||| merged common ancestors\r\nclass E { }\r\n"), 487 comment("=======\r\nclass D { }\r\n"), 488 comment(">>>>>>> Branch - a"), 489 finalEndOfLineState(ts.EndOfLineState.None)); 490 }); 491 492 it("'of' keyword", () => { 493 testLexicalClassification("for (var of of of) { }", 494 ts.EndOfLineState.None, 495 keyword("for"), 496 punctuation("("), 497 keyword("var"), 498 keyword("of"), 499 keyword("of"), 500 keyword("of"), 501 punctuation(")"), 502 punctuation("{"), 503 punctuation("}"), 504 finalEndOfLineState(ts.EndOfLineState.None)); 505 }); 506 }); 507}); 508