• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 namespace Harness {
2     export const enum CompilerTestType {
3         Conformance,
4         Regressions,
5         Test262
6     }
7 
8     interface CompilerFileBasedTest extends FileBasedTest {
9         readonly content?: string;
10     }
11 
12     export class CompilerBaselineRunner extends RunnerBase {
13         private basePath = "tests/cases";
14         private testSuiteName: TestRunnerKind;
15         private emit: boolean;
16 
17         public options: string | undefined;
18 
19         constructor(public testType: CompilerTestType) {
20             super();
21             this.emit = true;
22             if (testType === CompilerTestType.Conformance) {
23                 this.testSuiteName = "conformance";
24             }
25             else if (testType === CompilerTestType.Regressions) {
26                 this.testSuiteName = "compiler";
27             }
28             else if (testType === CompilerTestType.Test262) {
29                 this.testSuiteName = "test262";
30             }
31             else {
32                 this.testSuiteName = "compiler"; // default to this for historical reasons
33             }
34             this.basePath += "/" + this.testSuiteName;
35         }
36 
37         public kind() {
38             return this.testSuiteName;
39         }
40 
41         public enumerateTestFiles() {
42             // see also: `enumerateTestFiles` in tests/webTestServer.ts
43             return this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true }).map(CompilerTest.getConfigurations);
44         }
45 
46         public initializeTests() {
47             describe(this.testSuiteName + " tests", () => {
48                 describe("Setup compiler for compiler baselines", () => {
49                     this.parseOptions();
50                 });
51 
52                 // this will set up a series of describe/it blocks to run between the setup and cleanup phases
53                 const files = this.tests.length > 0 ? this.tests : IO.enumerateTestFiles(this);
54                 files.forEach(test => {
55                     const file = typeof test === "string" ? test : test.file;
56                     this.checkTestCodeOutput(vpath.normalizeSeparators(file), typeof test === "string" ? CompilerTest.getConfigurations(test) : test);
57                 });
58             });
59         }
60 
61         public checkTestCodeOutput(fileName: string, test?: CompilerFileBasedTest) {
62             if (test && ts.some(test.configurations)) {
63                 test.configurations.forEach(configuration => {
64                     describe(`${this.testSuiteName} tests for ${fileName}${configuration ? ` (${getFileBasedTestConfigurationDescription(configuration)})` : ``}`, () => {
65                         this.runSuite(fileName, test, configuration);
66                     });
67                 });
68             }
69             else {
70                 describe(`${this.testSuiteName} tests for ${fileName}`, () => {
71                     this.runSuite(fileName, test);
72                 });
73             }
74         }
75 
76         private runSuite(fileName: string, test?: CompilerFileBasedTest, configuration?: FileBasedTestConfiguration) {
77             // Mocha holds onto the closure environment of the describe callback even after the test is done.
78             // Everything declared here should be cleared out in the "after" callback.
79             let compilerTest!: CompilerTest;
80             before(() => {
81                 let payload;
82                 if (test && test.content) {
83                     const rootDir = test.file.indexOf("conformance") === -1 ? "tests/cases/compiler/" : ts.getDirectoryPath(test.file) + "/";
84                     payload = TestCaseParser.makeUnitsFromTest(test.content, test.file, rootDir);
85                 }
86                 compilerTest = new CompilerTest(fileName, payload, configuration);
87             });
88             it(`Correct errors for ${fileName}`, () => { compilerTest.verifyDiagnostics(); });
89             it(`Correct module resolution tracing for ${fileName}`, () => { compilerTest.verifyModuleResolution(); });
90             it(`Correct sourcemap content for ${fileName}`, () => { compilerTest.verifySourceMapRecord(); });
91             it(`Correct JS output for ${fileName}`, () => { if (this.emit) compilerTest.verifyJavaScriptOutput(); });
92             it(`Correct Sourcemap output for ${fileName}`, () => { compilerTest.verifySourceMapOutput(); });
93             it(`Correct type/symbol baselines for ${fileName}`, () => { compilerTest.verifyTypesAndSymbols(); });
94             after(() => { compilerTest = undefined!; });
95         }
96 
97         private parseOptions() {
98             if (this.options && this.options.length > 0) {
99                 this.emit = false;
100 
101                 const opts = this.options.split(",");
102                 for (const opt of opts) {
103                     switch (opt) {
104                         case "emit":
105                             this.emit = true;
106                             break;
107                         default:
108                             throw new Error("unsupported flag");
109                     }
110                 }
111             }
112         }
113     }
114 
115     class CompilerTest {
116         private static varyBy: readonly string[] = [
117             "module",
118             "target",
119             "jsx",
120             "removeComments",
121             "importHelpers",
122             "importHelpers",
123             "downlevelIteration",
124             "isolatedModules",
125             "strict",
126             "noImplicitAny",
127             "strictNullChecks",
128             "strictFunctionTypes",
129             "strictBindCallApply",
130             "strictPropertyInitialization",
131             "noImplicitThis",
132             "alwaysStrict",
133             "allowSyntheticDefaultImports",
134             "esModuleInterop",
135             "emitDecoratorMetadata",
136             "skipDefaultLibCheck",
137             "preserveConstEnums",
138             "skipLibCheck",
139         ];
140         private fileName: string;
141         private justName: string;
142         private configuredName: string;
143         private lastUnit: TestCaseParser.TestUnitData;
144         private harnessSettings: TestCaseParser.CompilerSettings;
145         private hasNonDtsFiles: boolean;
146         private result: compiler.CompilationResult;
147         private options: ts.CompilerOptions;
148         private tsConfigFiles: Compiler.TestFile[];
149         // equivalent to the files that will be passed on the command line
150         private toBeCompiled: Compiler.TestFile[];
151         // equivalent to other files on the file system not directly passed to the compiler (ie things that are referenced by other files)
152         private otherFiles: Compiler.TestFile[];
153 
154         constructor(fileName: string, testCaseContent?: TestCaseParser.TestCaseContent, configurationOverrides?: TestCaseParser.CompilerSettings) {
155             this.fileName = fileName;
156             this.justName = vpath.basename(fileName);
157             this.configuredName = this.justName;
158             if (configurationOverrides) {
159                 let configuredName = "";
160                 const keys = Object
161                     .keys(configurationOverrides)
162                     .sort();
163                 for (const key of keys) {
164                     if (configuredName) {
165                         configuredName += ",";
166                     }
167                     configuredName += `${key.toLowerCase()}=${configurationOverrides[key].toLowerCase()}`;
168                 }
169                 if (configuredName) {
170                     const extname = vpath.extname(this.justName);
171                     const basename = vpath.basename(this.justName, extname, /*ignoreCase*/ true);
172                     this.configuredName = `${basename}(${configuredName})${extname}`;
173                 }
174             }
175 
176             const rootDir = fileName.indexOf("conformance") === -1 ? "tests/cases/compiler/" : ts.getDirectoryPath(fileName) + "/";
177 
178             if (testCaseContent === undefined) {
179                 testCaseContent = TestCaseParser.makeUnitsFromTest(IO.readFile(fileName)!, fileName, rootDir);
180             }
181 
182             if (configurationOverrides) {
183                 testCaseContent = { ...testCaseContent, settings: { ...testCaseContent.settings, ...configurationOverrides } };
184             }
185 
186             const units = testCaseContent.testUnitData;
187             this.harnessSettings = testCaseContent.settings;
188             let tsConfigOptions: ts.CompilerOptions | undefined;
189             this.tsConfigFiles = [];
190             if (testCaseContent.tsConfig) {
191                 assert.equal(testCaseContent.tsConfig.fileNames.length, 0, `list of files in tsconfig is not currently supported`);
192                 assert.equal(testCaseContent.tsConfig.raw.exclude, undefined, `exclude in tsconfig is not currently supported`);
193 
194                 tsConfigOptions = ts.cloneCompilerOptions(testCaseContent.tsConfig.options);
195                 this.tsConfigFiles.push(this.createHarnessTestFile(testCaseContent.tsConfigFileUnitData!, rootDir, ts.combinePaths(rootDir, tsConfigOptions.configFilePath)));
196             }
197             else {
198                 const baseUrl = this.harnessSettings.baseUrl;
199                 if (baseUrl !== undefined && !ts.isRootedDiskPath(baseUrl)) {
200                     this.harnessSettings.baseUrl = ts.getNormalizedAbsolutePath(baseUrl, rootDir);
201                 }
202             }
203 
204             this.lastUnit = units[units.length - 1];
205             this.hasNonDtsFiles = units.some(unit => !ts.fileExtensionIs(unit.name, ts.Extension.Dts));
206             // We need to assemble the list of input files for the compiler and other related files on the 'filesystem' (ie in a multi-file test)
207             // If the last file in a test uses require or a triple slash reference we'll assume all other files will be brought in via references,
208             // otherwise, assume all files are just meant to be in the same compilation session without explicit references to one another.
209             this.toBeCompiled = [];
210             this.otherFiles = [];
211 
212             if (testCaseContent.settings.noImplicitReferences || /require\(/.test(this.lastUnit.content) || /reference\spath/.test(this.lastUnit.content)) {
213                 this.toBeCompiled.push(this.createHarnessTestFile(this.lastUnit, rootDir));
214                 units.forEach(unit => {
215                     if (unit.name !== this.lastUnit.name) {
216                         this.otherFiles.push(this.createHarnessTestFile(unit, rootDir));
217                     }
218                 });
219             }
220             else {
221                 this.toBeCompiled = units.map(unit => {
222                     return this.createHarnessTestFile(unit, rootDir);
223                 });
224             }
225 
226             if (tsConfigOptions && tsConfigOptions.configFilePath !== undefined) {
227                 tsConfigOptions.configFilePath = ts.combinePaths(rootDir, tsConfigOptions.configFilePath);
228                 tsConfigOptions.configFile!.fileName = tsConfigOptions.configFilePath;
229             }
230 
231             this.result = Compiler.compileFiles(
232                 this.toBeCompiled,
233                 this.otherFiles,
234                 this.harnessSettings,
235                 /*options*/ tsConfigOptions,
236                 /*currentDirectory*/ this.harnessSettings.currentDirectory,
237                 testCaseContent.symlinks
238             );
239 
240             this.options = this.result.options;
241         }
242 
243         public static getConfigurations(file: string): CompilerFileBasedTest {
244             // also see `parseCompilerTestConfigurations` in tests/webTestServer.ts
245             const content = IO.readFile(file)!;
246             const settings = TestCaseParser.extractCompilerSettings(content);
247             const configurations = getFileBasedTestConfigurations(settings, CompilerTest.varyBy);
248             return { file, configurations, content };
249         }
250 
251         public verifyDiagnostics() {
252             // check errors
253             Compiler.doErrorBaseline(
254                 this.configuredName,
255                 this.tsConfigFiles.concat(this.toBeCompiled, this.otherFiles),
256                 this.result.diagnostics,
257                 !!this.options.pretty);
258         }
259 
260         public verifyModuleResolution() {
261             if (this.options.traceResolution) {
262                 Baseline.runBaseline(this.configuredName.replace(/\.tsx?$/, ".trace.json"),
263                     JSON.stringify(this.result.traces.map(Utils.sanitizeTraceResolutionLogEntry), undefined, 4));
264             }
265         }
266 
267         public verifySourceMapRecord() {
268             if (this.options.sourceMap || this.options.inlineSourceMap || this.options.declarationMap) {
269                 const record = Utils.removeTestPathPrefixes(this.result.getSourceMapRecord()!);
270                 const baseline = (this.options.noEmitOnError && this.result.diagnostics.length !== 0) || record === undefined
271                     // Because of the noEmitOnError option no files are created. We need to return null because baselining isn't required.
272                     ? null // eslint-disable-line no-null/no-null
273                     : record;
274                 Baseline.runBaseline(this.configuredName.replace(/\.tsx?$/, ".sourcemap.txt"), baseline);
275             }
276         }
277 
278         public verifyJavaScriptOutput() {
279             if (this.hasNonDtsFiles) {
280                 Compiler.doJsEmitBaseline(
281                     this.configuredName,
282                     this.fileName,
283                     this.options,
284                     this.result,
285                     this.tsConfigFiles,
286                     this.toBeCompiled,
287                     this.otherFiles,
288                     this.harnessSettings);
289             }
290         }
291 
292         public verifySourceMapOutput() {
293             Compiler.doSourcemapBaseline(
294                 this.configuredName,
295                 this.options,
296                 this.result,
297                 this.harnessSettings);
298         }
299 
300         public verifyTypesAndSymbols() {
301             if (this.fileName.indexOf("APISample") >= 0) {
302                 return;
303             }
304 
305             const noTypesAndSymbols =
306                 this.harnessSettings.noTypesAndSymbols &&
307                 this.harnessSettings.noTypesAndSymbols.toLowerCase() === "true";
308             if (noTypesAndSymbols) {
309                 return;
310             }
311 
312             Compiler.doTypeAndSymbolBaseline(
313                 this.configuredName,
314                 this.result.program!,
315                 this.toBeCompiled.concat(this.otherFiles).filter(file => !!this.result.program!.getSourceFile(file.unitName)),
316                 /*opts*/ undefined,
317                 /*multifile*/ undefined,
318                 /*skipTypeBaselines*/ undefined,
319                 /*skipSymbolBaselines*/ undefined,
320                 !!ts.length(this.result.diagnostics)
321             );
322         }
323 
324         private makeUnitName(name: string, root: string) {
325             const path = ts.toPath(name, root, ts.identity);
326             const pathStart = ts.toPath(IO.getCurrentDirectory(), "", ts.identity);
327             return pathStart ? path.replace(pathStart, "/") : path;
328         }
329 
330         private createHarnessTestFile(lastUnit: TestCaseParser.TestUnitData, rootDir: string, unitName?: string): Compiler.TestFile {
331             return { unitName: unitName || this.makeUnitName(lastUnit.name, rootDir), content: lastUnit.content, fileOptions: lastUnit.fileOptions };
332         }
333     }
334 }