• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1namespace Harness {
2    export const enum CompilerTestType {
3        Conformance,
4        Regressions,
5        Test262
6    }
7
8    interface CompilerFileBasedTest extends FileBasedTest {
9        readonly content?: string;
10    }
11
12    export class CompilerBaselineRunner extends RunnerBase {
13        private basePath = "tests/cases";
14        private testSuiteName: TestRunnerKind;
15        private emit: boolean;
16
17        public options: string | undefined;
18
19        constructor(public testType: CompilerTestType) {
20            super();
21            this.emit = true;
22            if (testType === CompilerTestType.Conformance) {
23                this.testSuiteName = "conformance";
24            }
25            else if (testType === CompilerTestType.Regressions) {
26                this.testSuiteName = "compiler";
27            }
28            else if (testType === CompilerTestType.Test262) {
29                this.testSuiteName = "test262";
30            }
31            else {
32                this.testSuiteName = "compiler"; // default to this for historical reasons
33            }
34            this.basePath += "/" + this.testSuiteName;
35        }
36
37        public kind() {
38            return this.testSuiteName;
39        }
40
41        public enumerateTestFiles() {
42            // see also: `enumerateTestFiles` in tests/webTestServer.ts
43            return this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true }).map(CompilerTest.getConfigurations);
44        }
45
46        public initializeTests() {
47            describe(this.testSuiteName + " tests", () => {
48                describe("Setup compiler for compiler baselines", () => {
49                    this.parseOptions();
50                });
51
52                // this will set up a series of describe/it blocks to run between the setup and cleanup phases
53                const files = this.tests.length > 0 ? this.tests : IO.enumerateTestFiles(this);
54                files.forEach(test => {
55                    const file = typeof test === "string" ? test : test.file;
56                    this.checkTestCodeOutput(vpath.normalizeSeparators(file), typeof test === "string" ? CompilerTest.getConfigurations(test) : test);
57                });
58            });
59        }
60
61        public checkTestCodeOutput(fileName: string, test?: CompilerFileBasedTest) {
62            if (test && ts.some(test.configurations)) {
63                test.configurations.forEach(configuration => {
64                    describe(`${this.testSuiteName} tests for ${fileName}${configuration ? ` (${getFileBasedTestConfigurationDescription(configuration)})` : ``}`, () => {
65                        this.runSuite(fileName, test, configuration);
66                    });
67                });
68            }
69            else {
70                describe(`${this.testSuiteName} tests for ${fileName}`, () => {
71                    this.runSuite(fileName, test);
72                });
73            }
74        }
75
76        private runSuite(fileName: string, test?: CompilerFileBasedTest, configuration?: FileBasedTestConfiguration) {
77            // Mocha holds onto the closure environment of the describe callback even after the test is done.
78            // Everything declared here should be cleared out in the "after" callback.
79            let compilerTest!: CompilerTest;
80            before(() => {
81                let payload;
82                if (test && test.content) {
83                    const rootDir = test.file.indexOf("conformance") === -1 ? "tests/cases/compiler/" : ts.getDirectoryPath(test.file) + "/";
84                    payload = TestCaseParser.makeUnitsFromTest(test.content, test.file, rootDir);
85                }
86                compilerTest = new CompilerTest(fileName, payload, configuration);
87            });
88            it(`Correct errors for ${fileName}`, () => { compilerTest.verifyDiagnostics(); });
89            it(`Correct module resolution tracing for ${fileName}`, () => { compilerTest.verifyModuleResolution(); });
90            it(`Correct sourcemap content for ${fileName}`, () => { compilerTest.verifySourceMapRecord(); });
91            it(`Correct JS output for ${fileName}`, () => { if (this.emit) compilerTest.verifyJavaScriptOutput(); });
92            it(`Correct Sourcemap output for ${fileName}`, () => { compilerTest.verifySourceMapOutput(); });
93            it(`Correct type/symbol baselines for ${fileName}`, () => { compilerTest.verifyTypesAndSymbols(); });
94            after(() => { compilerTest = undefined!; });
95        }
96
97        private parseOptions() {
98            if (this.options && this.options.length > 0) {
99                this.emit = false;
100
101                const opts = this.options.split(",");
102                for (const opt of opts) {
103                    switch (opt) {
104                        case "emit":
105                            this.emit = true;
106                            break;
107                        default:
108                            throw new Error("unsupported flag");
109                    }
110                }
111            }
112        }
113    }
114
115    class CompilerTest {
116        private static varyBy: readonly string[] = [
117            "module",
118            "target",
119            "jsx",
120            "removeComments",
121            "importHelpers",
122            "importHelpers",
123            "downlevelIteration",
124            "isolatedModules",
125            "strict",
126            "noImplicitAny",
127            "strictNullChecks",
128            "strictFunctionTypes",
129            "strictBindCallApply",
130            "strictPropertyInitialization",
131            "noImplicitThis",
132            "alwaysStrict",
133            "allowSyntheticDefaultImports",
134            "esModuleInterop",
135            "emitDecoratorMetadata",
136            "skipDefaultLibCheck",
137            "preserveConstEnums",
138            "skipLibCheck",
139        ];
140        private fileName: string;
141        private justName: string;
142        private configuredName: string;
143        private lastUnit: TestCaseParser.TestUnitData;
144        private harnessSettings: TestCaseParser.CompilerSettings;
145        private hasNonDtsFiles: boolean;
146        private result: compiler.CompilationResult;
147        private options: ts.CompilerOptions;
148        private tsConfigFiles: Compiler.TestFile[];
149        // equivalent to the files that will be passed on the command line
150        private toBeCompiled: Compiler.TestFile[];
151        // equivalent to other files on the file system not directly passed to the compiler (ie things that are referenced by other files)
152        private otherFiles: Compiler.TestFile[];
153
154        constructor(fileName: string, testCaseContent?: TestCaseParser.TestCaseContent, configurationOverrides?: TestCaseParser.CompilerSettings) {
155            this.fileName = fileName;
156            this.justName = vpath.basename(fileName);
157            this.configuredName = this.justName;
158            if (configurationOverrides) {
159                let configuredName = "";
160                const keys = Object
161                    .keys(configurationOverrides)
162                    .sort();
163                for (const key of keys) {
164                    if (configuredName) {
165                        configuredName += ",";
166                    }
167                    configuredName += `${key.toLowerCase()}=${configurationOverrides[key].toLowerCase()}`;
168                }
169                if (configuredName) {
170                    const extname = vpath.extname(this.justName);
171                    const basename = vpath.basename(this.justName, extname, /*ignoreCase*/ true);
172                    this.configuredName = `${basename}(${configuredName})${extname}`;
173                }
174            }
175
176            const rootDir = fileName.indexOf("conformance") === -1 ? "tests/cases/compiler/" : ts.getDirectoryPath(fileName) + "/";
177
178            if (testCaseContent === undefined) {
179                testCaseContent = TestCaseParser.makeUnitsFromTest(IO.readFile(fileName)!, fileName, rootDir);
180            }
181
182            if (configurationOverrides) {
183                testCaseContent = { ...testCaseContent, settings: { ...testCaseContent.settings, ...configurationOverrides } };
184            }
185
186            const units = testCaseContent.testUnitData;
187            this.harnessSettings = testCaseContent.settings;
188            let tsConfigOptions: ts.CompilerOptions | undefined;
189            this.tsConfigFiles = [];
190            if (testCaseContent.tsConfig) {
191                assert.equal(testCaseContent.tsConfig.fileNames.length, 0, `list of files in tsconfig is not currently supported`);
192                assert.equal(testCaseContent.tsConfig.raw.exclude, undefined, `exclude in tsconfig is not currently supported`);
193
194                tsConfigOptions = ts.cloneCompilerOptions(testCaseContent.tsConfig.options);
195                this.tsConfigFiles.push(this.createHarnessTestFile(testCaseContent.tsConfigFileUnitData!, rootDir, ts.combinePaths(rootDir, tsConfigOptions.configFilePath)));
196            }
197            else {
198                const baseUrl = this.harnessSettings.baseUrl;
199                if (baseUrl !== undefined && !ts.isRootedDiskPath(baseUrl)) {
200                    this.harnessSettings.baseUrl = ts.getNormalizedAbsolutePath(baseUrl, rootDir);
201                }
202            }
203
204            this.lastUnit = units[units.length - 1];
205            this.hasNonDtsFiles = units.some(unit => !ts.fileExtensionIs(unit.name, ts.Extension.Dts));
206            // We need to assemble the list of input files for the compiler and other related files on the 'filesystem' (ie in a multi-file test)
207            // If the last file in a test uses require or a triple slash reference we'll assume all other files will be brought in via references,
208            // otherwise, assume all files are just meant to be in the same compilation session without explicit references to one another.
209            this.toBeCompiled = [];
210            this.otherFiles = [];
211
212            if (testCaseContent.settings.noImplicitReferences || /require\(/.test(this.lastUnit.content) || /reference\spath/.test(this.lastUnit.content)) {
213                this.toBeCompiled.push(this.createHarnessTestFile(this.lastUnit, rootDir));
214                units.forEach(unit => {
215                    if (unit.name !== this.lastUnit.name) {
216                        this.otherFiles.push(this.createHarnessTestFile(unit, rootDir));
217                    }
218                });
219            }
220            else {
221                this.toBeCompiled = units.map(unit => {
222                    return this.createHarnessTestFile(unit, rootDir);
223                });
224            }
225
226            if (tsConfigOptions && tsConfigOptions.configFilePath !== undefined) {
227                tsConfigOptions.configFilePath = ts.combinePaths(rootDir, tsConfigOptions.configFilePath);
228                tsConfigOptions.configFile!.fileName = tsConfigOptions.configFilePath;
229            }
230
231            this.result = Compiler.compileFiles(
232                this.toBeCompiled,
233                this.otherFiles,
234                this.harnessSettings,
235                /*options*/ tsConfigOptions,
236                /*currentDirectory*/ this.harnessSettings.currentDirectory,
237                testCaseContent.symlinks
238            );
239
240            this.options = this.result.options;
241        }
242
243        public static getConfigurations(file: string): CompilerFileBasedTest {
244            // also see `parseCompilerTestConfigurations` in tests/webTestServer.ts
245            const content = IO.readFile(file)!;
246            const settings = TestCaseParser.extractCompilerSettings(content);
247            const configurations = getFileBasedTestConfigurations(settings, CompilerTest.varyBy);
248            return { file, configurations, content };
249        }
250
251        public verifyDiagnostics() {
252            // check errors
253            Compiler.doErrorBaseline(
254                this.configuredName,
255                this.tsConfigFiles.concat(this.toBeCompiled, this.otherFiles),
256                this.result.diagnostics,
257                !!this.options.pretty);
258        }
259
260        public verifyModuleResolution() {
261            if (this.options.traceResolution) {
262                Baseline.runBaseline(this.configuredName.replace(/\.tsx?$/, ".trace.json"),
263                    JSON.stringify(this.result.traces.map(Utils.sanitizeTraceResolutionLogEntry), undefined, 4));
264            }
265        }
266
267        public verifySourceMapRecord() {
268            if (this.options.sourceMap || this.options.inlineSourceMap || this.options.declarationMap) {
269                const record = Utils.removeTestPathPrefixes(this.result.getSourceMapRecord()!);
270                const baseline = (this.options.noEmitOnError && this.result.diagnostics.length !== 0) || record === undefined
271                    // Because of the noEmitOnError option no files are created. We need to return null because baselining isn't required.
272                    ? null // eslint-disable-line no-null/no-null
273                    : record;
274                Baseline.runBaseline(this.configuredName.replace(/\.tsx?$/, ".sourcemap.txt"), baseline);
275            }
276        }
277
278        public verifyJavaScriptOutput() {
279            if (this.hasNonDtsFiles) {
280                Compiler.doJsEmitBaseline(
281                    this.configuredName,
282                    this.fileName,
283                    this.options,
284                    this.result,
285                    this.tsConfigFiles,
286                    this.toBeCompiled,
287                    this.otherFiles,
288                    this.harnessSettings);
289            }
290        }
291
292        public verifySourceMapOutput() {
293            Compiler.doSourcemapBaseline(
294                this.configuredName,
295                this.options,
296                this.result,
297                this.harnessSettings);
298        }
299
300        public verifyTypesAndSymbols() {
301            if (this.fileName.indexOf("APISample") >= 0) {
302                return;
303            }
304
305            const noTypesAndSymbols =
306                this.harnessSettings.noTypesAndSymbols &&
307                this.harnessSettings.noTypesAndSymbols.toLowerCase() === "true";
308            if (noTypesAndSymbols) {
309                return;
310            }
311
312            Compiler.doTypeAndSymbolBaseline(
313                this.configuredName,
314                this.result.program!,
315                this.toBeCompiled.concat(this.otherFiles).filter(file => !!this.result.program!.getSourceFile(file.unitName)),
316                /*opts*/ undefined,
317                /*multifile*/ undefined,
318                /*skipTypeBaselines*/ undefined,
319                /*skipSymbolBaselines*/ undefined,
320                !!ts.length(this.result.diagnostics)
321            );
322        }
323
324        private makeUnitName(name: string, root: string) {
325            const path = ts.toPath(name, root, ts.identity);
326            const pathStart = ts.toPath(IO.getCurrentDirectory(), "", ts.identity);
327            return pathStart ? path.replace(pathStart, "/") : path;
328        }
329
330        private createHarnessTestFile(lastUnit: TestCaseParser.TestUnitData, rootDir: string, unitName?: string): Compiler.TestFile {
331            return { unitName: unitName || this.makeUnitName(lastUnit.name, rootDir), content: lastUnit.content, fileOptions: lastUnit.fileOptions };
332        }
333    }
334}