• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1namespace Harness {
2    export const enum CompilerTestType {
3        Conformance,
4        Regressions,
5        Test262,
6        OH
7    }
8
9    interface CompilerFileBasedTest extends FileBasedTest {
10        readonly content?: string;
11    }
12
13    export class CompilerBaselineRunner extends RunnerBase {
14        private basePath = "tests/cases";
15        private testSuiteName: TestRunnerKind;
16        private emit: boolean;
17
18        public options: string | undefined;
19
20        constructor(public testType: CompilerTestType) {
21            super();
22            this.emit = true;
23            if (testType === CompilerTestType.Conformance) {
24                this.testSuiteName = "conformance";
25            }
26            else if (testType === CompilerTestType.Regressions) {
27                this.testSuiteName = "compiler";
28            }
29            else if (testType === CompilerTestType.Test262) {
30                this.testSuiteName = "test262";
31            }
32            else if (testType === CompilerTestType.OH) {
33                this.testSuiteName = "compiler-oh";
34            }
35            else {
36                this.testSuiteName = "compiler"; // default to this for historical reasons
37            }
38            this.basePath += "/" + this.testSuiteName;
39        }
40
41        public kind() {
42            return this.testSuiteName;
43        }
44
45        public enumerateTestFiles() {
46            // see also: `enumerateTestFiles` in tests/webTestServer.ts
47            return this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true }).map(CompilerTest.getConfigurations);
48        }
49
50        public initializeTests() {
51            describe(this.testSuiteName + " tests", () => {
52                describe("Setup compiler for compiler baselines", () => {
53                    this.parseOptions();
54                });
55
56                // this will set up a series of describe/it blocks to run between the setup and cleanup phases
57                const files = this.tests.length > 0 ? this.tests : IO.enumerateTestFiles(this);
58                files.forEach(test => {
59                    const file = typeof test === "string" ? test : test.file;
60                    this.checkTestCodeOutput(vpath.normalizeSeparators(file), typeof test === "string" ? CompilerTest.getConfigurations(test) : test);
61                });
62            });
63        }
64
65        public checkTestCodeOutput(fileName: string, test?: CompilerFileBasedTest) {
66            if (test && ts.some(test.configurations)) {
67                test.configurations.forEach(configuration => {
68                    describe(`${this.testSuiteName} tests for ${fileName}${configuration ? ` (${getFileBasedTestConfigurationDescription(configuration)})` : ``}`, () => {
69                        this.runSuite(fileName, test, configuration);
70                    });
71                });
72            }
73            else {
74                describe(`${this.testSuiteName} tests for ${fileName}`, () => {
75                    this.runSuite(fileName, test);
76                });
77            }
78        }
79
80        private runSuite(fileName: string, test?: CompilerFileBasedTest, configuration?: FileBasedTestConfiguration) {
81            // Mocha holds onto the closure environment of the describe callback even after the test is done.
82            // Everything declared here should be cleared out in the "after" callback.
83            let compilerTest!: CompilerTest;
84            before(() => {
85                let payload;
86                if (test && test.content) {
87                    const rootDir = test.file.indexOf("conformance") === -1 ? "tests/cases/compiler/" : ts.getDirectoryPath(test.file) + "/";
88                    payload = TestCaseParser.makeUnitsFromTest(test.content, test.file, rootDir);
89                }
90                compilerTest = new CompilerTest(fileName, payload, configuration);
91            });
92            it(`Correct errors for ${fileName}`, () => compilerTest.verifyDiagnostics());
93            it(`Correct module resolution tracing for ${fileName}`, () => compilerTest.verifyModuleResolution());
94            it(`Correct sourcemap content for ${fileName}`, () => compilerTest.verifySourceMapRecord());
95            it(`Correct JS output for ${fileName}`, () => (this.emit && compilerTest.verifyJavaScriptOutput()));
96            it(`Correct Sourcemap output for ${fileName}`, () => compilerTest.verifySourceMapOutput());
97            it(`Correct type/symbol baselines for ${fileName}`, () => compilerTest.verifyTypesAndSymbols());
98            after(() => {
99                compilerTest = undefined!;
100            });
101        }
102
103        private parseOptions() {
104            if (this.options && this.options.length > 0) {
105                this.emit = false;
106
107                const opts = this.options.split(",");
108                for (const opt of opts) {
109                    switch (opt) {
110                        case "emit":
111                            this.emit = true;
112                            break;
113                        default:
114                            throw new Error("unsupported flag");
115                    }
116                }
117            }
118        }
119    }
120
121    class CompilerTest {
122        private static varyBy: readonly string[] = [
123            "module",
124            "moduleResolution",
125            "moduleDetection",
126            "target",
127            "jsx",
128            "removeComments",
129            "importHelpers",
130            "importHelpers",
131            "downlevelIteration",
132            "isolatedModules",
133            "strict",
134            "noImplicitAny",
135            "strictNullChecks",
136            "strictFunctionTypes",
137            "strictBindCallApply",
138            "strictPropertyInitialization",
139            "noImplicitThis",
140            "alwaysStrict",
141            "allowSyntheticDefaultImports",
142            "esModuleInterop",
143            "emitDecoratorMetadata",
144            "skipDefaultLibCheck",
145            "preserveConstEnums",
146            "skipLibCheck",
147            "exactOptionalPropertyTypes",
148            "useDefineForClassFields",
149            "useUnknownInCatchVariables",
150            "noUncheckedIndexedAccess",
151            "noPropertyAccessFromIndexSignature",
152        ];
153        private fileName: string;
154        private justName: string;
155        private configuredName: string;
156        private lastUnit: TestCaseParser.TestUnitData;
157        private harnessSettings: TestCaseParser.CompilerSettings;
158        private hasNonDtsFiles: boolean;
159        private result: compiler.CompilationResult;
160        private options: ts.CompilerOptions;
161        private tsConfigFiles: Compiler.TestFile[];
162        // equivalent to the files that will be passed on the command line
163        private toBeCompiled: Compiler.TestFile[];
164        // equivalent to other files on the file system not directly passed to the compiler (ie things that are referenced by other files)
165        private otherFiles: Compiler.TestFile[];
166
167        constructor(fileName: string, testCaseContent?: TestCaseParser.TestCaseContent, configurationOverrides?: TestCaseParser.CompilerSettings) {
168            this.fileName = fileName;
169            this.justName = vpath.basename(fileName);
170            this.configuredName = this.justName;
171            if (configurationOverrides) {
172                let configuredName = "";
173                const keys = Object
174                    .keys(configurationOverrides)
175                    .sort();
176                for (const key of keys) {
177                    if (configuredName) {
178                        configuredName += ",";
179                    }
180                    configuredName += `${key.toLowerCase()}=${configurationOverrides[key].toLowerCase()}`;
181                }
182                if (configuredName) {
183                    const extname = vpath.extname(this.justName);
184                    const basename = vpath.basename(this.justName, extname, /*ignoreCase*/ true);
185                    this.configuredName = `${basename}(${configuredName})${extname}`;
186                }
187            }
188
189            const rootDir = fileName.indexOf("conformance") === -1 ? "tests/cases/compiler/" : ts.getDirectoryPath(fileName) + "/";
190
191            if (testCaseContent === undefined) {
192                testCaseContent = TestCaseParser.makeUnitsFromTest(IO.readFile(fileName)!, fileName, rootDir);
193            }
194
195            if (configurationOverrides) {
196                testCaseContent = { ...testCaseContent, settings: { ...testCaseContent.settings, ...configurationOverrides } };
197            }
198
199            const units = testCaseContent.testUnitData;
200            this.harnessSettings = testCaseContent.settings;
201            let tsConfigOptions: ts.CompilerOptions | undefined;
202            this.tsConfigFiles = [];
203            if (testCaseContent.tsConfig) {
204                assert.equal(testCaseContent.tsConfig.fileNames.length, 0, `list of files in tsconfig is not currently supported`);
205                assert.equal(testCaseContent.tsConfig.raw.exclude, undefined, `exclude in tsconfig is not currently supported`);
206
207                tsConfigOptions = ts.cloneCompilerOptions(testCaseContent.tsConfig.options);
208                this.tsConfigFiles.push(this.createHarnessTestFile(testCaseContent.tsConfigFileUnitData!, rootDir, ts.combinePaths(rootDir, tsConfigOptions.configFilePath)));
209            }
210            else {
211                const baseUrl = this.harnessSettings.baseUrl;
212                if (baseUrl !== undefined && !ts.isRootedDiskPath(baseUrl)) {
213                    this.harnessSettings.baseUrl = ts.getNormalizedAbsolutePath(baseUrl, rootDir);
214                }
215            }
216
217            this.lastUnit = units[units.length - 1];
218            this.hasNonDtsFiles = units.some(unit => !ts.isDeclarationFileName(unit.name));
219            // We need to assemble the list of input files for the compiler and other related files on the 'filesystem' (ie in a multi-file test)
220            // If the last file in a test uses require or a triple slash reference we'll assume all other files will be brought in via references,
221            // otherwise, assume all files are just meant to be in the same compilation session without explicit references to one another.
222            this.toBeCompiled = [];
223            this.otherFiles = [];
224
225            if (testCaseContent.settings.noImplicitReferences || /require\(/.test(this.lastUnit.content) || /reference\spath/.test(this.lastUnit.content)) {
226                this.toBeCompiled.push(this.createHarnessTestFile(this.lastUnit, rootDir));
227                units.forEach(unit => {
228                    if (unit.name !== this.lastUnit.name) {
229                        this.otherFiles.push(this.createHarnessTestFile(unit, rootDir));
230                    }
231                });
232            }
233            else {
234                this.toBeCompiled = units.map(unit => {
235                    return this.createHarnessTestFile(unit, rootDir);
236                });
237            }
238
239            if (tsConfigOptions && tsConfigOptions.configFilePath !== undefined) {
240                tsConfigOptions.configFilePath = ts.combinePaths(rootDir, tsConfigOptions.configFilePath);
241                tsConfigOptions.configFile!.fileName = tsConfigOptions.configFilePath;
242            }
243
244            this.result = Compiler.compileFiles(
245                this.toBeCompiled,
246                this.otherFiles,
247                this.harnessSettings,
248                /*options*/ tsConfigOptions,
249                /*currentDirectory*/ this.harnessSettings.currentDirectory,
250                testCaseContent.symlinks
251            );
252
253            this.options = this.result.options;
254        }
255
256        public static getConfigurations(file: string): CompilerFileBasedTest {
257            // also see `parseCompilerTestConfigurations` in tests/webTestServer.ts
258            const content = IO.readFile(file)!;
259            const settings = TestCaseParser.extractCompilerSettings(content);
260            const configurations = getFileBasedTestConfigurations(settings, CompilerTest.varyBy);
261            return { file, configurations, content };
262        }
263
264        public verifyDiagnostics() {
265            // check errors
266            Compiler.doErrorBaseline(
267                this.configuredName,
268                this.tsConfigFiles.concat(this.toBeCompiled, this.otherFiles),
269                this.result.diagnostics,
270                !!this.options.pretty);
271        }
272
273        public verifyModuleResolution() {
274            if (this.options.traceResolution) {
275                Baseline.runBaseline(this.configuredName.replace(/\.tsx?$/, ".trace.json"),
276                    JSON.stringify(this.result.traces.map(Utils.sanitizeTraceResolutionLogEntry), undefined, 4));
277            }
278        }
279
280        public verifySourceMapRecord() {
281            if (this.options.sourceMap || this.options.inlineSourceMap || this.options.declarationMap) {
282                const record = Utils.removeTestPathPrefixes(this.result.getSourceMapRecord()!);
283                const baseline = (this.options.noEmitOnError && this.result.diagnostics.length !== 0) || record === undefined
284                    // Because of the noEmitOnError option no files are created. We need to return null because baselining isn't required.
285                    ? null // eslint-disable-line no-null/no-null
286                    : record;
287                Baseline.runBaseline(this.configuredName.replace(/\.tsx?$/, ".sourcemap.txt"), baseline);
288            }
289        }
290
291        public verifyJavaScriptOutput() {
292            if (this.hasNonDtsFiles) {
293                Compiler.doJsEmitBaseline(
294                    this.configuredName,
295                    this.fileName,
296                    this.options,
297                    this.result,
298                    this.tsConfigFiles,
299                    this.toBeCompiled,
300                    this.otherFiles,
301                    this.harnessSettings);
302            }
303        }
304
305        public verifySourceMapOutput() {
306            Compiler.doSourcemapBaseline(
307                this.configuredName,
308                this.options,
309                this.result,
310                this.harnessSettings);
311        }
312
313        public verifyTypesAndSymbols() {
314            if (this.fileName.indexOf("APISample") >= 0) {
315                return;
316            }
317
318            const noTypesAndSymbols =
319                this.harnessSettings.noTypesAndSymbols &&
320                this.harnessSettings.noTypesAndSymbols.toLowerCase() === "true";
321            if (noTypesAndSymbols) {
322                return;
323            }
324
325            Compiler.doTypeAndSymbolBaseline(
326                this.configuredName,
327                this.result.program!,
328                this.toBeCompiled.concat(this.otherFiles).filter(file => !!this.result.program!.getSourceFile(file.unitName)),
329                /*opts*/ undefined,
330                /*multifile*/ undefined,
331                /*skipTypeBaselines*/ undefined,
332                /*skipSymbolBaselines*/ undefined,
333                !!ts.length(this.result.diagnostics)
334            );
335        }
336
337        private makeUnitName(name: string, root: string) {
338            const path = ts.toPath(name, root, ts.identity);
339            const pathStart = ts.toPath(IO.getCurrentDirectory(), "", ts.identity);
340            return pathStart ? path.replace(pathStart, "/") : path;
341        }
342
343        private createHarnessTestFile(lastUnit: TestCaseParser.TestUnitData, rootDir: string, unitName?: string): Compiler.TestFile {
344            return { unitName: unitName || this.makeUnitName(lastUnit.name, rootDir), content: lastUnit.content, fileOptions: lastUnit.fileOptions };
345        }
346    }
347}
348