• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1namespace Harness {
2    /* eslint-disable prefer-const */
3    export let runners: RunnerBase[] = [];
4    export let iterations = 1;
5    /* eslint-enable prefer-const */
6
7    function runTests(runners: RunnerBase[]) {
8        for (let i = iterations; i > 0; i--) {
9            for (const runner of runners) {
10                runner.initializeTests();
11            }
12        }
13    }
14
15    function tryGetConfig(args: string[]) {
16        const prefix = "--config=";
17        const configPath = ts.forEach(args, arg => arg.lastIndexOf(prefix, 0) === 0 && arg.substr(prefix.length));
18        // strip leading and trailing quotes from the path (necessary on Windows since shell does not do it automatically)
19        return configPath && configPath.replace(/(^[\"'])|([\"']$)/g, "");
20    }
21
22    export function createRunner(kind: TestRunnerKind): RunnerBase {
23        switch (kind) {
24            case "conformance":
25                return new CompilerBaselineRunner(CompilerTestType.Conformance);
26            case "compiler":
27                return new CompilerBaselineRunner(CompilerTestType.Regressions);
28            case "compiler-oh":
29                return new CompilerBaselineRunner(CompilerTestType.OH);
30            case "fourslash":
31                return new FourSlashRunner(FourSlash.FourSlashTestType.Native);
32            case "fourslash-oh":
33                return new FourSlashRunner(FourSlash.FourSlashTestType.OH);
34            case "fourslash-shims":
35                return new FourSlashRunner(FourSlash.FourSlashTestType.Shims);
36            case "fourslash-shims-pp":
37                return new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess);
38            case "fourslash-server":
39                return new FourSlashRunner(FourSlash.FourSlashTestType.Server);
40            case "project":
41                return new project.ProjectRunner();
42            case "rwc":
43                return new RWC.RWCRunner();
44            case "test262":
45                return new Test262BaselineRunner();
46            case "user":
47                return new UserCodeRunner();
48            case "dt":
49                return new DefinitelyTypedRunner();
50            case "docker":
51                return new DockerfileRunner();
52        }
53        return ts.Debug.fail(`Unknown runner kind ${kind}`);
54    }
55
56    // users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options
57
58    const mytestconfigFileName = "mytest.config";
59    const testconfigFileName = "test.config";
60
61    const customConfig = tryGetConfig(IO.args());
62    const testConfigContent =
63        customConfig && IO.fileExists(customConfig)
64            ? IO.readFile(customConfig)!
65            : IO.fileExists(mytestconfigFileName)
66                ? IO.readFile(mytestconfigFileName)!
67                : IO.fileExists(testconfigFileName) ? IO.readFile(testconfigFileName)! : "";
68
69    export let taskConfigsFolder: string;
70    export let workerCount: number;
71    export let runUnitTests: boolean | undefined;
72    export let stackTraceLimit: number | "full" | undefined;
73    export let noColors = false;
74    export let keepFailed = false;
75
76    export interface TestConfig {
77        light?: boolean;
78        taskConfigsFolder?: string;
79        listenForWork?: boolean;
80        workerCount?: number;
81        stackTraceLimit?: number | "full";
82        test?: string[];
83        runners?: string[];
84        runUnitTests?: boolean;
85        noColors?: boolean;
86        timeout?: number;
87        keepFailed?: boolean;
88        shardId?: number;
89        shards?: number;
90    }
91
92    export interface TaskSet {
93        runner: TestRunnerKind;
94        files: string[];
95    }
96
97    export let configOption: string;
98    export let globalTimeout: number;
99    function handleTestConfig() {
100        if (testConfigContent !== "") {
101            const testConfig = <TestConfig>JSON.parse(testConfigContent);
102            if (testConfig.light) {
103                setLightMode(true);
104            }
105            if (testConfig.timeout) {
106                globalTimeout = testConfig.timeout;
107            }
108            runUnitTests = testConfig.runUnitTests;
109            if (testConfig.workerCount) {
110                workerCount = +testConfig.workerCount;
111            }
112            if (testConfig.taskConfigsFolder) {
113                taskConfigsFolder = testConfig.taskConfigsFolder;
114            }
115            if (testConfig.noColors !== undefined) {
116                noColors = testConfig.noColors;
117            }
118            if (testConfig.keepFailed) {
119                keepFailed = true;
120            }
121            if (testConfig.shardId) {
122                setShardId(testConfig.shardId);
123            }
124            if (testConfig.shards) {
125                setShards(testConfig.shards);
126            }
127
128            if (testConfig.stackTraceLimit === "full") {
129                (<any>Error).stackTraceLimit = Infinity;
130                stackTraceLimit = testConfig.stackTraceLimit;
131            }
132            else if ((+testConfig.stackTraceLimit! | 0) > 0) {
133                (<any>Error).stackTraceLimit = +testConfig.stackTraceLimit! | 0;
134                stackTraceLimit = +testConfig.stackTraceLimit! | 0;
135            }
136            if (testConfig.listenForWork) {
137                return true;
138            }
139
140            const runnerConfig = testConfig.runners || testConfig.test;
141            if (runnerConfig && runnerConfig.length > 0) {
142                if (testConfig.runners) {
143                    runUnitTests = runnerConfig.indexOf("unittest") !== -1;
144                }
145                for (const option of runnerConfig) {
146                    if (!option) {
147                        continue;
148                    }
149
150                    if (!configOption) {
151                        configOption = option;
152                    }
153                    else {
154                        configOption += "+" + option;
155                    }
156
157                    switch (option) {
158                        case "compiler":
159                            runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));
160                            runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions));
161                            break;
162                        case "compiler-oh":
163                            runners.push(new CompilerBaselineRunner(CompilerTestType.OH));
164                            break;
165                        case "conformance":
166                            runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));
167                            break;
168                        case "project":
169                            runners.push(new project.ProjectRunner());
170                            break;
171                        case "fourslash":
172                            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Native));
173                            break;
174                        case "fourslash-oh":
175                            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.OH));
176                            break;
177                        case "fourslash-shims":
178                            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Shims));
179                            break;
180                        case "fourslash-shims-pp":
181                            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess));
182                            break;
183                        case "fourslash-server":
184                            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server));
185                            break;
186                        case "fourslash-generated":
187                            runners.push(new GeneratedFourslashRunner(FourSlash.FourSlashTestType.Native));
188                            break;
189                        case "rwc":
190                            runners.push(new RWC.RWCRunner());
191                            break;
192                        case "test262":
193                            runners.push(new Test262BaselineRunner());
194                            break;
195                        case "user":
196                            runners.push(new UserCodeRunner());
197                            break;
198                        case "dt":
199                            runners.push(new DefinitelyTypedRunner());
200                            break;
201                        case "docker":
202                            runners.push(new DockerfileRunner());
203                            break;
204                    }
205                }
206            }
207        }
208
209        if (runners.length === 0) {
210            // compiler
211            runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));
212            runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions));
213            runners.push(new CompilerBaselineRunner(CompilerTestType.OH));
214            runners.push(new project.ProjectRunner());
215
216            // language services
217            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Native));
218            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.OH));
219            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Shims));
220            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess));
221            runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server));
222            // runners.push(new GeneratedFourslashRunner());
223
224            // CRON-only tests
225            if (process.env.TRAVIS_EVENT_TYPE === "cron") {
226                runners.push(new UserCodeRunner());
227                runners.push(new DockerfileRunner());
228            }
229        }
230        if (runUnitTests === undefined) {
231            runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for
232        }
233        return false;
234    }
235
236    function beginTests() {
237        ts.Debug.loggingHost = {
238            log(_level, s) {
239                console.log(s || "");
240            }
241        };
242
243        if (ts.Debug.isDebugging) {
244            ts.Debug.enableDebugInfo();
245        }
246
247        // run tests in en-US by default.
248        let savedUILocale: string | undefined;
249        beforeEach(() => {
250            savedUILocale = ts.getUILocale();
251            ts.setUILocale("en-US");
252        });
253        afterEach(() => ts.setUILocale(savedUILocale));
254
255        runTests(runners);
256
257        if (!runUnitTests) {
258            // patch `describe` to skip unit tests
259            (global as any).describe = ts.noop;
260        }
261    }
262
263    export let isWorker: boolean;
264    function startTestEnvironment() {
265        isWorker = handleTestConfig();
266        if (isWorker) {
267            return Parallel.Worker.start();
268        }
269        else if (taskConfigsFolder && workerCount && workerCount > 1) {
270            return Parallel.Host.start();
271        }
272        beginTests();
273    }
274
275    startTestEnvironment();
276}
277