namespace Harness { /* eslint-disable prefer-const */ export let runners: RunnerBase[] = []; export let iterations = 1; /* eslint-enable prefer-const */ function runTests(runners: RunnerBase[]) { for (let i = iterations; i > 0; i--) { for (const runner of runners) { runner.initializeTests(); } } } function tryGetConfig(args: string[]) { const prefix = "--config="; const configPath = ts.forEach(args, arg => arg.lastIndexOf(prefix, 0) === 0 && arg.substr(prefix.length)); // strip leading and trailing quotes from the path (necessary on Windows since shell does not do it automatically) return configPath && configPath.replace(/(^[\"'])|([\"']$)/g, ""); } export function createRunner(kind: TestRunnerKind): RunnerBase { switch (kind) { case "conformance": return new CompilerBaselineRunner(CompilerTestType.Conformance); case "compiler": return new CompilerBaselineRunner(CompilerTestType.Regressions); case "compiler-oh": return new CompilerBaselineRunner(CompilerTestType.OH); case "fourslash": return new FourSlashRunner(FourSlash.FourSlashTestType.Native); case "fourslash-oh": return new FourSlashRunner(FourSlash.FourSlashTestType.OH); case "fourslash-shims": return new FourSlashRunner(FourSlash.FourSlashTestType.Shims); case "fourslash-shims-pp": return new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess); case "fourslash-server": return new FourSlashRunner(FourSlash.FourSlashTestType.Server); case "project": return new project.ProjectRunner(); case "rwc": return new RWC.RWCRunner(); case "test262": return new Test262BaselineRunner(); case "user": return new UserCodeRunner(); case "dt": return new DefinitelyTypedRunner(); case "docker": return new DockerfileRunner(); } return ts.Debug.fail(`Unknown runner kind ${kind}`); } // users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options const mytestconfigFileName = "mytest.config"; const testconfigFileName = "test.config"; const customConfig = tryGetConfig(IO.args()); const testConfigContent = customConfig && IO.fileExists(customConfig) ? IO.readFile(customConfig)! : IO.fileExists(mytestconfigFileName) ? IO.readFile(mytestconfigFileName)! : IO.fileExists(testconfigFileName) ? IO.readFile(testconfigFileName)! : ""; export let taskConfigsFolder: string; export let workerCount: number; export let runUnitTests: boolean | undefined; export let stackTraceLimit: number | "full" | undefined; export let noColors = false; export let keepFailed = false; export interface TestConfig { light?: boolean; taskConfigsFolder?: string; listenForWork?: boolean; workerCount?: number; stackTraceLimit?: number | "full"; test?: string[]; runners?: string[]; runUnitTests?: boolean; noColors?: boolean; timeout?: number; keepFailed?: boolean; shardId?: number; shards?: number; } export interface TaskSet { runner: TestRunnerKind; files: string[]; } export let configOption: string; export let globalTimeout: number; function handleTestConfig() { if (testConfigContent !== "") { const testConfig = JSON.parse(testConfigContent); if (testConfig.light) { setLightMode(true); } if (testConfig.timeout) { globalTimeout = testConfig.timeout; } runUnitTests = testConfig.runUnitTests; if (testConfig.workerCount) { workerCount = +testConfig.workerCount; } if (testConfig.taskConfigsFolder) { taskConfigsFolder = testConfig.taskConfigsFolder; } if (testConfig.noColors !== undefined) { noColors = testConfig.noColors; } if (testConfig.keepFailed) { keepFailed = true; } if (testConfig.shardId) { setShardId(testConfig.shardId); } if (testConfig.shards) { setShards(testConfig.shards); } if (testConfig.stackTraceLimit === "full") { (Error).stackTraceLimit = Infinity; stackTraceLimit = testConfig.stackTraceLimit; } else if ((+testConfig.stackTraceLimit! | 0) > 0) { (Error).stackTraceLimit = +testConfig.stackTraceLimit! | 0; stackTraceLimit = +testConfig.stackTraceLimit! | 0; } if (testConfig.listenForWork) { return true; } const runnerConfig = testConfig.runners || testConfig.test; if (runnerConfig && runnerConfig.length > 0) { if (testConfig.runners) { runUnitTests = runnerConfig.indexOf("unittest") !== -1; } for (const option of runnerConfig) { if (!option) { continue; } if (!configOption) { configOption = option; } else { configOption += "+" + option; } switch (option) { case "compiler": runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); break; case "compiler-oh": runners.push(new CompilerBaselineRunner(CompilerTestType.OH)); break; case "conformance": runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); break; case "project": runners.push(new project.ProjectRunner()); break; case "fourslash": runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Native)); break; case "fourslash-oh": runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.OH)); break; case "fourslash-shims": runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Shims)); break; case "fourslash-shims-pp": runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess)); break; case "fourslash-server": runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server)); break; case "fourslash-generated": runners.push(new GeneratedFourslashRunner(FourSlash.FourSlashTestType.Native)); break; case "rwc": runners.push(new RWC.RWCRunner()); break; case "test262": runners.push(new Test262BaselineRunner()); break; case "user": runners.push(new UserCodeRunner()); break; case "dt": runners.push(new DefinitelyTypedRunner()); break; case "docker": runners.push(new DockerfileRunner()); break; } } } } if (runners.length === 0) { // compiler runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); runners.push(new CompilerBaselineRunner(CompilerTestType.OH)); runners.push(new project.ProjectRunner()); // language services runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Native)); runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.OH)); runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Shims)); runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess)); runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server)); // runners.push(new GeneratedFourslashRunner()); // CRON-only tests if (process.env.TRAVIS_EVENT_TYPE === "cron") { runners.push(new UserCodeRunner()); runners.push(new DockerfileRunner()); } } if (runUnitTests === undefined) { runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for } return false; } function beginTests() { ts.Debug.loggingHost = { log(_level, s) { console.log(s || ""); } }; if (ts.Debug.isDebugging) { ts.Debug.enableDebugInfo(); } // run tests in en-US by default. let savedUILocale: string | undefined; beforeEach(() => { savedUILocale = ts.getUILocale(); ts.setUILocale("en-US"); }); afterEach(() => ts.setUILocale(savedUILocale)); runTests(runners); if (!runUnitTests) { // patch `describe` to skip unit tests (global as any).describe = ts.noop; } } export let isWorker: boolean; function startTestEnvironment() { isWorker = handleTestConfig(); if (isWorker) { return Parallel.Worker.start(); } else if (taskConfigsFolder && workerCount && workerCount > 1) { return Parallel.Host.start(); } beginTests(); } startTestEnvironment(); }