1namespace Harness { 2 /* eslint-disable prefer-const */ 3 export let runners: RunnerBase[] = []; 4 export let iterations = 1; 5 /* eslint-enable prefer-const */ 6 7 function runTests(runners: RunnerBase[]) { 8 for (let i = iterations; i > 0; i--) { 9 const seen = new Map<string, string>(); 10 const dupes: [string, string][] = []; 11 for (const runner of runners) { 12 if (runner instanceof CompilerBaselineRunner || runner instanceof FourSlashRunner) { 13 for (const sf of runner.enumerateTestFiles()) { 14 const full = typeof sf === "string" ? sf : sf.file; 15 const base = vpath.basename(full).toLowerCase(); 16 // allow existing dupes in fourslash/shims and fourslash/server 17 if (seen.has(base) && !/fourslash\/(shim|server)/.test(full)) { 18 dupes.push([seen.get(base)!, full]); 19 } 20 else { 21 seen.set(base, full); 22 } 23 } 24 } 25 runner.initializeTests(); 26 } 27 if (dupes.length) { 28 throw new Error(`${dupes.length} Tests with duplicate baseline names: 29${JSON.stringify(dupes, undefined, 2)}`); 30 } 31 } 32 } 33 34 function tryGetConfig(args: string[]) { 35 const prefix = "--config="; 36 const configPath = ts.forEach(args, arg => arg.lastIndexOf(prefix, 0) === 0 && arg.substr(prefix.length)); 37 // strip leading and trailing quotes from the path (necessary on Windows since shell does not do it automatically) 38 return configPath && configPath.replace(/(^[\"'])|([\"']$)/g, ""); 39 } 40 41 export function createRunner(kind: TestRunnerKind): RunnerBase { 42 switch (kind) { 43 case "conformance": 44 return new CompilerBaselineRunner(CompilerTestType.Conformance); 45 case "compiler": 46 return new CompilerBaselineRunner(CompilerTestType.Regressions); 47 case "compiler-oh": 48 return new CompilerBaselineRunner(CompilerTestType.OH); 49 case "fourslash": 50 return new FourSlashRunner(FourSlash.FourSlashTestType.Native); 51 case "fourslash-oh": 52 return new FourSlashRunner(FourSlash.FourSlashTestType.OH); 53 case "fourslash-shims": 54 return new FourSlashRunner(FourSlash.FourSlashTestType.Shims); 55 case "fourslash-shims-pp": 56 return new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess); 57 case "fourslash-server": 58 return new FourSlashRunner(FourSlash.FourSlashTestType.Server); 59 case "project": 60 return new project.ProjectRunner(); 61 case "rwc": 62 return new RWC.RWCRunner(); 63 case "test262": 64 return new Test262BaselineRunner(); 65 case "user": 66 return new UserCodeRunner(); 67 case "dt": 68 return new DefinitelyTypedRunner(); 69 case "docker": 70 return new DockerfileRunner(); 71 } 72 return ts.Debug.fail(`Unknown runner kind ${kind}`); 73 } 74 75 // users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options 76 77 const mytestconfigFileName = "mytest.config"; 78 const testconfigFileName = "test.config"; 79 80 const customConfig = tryGetConfig(IO.args()); 81 const testConfigContent = 82 customConfig && IO.fileExists(customConfig) 83 ? IO.readFile(customConfig)! 84 : IO.fileExists(mytestconfigFileName) 85 ? IO.readFile(mytestconfigFileName)! 86 : IO.fileExists(testconfigFileName) ? IO.readFile(testconfigFileName)! : ""; 87 88 export let taskConfigsFolder: string; 89 export let workerCount: number; 90 export let runUnitTests: boolean | undefined; 91 export let stackTraceLimit: number | "full" | undefined; 92 export let noColors = false; 93 export let keepFailed = false; 94 95 export interface TestConfig { 96 light?: boolean; 97 taskConfigsFolder?: string; 98 listenForWork?: boolean; 99 workerCount?: number; 100 stackTraceLimit?: number | "full"; 101 test?: string[]; 102 runners?: string[]; 103 runUnitTests?: boolean; 104 noColors?: boolean; 105 timeout?: number; 106 keepFailed?: boolean; 107 shardId?: number; 108 shards?: number; 109 } 110 111 export interface TaskSet { 112 runner: TestRunnerKind; 113 files: string[]; 114 } 115 116 export let configOption: string; 117 export let globalTimeout: number; 118 function handleTestConfig() { 119 if (testConfigContent !== "") { 120 const testConfig = JSON.parse(testConfigContent) as TestConfig; 121 if (testConfig.light) { 122 setLightMode(true); 123 } 124 if (testConfig.timeout) { 125 globalTimeout = testConfig.timeout; 126 } 127 runUnitTests = testConfig.runUnitTests; 128 if (testConfig.workerCount) { 129 workerCount = +testConfig.workerCount; 130 } 131 if (testConfig.taskConfigsFolder) { 132 taskConfigsFolder = testConfig.taskConfigsFolder; 133 } 134 if (testConfig.noColors !== undefined) { 135 noColors = testConfig.noColors; 136 } 137 if (testConfig.keepFailed) { 138 keepFailed = true; 139 } 140 if (testConfig.shardId) { 141 setShardId(testConfig.shardId); 142 } 143 if (testConfig.shards) { 144 setShards(testConfig.shards); 145 } 146 147 if (testConfig.stackTraceLimit === "full") { 148 (Error as any).stackTraceLimit = Infinity; 149 stackTraceLimit = testConfig.stackTraceLimit; 150 } 151 else if ((+testConfig.stackTraceLimit! | 0) > 0) { 152 (Error as any).stackTraceLimit = +testConfig.stackTraceLimit! | 0; 153 stackTraceLimit = +testConfig.stackTraceLimit! | 0; 154 } 155 if (testConfig.listenForWork) { 156 return true; 157 } 158 159 const runnerConfig = testConfig.runners || testConfig.test; 160 if (runnerConfig && runnerConfig.length > 0) { 161 if (testConfig.runners) { 162 runUnitTests = runnerConfig.indexOf("unittest") !== -1; 163 } 164 for (const option of runnerConfig) { 165 if (!option) { 166 continue; 167 } 168 169 if (!configOption) { 170 configOption = option; 171 } 172 else { 173 configOption += "+" + option; 174 } 175 176 switch (option) { 177 case "compiler": 178 runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); 179 runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); 180 break; 181 case "compiler-oh": 182 runners.push(new CompilerBaselineRunner(CompilerTestType.OH)); 183 break; 184 case "conformance": 185 runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); 186 break; 187 case "project": 188 runners.push(new project.ProjectRunner()); 189 break; 190 case "fourslash": 191 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Native)); 192 break; 193 case "fourslash-oh": 194 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.OH)); 195 break; 196 case "fourslash-shims": 197 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Shims)); 198 break; 199 case "fourslash-shims-pp": 200 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess)); 201 break; 202 case "fourslash-server": 203 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server)); 204 break; 205 case "fourslash-generated": 206 runners.push(new GeneratedFourslashRunner(FourSlash.FourSlashTestType.Native)); 207 break; 208 case "rwc": 209 runners.push(new RWC.RWCRunner()); 210 break; 211 case "test262": 212 runners.push(new Test262BaselineRunner()); 213 break; 214 case "user": 215 runners.push(new UserCodeRunner()); 216 break; 217 case "dt": 218 runners.push(new DefinitelyTypedRunner()); 219 break; 220 case "docker": 221 runners.push(new DockerfileRunner()); 222 break; 223 } 224 } 225 } 226 } 227 228 if (runners.length === 0) { 229 // compiler 230 runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance)); 231 runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions)); 232 runners.push(new CompilerBaselineRunner(CompilerTestType.OH)); 233 runners.push(new project.ProjectRunner()); 234 235 // language services 236 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Native)); 237 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.OH)); 238 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Shims)); 239 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess)); 240 runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server)); 241 // runners.push(new GeneratedFourslashRunner()); 242 243 // CRON-only tests 244 if (process.env.TRAVIS_EVENT_TYPE === "cron") { 245 runners.push(new UserCodeRunner()); 246 runners.push(new DockerfileRunner()); 247 } 248 } 249 if (runUnitTests === undefined) { 250 runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for 251 } 252 return false; 253 } 254 255 function beginTests() { 256 ts.Debug.loggingHost = { 257 log(_level, s) { 258 console.log(s || ""); 259 } 260 }; 261 262 if (ts.Debug.isDebugging) { 263 ts.Debug.enableDebugInfo(); 264 } 265 266 // run tests in en-US by default. 267 let savedUILocale: string | undefined; 268 beforeEach(() => { 269 savedUILocale = ts.getUILocale(); 270 ts.setUILocale("en-US"); 271 }); 272 afterEach(() => ts.setUILocale(savedUILocale)); 273 274 runTests(runners); 275 276 if (!runUnitTests) { 277 // patch `describe` to skip unit tests 278 (global as any).describe = ts.noop; 279 } 280 } 281 282 export let isWorker: boolean; 283 function startTestEnvironment() { 284 isWorker = handleTestConfig(); 285 if (isWorker) { 286 return Parallel.Worker.start(); 287 } 288 else if (taskConfigsFolder && workerCount && workerCount > 1) { 289 return Parallel.Host.start(); 290 } 291 beginTests(); 292 } 293 294 startTestEnvironment(); 295} 296