1namespace Harness { 2 const fs: typeof import("fs") = require("fs"); 3 const path: typeof import("path") = require("path"); 4 const del: typeof import("del") = require("del"); 5 6 interface ExecResult { 7 stdout: Buffer; 8 stderr: Buffer; 9 status: number | null; 10 } 11 12 interface UserConfig { 13 types: string[]; 14 cloneUrl: string; 15 path?: string; 16 } 17 18 abstract class ExternalCompileRunnerBase extends RunnerBase { 19 abstract testDir: string; 20 abstract report(result: ExecResult, cwd: string): string | null; 21 enumerateTestFiles() { 22 return IO.getDirectories(this.testDir); 23 } 24 /** Setup the runner's tests so that they are ready to be executed by the harness 25 * The first test should be a describe/it block that sets up the harness's compiler instance appropriately 26 */ 27 initializeTests(): void { 28 // Read in and evaluate the test list 29 const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); 30 31 // eslint-disable-next-line @typescript-eslint/no-this-alias 32 const cls = this; 33 describe(`${this.kind()} code samples`, function (this: Mocha.Suite) { 34 this.timeout(600_000); // 10 minutes 35 for (const test of testList) { 36 cls.runTest(typeof test === "string" ? test : test.file); 37 } 38 }); 39 } 40 private runTest(directoryName: string) { 41 // eslint-disable-next-line @typescript-eslint/no-this-alias 42 const cls = this; 43 const timeout = 600_000; // 10 minutes 44 describe(directoryName, function (this: Mocha.Suite) { 45 this.timeout(timeout); 46 const cp: typeof import("child_process") = require("child_process"); 47 48 it("should build successfully", () => { 49 let cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directoryName); 50 const originalCwd = cwd; 51 const stdio = isWorker ? "pipe" : "inherit"; 52 let types: string[] | undefined; 53 if (fs.existsSync(path.join(cwd, "test.json"))) { 54 const config = JSON.parse(fs.readFileSync(path.join(cwd, "test.json"), { encoding: "utf8" })) as UserConfig; 55 ts.Debug.assert(!!config.types, "Bad format from test.json: Types field must be present."); 56 ts.Debug.assert(!!config.cloneUrl, "Bad format from test.json: cloneUrl field must be present."); 57 const submoduleDir = path.join(cwd, directoryName); 58 if (!fs.existsSync(submoduleDir)) { 59 exec("git", ["--work-tree", submoduleDir, "clone", config.cloneUrl, path.join(submoduleDir, ".git")], { cwd }); 60 } 61 else { 62 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "reset", "HEAD", "--hard"], { cwd: submoduleDir }); 63 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "clean", "-f"], { cwd: submoduleDir }); 64 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "pull", "-f"], { cwd: submoduleDir }); 65 } 66 67 types = config.types; 68 69 cwd = config.path ? path.join(cwd, config.path) : submoduleDir; 70 } 71 const npmVersionText = exec("npm", ["--version"], { cwd, stdio: "pipe" })?.trim(); 72 const npmVersion = npmVersionText ? ts.Version.tryParse(npmVersionText.trim()) : undefined; 73 const isV7OrLater = !!npmVersion && npmVersion.major >= 7; 74 if (fs.existsSync(path.join(cwd, "package.json"))) { 75 if (fs.existsSync(path.join(cwd, "package-lock.json"))) { 76 fs.unlinkSync(path.join(cwd, "package-lock.json")); 77 } 78 if (fs.existsSync(path.join(cwd, "node_modules"))) { 79 del.sync(path.join(cwd, "node_modules"), { force: true }); 80 } 81 exec("npm", ["i", "--ignore-scripts", ...(isV7OrLater ? ["--legacy-peer-deps"] : [])], { cwd, timeout: timeout / 2 }); // NPM shouldn't take the entire timeout - if it takes a long time, it should be terminated and we should log the failure 82 } 83 const args = [path.join(IO.getWorkspaceRoot(), "built/local/tsc.js")]; 84 if (types) { 85 args.push("--types", types.join(",")); 86 // Also actually install those types (for, eg, the js projects which need node) 87 if (types.length) { 88 exec("npm", ["i", ...types.map(t => `@types/${t}`), "--no-save", "--ignore-scripts", ...(isV7OrLater ? ["--legacy-peer-deps"] : [])], { cwd: originalCwd, timeout: timeout / 2 }); // NPM shouldn't take the entire timeout - if it takes a long time, it should be terminated and we should log the failure 89 } 90 } 91 args.push("--noEmit"); 92 Baseline.runBaseline(`${cls.kind()}/${directoryName}.log`, cls.report(cp.spawnSync(`node`, args, { cwd, timeout, shell: true }), cwd)); 93 94 function exec(command: string, args: string[], options: { cwd: string, timeout?: number, stdio?: import("child_process").StdioOptions }): string | undefined { 95 const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { shell: true, stdio, ...options }); 96 if (res.status !== 0) { 97 throw new Error(`${command} ${args.join(" ")} for ${directoryName} failed: ${res.stdout && res.stdout.toString()}`); 98 } 99 return options.stdio === "pipe" ? res.stdout.toString("utf8") : undefined; 100 } 101 }); 102 }); 103 } 104 } 105 106 export class UserCodeRunner extends ExternalCompileRunnerBase { 107 readonly testDir = "tests/cases/user/"; 108 kind(): TestRunnerKind { 109 return "user"; 110 } 111 report(result: ExecResult) { 112 // eslint-disable-next-line no-null/no-null 113 return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} 114Standard output: 115${sortErrors(stripAbsoluteImportPaths(result.stdout.toString().replace(/\r\n/g, "\n")))} 116 117 118Standard error: 119${stripAbsoluteImportPaths(result.stderr.toString().replace(/\r\n/g, "\n"))}`; 120 } 121 } 122 123 export class DockerfileRunner extends ExternalCompileRunnerBase { 124 readonly testDir = "tests/cases/docker/"; 125 kind(): TestRunnerKind { 126 return "docker"; 127 } 128 initializeTests(): void { 129 // Read in and evaluate the test list 130 const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); 131 132 // eslint-disable-next-line @typescript-eslint/no-this-alias 133 const cls = this; 134 describe(`${this.kind()} code samples`, function (this: Mocha.Suite) { 135 this.timeout(cls.timeout); // 20 minutes 136 before(() => { 137 cls.exec("docker", ["build", ".", "-t", "typescript/typescript"], { cwd: IO.getWorkspaceRoot() }); // cached because workspace is hashed to determine cacheability 138 }); 139 for (const test of testList) { 140 const directory = typeof test === "string" ? test : test.file; 141 const cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directory); 142 it(`should build ${directory} successfully`, () => { 143 const imageName = `tstest/${directory}`; 144 cls.exec("docker", ["build", "--no-cache", ".", "-t", imageName], { cwd }); // --no-cache so the latest version of the repos referenced is always fetched 145 const cp: typeof import("child_process") = require("child_process"); 146 Baseline.runBaseline(`${cls.kind()}/${directory}.log`, cls.report(cp.spawnSync(`docker`, ["run", imageName], { cwd, timeout: cls.timeout, shell: true }))); 147 }); 148 } 149 }); 150 } 151 152 private timeout = 1_200_000; // 20 minutes; 153 private exec(command: string, args: string[], options: { cwd: string }): void { 154 const cp: typeof import("child_process") = require("child_process"); 155 const stdio = isWorker ? "pipe" : "inherit"; 156 const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { timeout: this.timeout, shell: true, stdio, ...options }); 157 if (res.status !== 0) { 158 throw new Error(`${command} ${args.join(" ")} for ${options.cwd} failed: ${res.stdout && res.stdout.toString()}`); 159 } 160 } 161 report(result: ExecResult) { 162 // eslint-disable-next-line no-null/no-null 163 return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} 164Standard output: 165${sanitizeDockerfileOutput(result.stdout.toString())} 166 167 168Standard error: 169${sanitizeDockerfileOutput(result.stderr.toString())}`; 170 } 171 } 172 173 function sanitizeDockerfileOutput(result: string): string { 174 return [ 175 normalizeNewlines, 176 stripANSIEscapes, 177 stripRushStageNumbers, 178 stripWebpackHash, 179 sanitizeVersionSpecifiers, 180 sanitizeTimestamps, 181 sanitizeSizes, 182 sanitizeUnimportantGulpOutput, 183 stripAbsoluteImportPaths, 184 ].reduce((result, f) => f(result), result); 185 } 186 187 function normalizeNewlines(result: string): string { 188 return result.replace(/\r\n/g, "\n"); 189 } 190 191 function stripANSIEscapes(result: string): string { 192 return result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, ""); 193 } 194 195 function stripRushStageNumbers(result: string): string { 196 return result.replace(/\d+ of \d+:/g, "XX of XX:"); 197 } 198 199 function stripWebpackHash(result: string): string { 200 return result.replace(/Hash: \w+/g, "Hash: [redacted]"); 201 } 202 203 function sanitizeSizes(result: string): string { 204 return result.replace(/\d+(\.\d+)? ((Ki|M)B|bytes)/g, "X KiB"); 205 } 206 207 /** 208 * Gulp's output order within a `parallel` block is nondeterministic (and there's no way to configure it to execute in series), 209 * so we purge as much of the gulp output as we can 210 */ 211 function sanitizeUnimportantGulpOutput(result: string): string { 212 return result.replace(/^.*(\] (Starting)|(Finished)).*$/gm, "") // "gulp" task start/end messages (nondeterministic order) 213 .replace(/^.*(\] . (finished)|(started)).*$/gm, "") // "just" task start/end messages (nondeterministic order) 214 .replace(/^.*\] Respawned to PID: \d+.*$/gm, "") // PID of child is OS and system-load dependent (likely stableish in a container but still dangerous) 215 .replace(/\n+/g, "\n") 216 .replace(/\/tmp\/yarn--.*?\/node/g, ""); 217 } 218 219 function sanitizeTimestamps(result: string): string { 220 return result.replace(/\[\d?\d:\d\d:\d\d (A|P)M\]/g, "[XX:XX:XX XM]") 221 .replace(/\[\d?\d:\d\d:\d\d\]/g, "[XX:XX:XX]") 222 .replace(/\/\d+-\d+-[\d_TZ]+-debug.log/g, "\/XXXX-XX-XXXXXXXXX-debug.log") 223 .replace(/\d+\/\d+\/\d+ \d+:\d+:\d+ (AM|PM)/g, "XX/XX/XX XX:XX:XX XM") 224 .replace(/\d+(\.\d+)? sec(onds?)?/g, "? seconds") 225 .replace(/\d+(\.\d+)? min(utes?)?/g, "") 226 .replace(/\d+(\.\d+)? ?m?s/g, "?s") 227 .replace(/ \(\?s\)/g, ""); 228 } 229 230 function sanitizeVersionSpecifiers(result: string): string { 231 return result 232 .replace(/\d+.\d+.\d+-insiders.\d\d\d\d\d\d\d\d/g, "X.X.X-insiders.xxxxxxxx") 233 .replace(/Rush Multi-Project Build Tool (\d+)\.\d+\.\d+/g, "Rush Multi-Project Build Tool $1.X.X") 234 .replace(/([@v\()])\d+\.\d+\.\d+/g, "$1X.X.X") 235 .replace(/webpack (\d+)\.\d+\.\d+/g, "webpack $1.X.X") 236 .replace(/Webpack version: (\d+)\.\d+\.\d+/g, "Webpack version: $1.X.X"); 237 } 238 239 /** 240 * Import types and some other error messages use absolute paths in errors as they have no context to be written relative to; 241 * This is problematic for error baselines, so we grep for them and strip them out. 242 */ 243 function stripAbsoluteImportPaths(result: string) { 244 const workspaceRegexp = new RegExp(IO.getWorkspaceRoot().replace(/\\/g, "\\\\"), "g"); 245 return result 246 .replace(/import\(".*?\/tests\/cases\/user\//g, `import("/`) 247 .replace(/Module '".*?\/tests\/cases\/user\//g, `Module '"/`) 248 .replace(workspaceRegexp, "../../.."); 249 } 250 251 function sortErrors(result: string) { 252 return ts.flatten(splitBy(result.split("\n"), s => /^\S+/.test(s)).sort(compareErrorStrings)).join("\n"); 253 } 254 255 const errorRegexp = /^(.+\.[tj]sx?)\((\d+),(\d+)\)(: error TS.*)/; 256 function compareErrorStrings(a: string[], b: string[]) { 257 ts.Debug.assertGreaterThanOrEqual(a.length, 1); 258 ts.Debug.assertGreaterThanOrEqual(b.length, 1); 259 const matchA = a[0].match(errorRegexp); 260 if (!matchA) { 261 return -1; 262 } 263 const matchB = b[0].match(errorRegexp); 264 if (!matchB) { 265 return 1; 266 } 267 const [, errorFileA, lineNumberStringA, columnNumberStringA, remainderA] = matchA; 268 const [, errorFileB, lineNumberStringB, columnNumberStringB, remainderB] = matchB; 269 return ts.comparePathsCaseSensitive(errorFileA, errorFileB) || 270 ts.compareValues(parseInt(lineNumberStringA), parseInt(lineNumberStringB)) || 271 ts.compareValues(parseInt(columnNumberStringA), parseInt(columnNumberStringB)) || 272 ts.compareStringsCaseSensitive(remainderA, remainderB) || 273 ts.compareStringsCaseSensitive(a.slice(1).join("\n"), b.slice(1).join("\n")); 274 } 275 276 export class DefinitelyTypedRunner extends ExternalCompileRunnerBase { 277 readonly testDir = "../DefinitelyTyped/types/"; 278 workingDirectory = this.testDir; 279 kind(): TestRunnerKind { 280 return "dt"; 281 } 282 report(result: ExecResult, cwd: string) { 283 const stdout = removeExpectedErrors(result.stdout.toString(), cwd); 284 const stderr = result.stderr.toString(); 285 286 // eslint-disable-next-line no-null/no-null 287 return !stdout.length && !stderr.length ? null : `Exit Code: ${result.status} 288Standard output: 289${stdout.replace(/\r\n/g, "\n")} 290 291 292Standard error: 293${stderr.replace(/\r\n/g, "\n")}`; 294 } 295 } 296 297 function removeExpectedErrors(errors: string, cwd: string): string { 298 return ts.flatten(splitBy(errors.split("\n"), s => /^\S+/.test(s)).filter(isUnexpectedError(cwd))).join("\n"); 299 } 300 /** 301 * Returns true if the line that caused the error contains '$ExpectError', 302 * or if the line before that one contains '$ExpectError'. 303 * '$ExpectError' is a marker used in Definitely Typed tests, 304 * meaning that the error should not contribute toward our error baslines. 305 */ 306 function isUnexpectedError(cwd: string) { 307 return (error: string[]) => { 308 ts.Debug.assertGreaterThanOrEqual(error.length, 1); 309 const match = error[0].match(/(.+\.tsx?)\((\d+),\d+\): error TS/); 310 if (!match) { 311 return true; 312 } 313 const [, errorFile, lineNumberString] = match; 314 const lines = fs.readFileSync(path.join(cwd, errorFile), { encoding: "utf8" }).split("\n"); 315 const lineNumber = parseInt(lineNumberString) - 1; 316 ts.Debug.assertGreaterThanOrEqual(lineNumber, 0); 317 ts.Debug.assertLessThan(lineNumber, lines.length); 318 const previousLine = lineNumber - 1 > 0 ? lines[lineNumber - 1] : ""; 319 return !ts.stringContains(lines[lineNumber], "$ExpectError") && !ts.stringContains(previousLine, "$ExpectError"); 320 }; 321 } 322 /** 323 * Split an array into multiple arrays whenever `isStart` returns true. 324 * @example 325 * splitBy([1,2,3,4,5,6], isOdd) 326 * ==> [[1, 2], [3, 4], [5, 6]] 327 * where 328 * const isOdd = n => !!(n % 2) 329 */ 330 function splitBy<T>(xs: T[], isStart: (x: T) => boolean): T[][] { 331 const result = []; 332 let group: T[] = []; 333 for (const x of xs) { 334 if (isStart(x)) { 335 if (group.length) { 336 result.push(group); 337 } 338 group = [x]; 339 } 340 else { 341 group.push(x); 342 } 343 } 344 if (group.length) { 345 result.push(group); 346 } 347 return result; 348 } 349} 350