1import * as del from "del"; 2import * as fs from "fs"; 3import * as path from "path"; 4 5import * as ts from "./_namespaces/ts"; 6import { Baseline, IO, isWorker, RunnerBase, TestRunnerKind } from "./_namespaces/Harness"; 7 8 9interface ExecResult { 10 stdout: Buffer; 11 stderr: Buffer; 12 status: number | null; 13} 14 15interface UserConfig { 16 types: string[]; 17 cloneUrl: string; 18 branch?: string; 19 path?: string; 20} 21 22abstract class ExternalCompileRunnerBase extends RunnerBase { 23 abstract testDir: string; 24 abstract report(result: ExecResult): string | null; 25 enumerateTestFiles() { 26 return IO.getDirectories(this.testDir); 27 } 28 /** Setup the runner's tests so that they are ready to be executed by the harness 29 * The first test should be a describe/it block that sets up the harness's compiler instance appropriately 30 */ 31 initializeTests(): void { 32 // Read in and evaluate the test list 33 const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); 34 35 // eslint-disable-next-line @typescript-eslint/no-this-alias 36 const cls = this; 37 describe(`${this.kind()} code samples`, function (this: Mocha.Suite) { 38 this.timeout(600_000); // 10 minutes 39 for (const test of testList) { 40 cls.runTest(typeof test === "string" ? test : test.file); 41 } 42 }); 43 } 44 private runTest(directoryName: string) { 45 // eslint-disable-next-line @typescript-eslint/no-this-alias 46 const cls = this; 47 const timeout = 600_000; // 10 minutes 48 describe(directoryName, function (this: Mocha.Suite) { 49 this.timeout(timeout); 50 const cp: typeof import("child_process") = require("child_process"); 51 52 it("should build successfully", () => { 53 let cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directoryName); 54 const originalCwd = cwd; 55 const stdio = isWorker ? "pipe" : "inherit"; 56 let types: string[] | undefined; 57 if (fs.existsSync(path.join(cwd, "test.json"))) { 58 const config = JSON.parse(fs.readFileSync(path.join(cwd, "test.json"), { encoding: "utf8" })) as UserConfig; 59 ts.Debug.assert(!!config.types, "Bad format from test.json: Types field must be present."); 60 ts.Debug.assert(!!config.cloneUrl, "Bad format from test.json: cloneUrl field must be present."); 61 const submoduleDir = path.join(cwd, directoryName); 62 if (!fs.existsSync(submoduleDir)) { 63 exec("git", ["--work-tree", submoduleDir, "clone", "-b", config.branch || "master", config.cloneUrl, path.join(submoduleDir, ".git")], { cwd }); 64 } 65 else { 66 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "checkout", config.branch || "master"], { cwd: submoduleDir }); 67 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "reset", "HEAD", "--hard"], { cwd: submoduleDir }); 68 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "clean", "-f"], { cwd: submoduleDir }); 69 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "pull", "-f"], { cwd: submoduleDir }); 70 } 71 72 types = config.types; 73 74 cwd = config.path ? path.join(cwd, config.path) : submoduleDir; 75 } 76 const npmVersionText = exec("npm", ["--version"], { cwd, stdio: "pipe" })?.trim(); 77 const npmVersion = npmVersionText ? ts.Version.tryParse(npmVersionText.trim()) : undefined; 78 const isV7OrLater = !!npmVersion && npmVersion.major >= 7; 79 if (fs.existsSync(path.join(cwd, "package.json"))) { 80 if (fs.existsSync(path.join(cwd, "package-lock.json"))) { 81 fs.unlinkSync(path.join(cwd, "package-lock.json")); 82 } 83 if (fs.existsSync(path.join(cwd, "node_modules"))) { 84 del.sync(path.join(cwd, "node_modules"), { force: true }); 85 } 86 exec("npm", ["i", "--ignore-scripts", ...(isV7OrLater ? ["--legacy-peer-deps"] : [])], { cwd, timeout: timeout / 2 }); // NPM shouldn't take the entire timeout - if it takes a long time, it should be terminated and we should log the failure 87 } 88 const args = [path.join(IO.getWorkspaceRoot(), "built/local/tsc.js")]; 89 if (types) { 90 args.push("--types", types.join(",")); 91 // Also actually install those types (for, eg, the js projects which need node) 92 if (types.length) { 93 exec("npm", ["i", ...types.map(t => `@types/${t}`), "--no-save", "--ignore-scripts", ...(isV7OrLater ? ["--legacy-peer-deps"] : [])], { cwd: originalCwd, timeout: timeout / 2 }); // NPM shouldn't take the entire timeout - if it takes a long time, it should be terminated and we should log the failure 94 } 95 } 96 args.push("--noEmit"); 97 Baseline.runBaseline(`${cls.kind()}/${directoryName}.log`, cls.report(cp.spawnSync(`node`, args, { cwd, timeout, shell: true }))); 98 99 function exec(command: string, args: string[], options: { cwd: string, timeout?: number, stdio?: import("child_process").StdioOptions }): string | undefined { 100 const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { shell: true, stdio, ...options }); 101 if (res.status !== 0) { 102 throw new Error(`${command} ${args.join(" ")} for ${directoryName} failed: ${res.stdout && res.stdout.toString()}`); 103 } 104 return options.stdio === "pipe" ? res.stdout.toString("utf8") : undefined; 105 } 106 }); 107 }); 108 } 109} 110 111export class UserCodeRunner extends ExternalCompileRunnerBase { 112 readonly testDir = "tests/cases/user/"; 113 kind(): TestRunnerKind { 114 return "user"; 115 } 116 report(result: ExecResult) { 117 // eslint-disable-next-line no-null/no-null 118 return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} 119Standard output: 120${sortErrors(stripAbsoluteImportPaths(result.stdout.toString().replace(/\r\n/g, "\n")))} 121 122 123Standard error: 124${stripAbsoluteImportPaths(result.stderr.toString().replace(/\r\n/g, "\n"))}`; 125 } 126} 127 128export class DockerfileRunner extends ExternalCompileRunnerBase { 129 readonly testDir = "tests/cases/docker/"; 130 kind(): TestRunnerKind { 131 return "docker"; 132 } 133 initializeTests(): void { 134 // Read in and evaluate the test list 135 const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); 136 137 // eslint-disable-next-line @typescript-eslint/no-this-alias 138 const cls = this; 139 describe(`${this.kind()} code samples`, function (this: Mocha.Suite) { 140 this.timeout(cls.timeout); // 20 minutes 141 before(() => { 142 cls.exec("docker", ["build", ".", "-t", "typescript/typescript"], { cwd: IO.getWorkspaceRoot() }); // cached because workspace is hashed to determine cacheability 143 }); 144 for (const test of testList) { 145 const directory = typeof test === "string" ? test : test.file; 146 const cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directory); 147 it(`should build ${directory} successfully`, () => { 148 const imageName = `tstest/${directory}`; 149 cls.exec("docker", ["build", "--no-cache", ".", "-t", imageName], { cwd }); // --no-cache so the latest version of the repos referenced is always fetched 150 const cp: typeof import("child_process") = require("child_process"); 151 Baseline.runBaseline(`${cls.kind()}/${directory}.log`, cls.report(cp.spawnSync(`docker`, ["run", imageName], { cwd, timeout: cls.timeout, shell: true }))); 152 }); 153 } 154 }); 155 } 156 157 private timeout = 1_200_000; // 20 minutes; 158 private exec(command: string, args: string[], options: { cwd: string }): void { 159 const cp: typeof import("child_process") = require("child_process"); 160 const stdio = isWorker ? "pipe" : "inherit"; 161 const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { timeout: this.timeout, shell: true, stdio, ...options }); 162 if (res.status !== 0) { 163 throw new Error(`${command} ${args.join(" ")} for ${options.cwd} failed: ${res.stdout && res.stdout.toString()}`); 164 } 165 } 166 report(result: ExecResult) { 167 // eslint-disable-next-line no-null/no-null 168 return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} 169Standard output: 170${sanitizeDockerfileOutput(result.stdout.toString())} 171 172 173Standard error: 174${sanitizeDockerfileOutput(result.stderr.toString())}`; 175 } 176} 177 178function sanitizeDockerfileOutput(result: string): string { 179 return [ 180 normalizeNewlines, 181 stripANSIEscapes, 182 stripRushStageNumbers, 183 stripWebpackHash, 184 sanitizeVersionSpecifiers, 185 sanitizeTimestamps, 186 sanitizeSizes, 187 sanitizeUnimportantGulpOutput, 188 stripAbsoluteImportPaths, 189 ].reduce((result, f) => f(result), result); 190} 191 192function normalizeNewlines(result: string): string { 193 return result.replace(/\r\n/g, "\n"); 194} 195 196function stripANSIEscapes(result: string): string { 197 return result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, ""); 198} 199 200function stripRushStageNumbers(result: string): string { 201 return result.replace(/\d+ of \d+:/g, "XX of XX:"); 202} 203 204function stripWebpackHash(result: string): string { 205 return result.replace(/Hash: \w+/g, "Hash: [redacted]"); 206} 207 208function sanitizeSizes(result: string): string { 209 return result.replace(/\d+(\.\d+)? ((Ki|M)B|bytes)/g, "X KiB"); 210} 211 212/** 213 * Gulp's output order within a `parallel` block is nondeterministic (and there's no way to configure it to execute in series), 214 * so we purge as much of the gulp output as we can 215 */ 216function sanitizeUnimportantGulpOutput(result: string): string { 217 return result.replace(/^.*(\] (Starting)|(Finished)).*$/gm, "") // "gulp" task start/end messages (nondeterministic order) 218 .replace(/^.*(\] . (finished)|(started)).*$/gm, "") // "just" task start/end messages (nondeterministic order) 219 .replace(/^.*\] Respawned to PID: \d+.*$/gm, "") // PID of child is OS and system-load dependent (likely stableish in a container but still dangerous) 220 .replace(/\n+/g, "\n") 221 .replace(/\/tmp\/yarn--.*?\/node/g, ""); 222} 223 224function sanitizeTimestamps(result: string): string { 225 return result.replace(/\[\d?\d:\d\d:\d\d (A|P)M\]/g, "[XX:XX:XX XM]") 226 .replace(/\[\d?\d:\d\d:\d\d\]/g, "[XX:XX:XX]") 227 .replace(/\/\d+-\d+-[\d_TZ]+-debug.log/g, "\/XXXX-XX-XXXXXXXXX-debug.log") 228 .replace(/\d+\/\d+\/\d+ \d+:\d+:\d+ (AM|PM)/g, "XX/XX/XX XX:XX:XX XM") 229 .replace(/\d+(\.\d+)? sec(onds?)?/g, "? seconds") 230 .replace(/\d+(\.\d+)? min(utes?)?/g, "") 231 .replace(/\d+(\.\d+)? ?m?s/g, "?s") 232 .replace(/ \(\?s\)/g, ""); 233} 234 235function sanitizeVersionSpecifiers(result: string): string { 236 return result 237 .replace(/\d+.\d+.\d+-insiders.\d\d\d\d\d\d\d\d/g, "X.X.X-insiders.xxxxxxxx") 238 .replace(/Rush Multi-Project Build Tool (\d+)\.\d+\.\d+/g, "Rush Multi-Project Build Tool $1.X.X") 239 .replace(/([@v\()])\d+\.\d+\.\d+/g, "$1X.X.X") 240 .replace(/webpack (\d+)\.\d+\.\d+/g, "webpack $1.X.X") 241 .replace(/Webpack version: (\d+)\.\d+\.\d+/g, "Webpack version: $1.X.X"); 242} 243 244/** 245 * Import types and some other error messages use absolute paths in errors as they have no context to be written relative to; 246 * This is problematic for error baselines, so we grep for them and strip them out. 247 */ 248function stripAbsoluteImportPaths(result: string) { 249 const workspaceRegexp = new RegExp(IO.getWorkspaceRoot().replace(/\\/g, "\\\\"), "g"); 250 return result 251 .replace(/import\(".*?\/tests\/cases\/user\//g, `import("/`) 252 .replace(/Module '".*?\/tests\/cases\/user\//g, `Module '"/`) 253 .replace(workspaceRegexp, "../../.."); 254} 255 256function sortErrors(result: string) { 257 return ts.flatten(splitBy(result.split("\n"), s => /^\S+/.test(s)).sort(compareErrorStrings)).join("\n"); 258} 259 260const errorRegexp = /^(.+\.[tj]sx?)\((\d+),(\d+)\)(: error TS.*)/; 261function compareErrorStrings(a: string[], b: string[]) { 262 ts.Debug.assertGreaterThanOrEqual(a.length, 1); 263 ts.Debug.assertGreaterThanOrEqual(b.length, 1); 264 const matchA = a[0].match(errorRegexp); 265 if (!matchA) { 266 return -1; 267 } 268 const matchB = b[0].match(errorRegexp); 269 if (!matchB) { 270 return 1; 271 } 272 const [, errorFileA, lineNumberStringA, columnNumberStringA, remainderA] = matchA; 273 const [, errorFileB, lineNumberStringB, columnNumberStringB, remainderB] = matchB; 274 return ts.comparePathsCaseSensitive(errorFileA, errorFileB) || 275 ts.compareValues(parseInt(lineNumberStringA), parseInt(lineNumberStringB)) || 276 ts.compareValues(parseInt(columnNumberStringA), parseInt(columnNumberStringB)) || 277 ts.compareStringsCaseSensitive(remainderA, remainderB) || 278 ts.compareStringsCaseSensitive(a.slice(1).join("\n"), b.slice(1).join("\n")); 279} 280 281export class DefinitelyTypedRunner extends ExternalCompileRunnerBase { 282 readonly testDir = "../DefinitelyTyped/types/"; 283 workingDirectory = this.testDir; 284 kind(): TestRunnerKind { 285 return "dt"; 286 } 287 report(result: ExecResult) { 288 // eslint-disable-next-line no-null/no-null 289 return !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} 290Standard output: 291${result.stdout.toString().replace(/\r\n/g, "\n")} 292 293 294Standard error: 295${result.stderr.toString().replace(/\r\n/g, "\n")}`; 296 } 297} 298 299/** 300 * Split an array into multiple arrays whenever `isStart` returns true. 301 * @example 302 * splitBy([1,2,3,4,5,6], isOdd) 303 * ==> [[1, 2], [3, 4], [5, 6]] 304 * where 305 * const isOdd = n => !!(n % 2) 306 */ 307function splitBy<T>(xs: T[], isStart: (x: T) => boolean): T[][] { 308 const result = []; 309 let group: T[] = []; 310 for (const x of xs) { 311 if (isStart(x)) { 312 if (group.length) { 313 result.push(group); 314 } 315 group = [x]; 316 } 317 else { 318 group.push(x); 319 } 320 } 321 if (group.length) { 322 result.push(group); 323 } 324 return result; 325} 326