src/testRunner/projectsRunner.ts

import * as ts from "./_namespaces/ts";
import * as documents from "./_namespaces/documents";
import * as Harness from "./_namespaces/Harness";
import * as fakes from "./_namespaces/fakes";
import * as vfs from "./_namespaces/vfs";
import * as vpath from "./_namespaces/vpath";
import * as Utils from "./_namespaces/Utils";

// Test case is json of below type in tests/cases/project/
interface ProjectRunnerTestCase {
    scenario: string;
    projectRoot: string; // project where it lives - this also is the current directory when compiling
    inputFiles: readonly string[]; // list of input files to be given to program
    resolveMapRoot?: boolean; // should we resolve this map root and give compiler the absolute disk path as map root?
    resolveSourceRoot?: boolean; // should we resolve this source root and give compiler the absolute disk path as map root?
    baselineCheck?: boolean; // Verify the baselines of output files, if this is false, we will write to output to the disk but there is no verification of baselines
    runTest?: boolean; // Run the resulting test
    bug?: string; // If there is any bug associated with this test case
}

interface ProjectRunnerTestCaseResolutionInfo extends ProjectRunnerTestCase {
    // Apart from actual test case the results of the resolution
    resolvedInputFiles: readonly string[]; // List of files that were asked to read by compiler
    emittedFiles: readonly string[]; // List of files that were emitted by the compiler
}

interface CompileProjectFilesResult {
    configFileSourceFiles: readonly ts.SourceFile[];
    moduleKind: ts.ModuleKind;
    program?: ts.Program;
    compilerOptions?: ts.CompilerOptions;
    errors: readonly ts.Diagnostic[];
    sourceMapData?: readonly ts.SourceMapEmitResult[];
}

interface BatchCompileProjectTestCaseResult extends CompileProjectFilesResult {
    outputFiles?: readonly documents.TextDocument[];
}

export class ProjectRunner extends Harness.RunnerBase {
    public enumerateTestFiles() {
        const all = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
        if (Harness.shards === 1) {
            return all;
        }
        return all.filter((_val, idx) => idx % Harness.shards === (Harness.shardId - 1));
    }

    public kind(): Harness.TestRunnerKind {
        return "project";
    }

    public initializeTests() {
        describe("projects tests", () => {
            const tests = this.tests.length === 0 ? this.enumerateTestFiles() : this.tests;
            for (const test of tests) {
                this.runProjectTestCase(typeof test === "string" ? test : test.file);
            }
        });
    }

    private runProjectTestCase(testCaseFileName: string) {
        for (const { name, payload } of ProjectTestCase.getConfigurations(testCaseFileName)) {
            describe("Compiling project for " + payload.testCase.scenario + ": testcase " + testCaseFileName + (name ? ` (${name})` : ``), () => {
                let projectTestCase: ProjectTestCase | undefined;
                before(() => {
                    projectTestCase = new ProjectTestCase(testCaseFileName, payload);
                });
                it(`Correct module resolution tracing for ${testCaseFileName}`, () => projectTestCase && projectTestCase.verifyResolution());
                it(`Correct errors for ${testCaseFileName}`, () => projectTestCase && projectTestCase.verifyDiagnostics());
                it(`Correct JS output for ${testCaseFileName}`, () => projectTestCase && projectTestCase.verifyJavaScriptOutput());
                // NOTE: This check was commented out in previous code. Leaving this here to eventually be restored if needed.
                // it(`Correct sourcemap content for ${testCaseFileName}`, () => projectTestCase && projectTestCase.verifySourceMapRecord());
                it(`Correct declarations for ${testCaseFileName}`, () => projectTestCase && projectTestCase.verifyDeclarations());
                after(() => {
                    projectTestCase = undefined;
                });
            });
        }
    }
}

class ProjectCompilerHost extends fakes.CompilerHost {
    private _testCase: ProjectRunnerTestCase & ts.CompilerOptions;
    private _projectParseConfigHost: ProjectParseConfigHost | undefined;

    constructor(sys: fakes.System | vfs.FileSystem, compilerOptions: ts.CompilerOptions, _testCaseJustName: string, testCase: ProjectRunnerTestCase & ts.CompilerOptions, _moduleKind: ts.ModuleKind) {
        super(sys, compilerOptions);
        this._testCase = testCase;
    }

    public get parseConfigHost(): fakes.ParseConfigHost {
        return this._projectParseConfigHost || (this._projectParseConfigHost = new ProjectParseConfigHost(this.sys, this._testCase));
    }

    public getDefaultLibFileName(_options: ts.CompilerOptions) {
        return vpath.resolve(this.getDefaultLibLocation(), "lib.es5.d.ts");
    }
}

class ProjectParseConfigHost extends fakes.ParseConfigHost {
    private _testCase: ProjectRunnerTestCase & ts.CompilerOptions;

    constructor(sys: fakes.System, testCase: ProjectRunnerTestCase & ts.CompilerOptions) {
        super(sys);
        this._testCase = testCase;
    }

    public readDirectory(path: string, extensions: string[], excludes: string[], includes: string[], depth: number): string[] {
        const result = super.readDirectory(path, extensions, excludes, includes, depth);
        const projectRoot = vpath.resolve(vfs.srcFolder, this._testCase.projectRoot);
        return result.map(item => vpath.relative(
            projectRoot,
            vpath.resolve(projectRoot, item),
            this.vfs.ignoreCase
        ));
    }
}

interface ProjectTestConfiguration {
    name: string;
    payload: ProjectTestPayload;
}

interface ProjectTestPayload {
    testCase: ProjectRunnerTestCase & ts.CompilerOptions;
    moduleKind: ts.ModuleKind;
    vfs: vfs.FileSystem;
}

class ProjectTestCase {
    private testCase: ProjectRunnerTestCase & ts.CompilerOptions;
    private testCaseJustName: string;
    private sys: fakes.System;
    private compilerOptions: ts.CompilerOptions;
    private compilerResult: BatchCompileProjectTestCaseResult;

    constructor(testCaseFileName: string, { testCase, moduleKind, vfs }: ProjectTestPayload) {
        this.testCase = testCase;
        this.testCaseJustName = testCaseFileName.replace(/^.*[\\\/]/, "").replace(/\.json/, "");
        this.compilerOptions = createCompilerOptions(testCase, moduleKind);
        this.sys = new fakes.System(vfs);

        let configFileName: string | undefined;
        let inputFiles = testCase.inputFiles;
        if (this.compilerOptions.project) {
            // Parse project
            configFileName = ts.normalizePath(ts.combinePaths(this.compilerOptions.project, "tsconfig.json"));
            assert(!inputFiles || inputFiles.length === 0, "cannot specify input files and project option together");
        }
        else if (!inputFiles || inputFiles.length === 0) {
            configFileName = ts.findConfigFile("", path => this.sys.fileExists(path));
        }

        let errors: ts.Diagnostic[] | undefined;
        const configFileSourceFiles: ts.SourceFile[] = [];
        if (configFileName) {
            const result = ts.readJsonConfigFile(configFileName, path => this.sys.readFile(path));
            configFileSourceFiles.push(result);
            const configParseHost = new ProjectParseConfigHost(this.sys, this.testCase);
            const configParseResult = ts.parseJsonSourceFileConfigFileContent(result, configParseHost, ts.getDirectoryPath(configFileName), this.compilerOptions);
            inputFiles = configParseResult.fileNames;
            this.compilerOptions = configParseResult.options;
            errors = [...result.parseDiagnostics, ...configParseResult.errors];
        }

        const compilerHost = new ProjectCompilerHost(this.sys, this.compilerOptions, this.testCaseJustName, this.testCase, moduleKind);
        const projectCompilerResult = this.compileProjectFiles(moduleKind, configFileSourceFiles, () => inputFiles, compilerHost, this.compilerOptions);

        this.compilerResult = {
            configFileSourceFiles,
            moduleKind,
            program: projectCompilerResult.program,
            compilerOptions: this.compilerOptions,
            sourceMapData: projectCompilerResult.sourceMapData,
            outputFiles: compilerHost.outputs,
            errors: errors ? ts.concatenate(errors, projectCompilerResult.errors) : projectCompilerResult.errors,
        };
    }

    private get vfs() {
        return this.sys.vfs;
    }

    public static getConfigurations(testCaseFileName: string): ProjectTestConfiguration[] {
        let testCase: ProjectRunnerTestCase & ts.CompilerOptions;

        let testFileText: string | undefined;
        try {
            testFileText = Harness.IO.readFile(testCaseFileName);
        }
        catch (e) {
            assert(false, "Unable to open testcase file: " + testCaseFileName + ": " + e.message);
        }

        try {
            testCase = JSON.parse(testFileText!) as ProjectRunnerTestCase & ts.CompilerOptions;
        }
        catch (e) {
            throw assert(false, "Testcase: " + testCaseFileName + " does not contain valid json format: " + e.message);
        }

        const fs = vfs.createFromFileSystem(Harness.IO, /*ignoreCase*/ false);
        fs.mountSync(vpath.resolve(Harness.IO.getWorkspaceRoot(), "tests"), vpath.combine(vfs.srcFolder, "tests"), vfs.createResolver(Harness.IO));
        fs.mkdirpSync(vpath.combine(vfs.srcFolder, testCase.projectRoot));
        fs.chdir(vpath.combine(vfs.srcFolder, testCase.projectRoot));
        fs.makeReadonly();

        return [
            { name: `@module: commonjs`, payload: { testCase, moduleKind: ts.ModuleKind.CommonJS, vfs: fs } },
            { name: `@module: amd`, payload: { testCase, moduleKind: ts.ModuleKind.AMD, vfs: fs } }
        ];
    }

    public verifyResolution() {
        const cwd = this.vfs.cwd();
        const ignoreCase = this.vfs.ignoreCase;
        const resolutionInfo: ProjectRunnerTestCaseResolutionInfo & ts.CompilerOptions = JSON.parse(JSON.stringify(this.testCase));
        resolutionInfo.resolvedInputFiles = this.compilerResult.program!.getSourceFiles()
            .map(({ fileName: input }) =>
                vpath.beneath(vfs.builtFolder, input, this.vfs.ignoreCase) || vpath.beneath(vfs.testLibFolder, input, this.vfs.ignoreCase) ? Utils.removeTestPathPrefixes(input) :
                vpath.isAbsolute(input) ? vpath.relative(cwd, input, ignoreCase) :
                input);

        resolutionInfo.emittedFiles = this.compilerResult.outputFiles!
            .map(output => output.meta.get("fileName") || output.file)
            .map(output => Utils.removeTestPathPrefixes(vpath.isAbsolute(output) ? vpath.relative(cwd, output, ignoreCase) : output));

        const content = JSON.stringify(resolutionInfo, undefined, "    ");
        Harness.Baseline.runBaseline(this.getBaselineFolder(this.compilerResult.moduleKind) + this.testCaseJustName + ".json", content);
    }

    public verifyDiagnostics() {
        if (this.compilerResult.errors.length) {
            Harness.Baseline.runBaseline(this.getBaselineFolder(this.compilerResult.moduleKind) + this.testCaseJustName + ".errors.txt", getErrorsBaseline(this.compilerResult));
        }
    }

    public verifyJavaScriptOutput() {
        if (this.testCase.baselineCheck) {
            const errs: Error[] = [];
            let nonSubfolderDiskFiles = 0;
            for (const output of this.compilerResult.outputFiles!) {
                try {
                    // convert file name to rooted name
                    // if filename is not rooted - concat it with project root and then expand project root relative to current directory
                    const fileName = output.meta.get("fileName") || output.file;
                    const diskFileName = vpath.isAbsolute(fileName) ? fileName : vpath.resolve(this.vfs.cwd(), fileName);

                    // compute file name relative to current directory (expanded project root)
                    let diskRelativeName = vpath.relative(this.vfs.cwd(), diskFileName, this.vfs.ignoreCase);
                    if (vpath.isAbsolute(diskRelativeName) || diskRelativeName.startsWith("../")) {
                        // If the generated output file resides in the parent folder or is rooted path,
                        // we need to instead create files that can live in the project reference folder
                        // but make sure extension of these files matches with the fileName the compiler asked to write
                        diskRelativeName = `diskFile${nonSubfolderDiskFiles}${vpath.extname(fileName, [".js.map", ".js", ".d.ts"], this.vfs.ignoreCase)}`;
                        nonSubfolderDiskFiles++;
                    }

                    const content = Utils.removeTestPathPrefixes(output.text, /*retainTrailingDirectorySeparator*/ true);
                    Harness.Baseline.runBaseline(this.getBaselineFolder(this.compilerResult.moduleKind) + diskRelativeName, content as string | null); // TODO: GH#18217
                }
                catch (e) {
                    errs.push(e);
                }
            }

            if (errs.length) {
                throw Error(errs.join("\n     "));
            }
        }
    }

    public verifySourceMapRecord() {
        // NOTE: This check was commented out in previous code. Leaving this here to eventually be restored if needed.
        // if (compilerResult.sourceMapData) {
        //     Harness.Baseline.runBaseline(getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + ".sourcemap.txt", () => {
        //         return Harness.SourceMapRecorder.getSourceMapRecord(compilerResult.sourceMapData, compilerResult.program,
        //             ts.filter(compilerResult.outputFiles, outputFile => Harness.Compiler.isJS(outputFile.emittedFileName)));
        //     });
        // }
    }

    public verifyDeclarations() {
        if (!this.compilerResult.errors.length && this.testCase.declaration) {
            const dTsCompileResult = this.compileDeclarations(this.compilerResult);
            if (dTsCompileResult && dTsCompileResult.errors.length) {
                Harness.Baseline.runBaseline(this.getBaselineFolder(this.compilerResult.moduleKind) + this.testCaseJustName + ".dts.errors.txt", getErrorsBaseline(dTsCompileResult));
            }
        }
    }

    // Project baselines verified go in project/testCaseName/moduleKind/
    private getBaselineFolder(moduleKind: ts.ModuleKind) {
        return "project/" + this.testCaseJustName + "/" + moduleNameToString(moduleKind) + "/";
    }

    private cleanProjectUrl(url: string) {
        let diskProjectPath = ts.normalizeSlashes(Harness.IO.resolvePath(this.testCase.projectRoot)!);
        let projectRootUrl = "file:///" + diskProjectPath;
        const normalizedProjectRoot = ts.normalizeSlashes(this.testCase.projectRoot);
        diskProjectPath = diskProjectPath.substr(0, diskProjectPath.lastIndexOf(normalizedProjectRoot));
        projectRootUrl = projectRootUrl.substr(0, projectRootUrl.lastIndexOf(normalizedProjectRoot));
        if (url && url.length) {
            if (url.indexOf(projectRootUrl) === 0) {
                // replace the disk specific project url path into project root url
                url = "file:///" + url.substr(projectRootUrl.length);
            }
            else if (url.indexOf(diskProjectPath) === 0) {
                // Replace the disk specific path into the project root path
                url = url.substr(diskProjectPath.length);
                if (url.charCodeAt(0) !== ts.CharacterCodes.slash) {
                    url = "/" + url;
                }
            }
        }

        return url;
    }

    private compileProjectFiles(moduleKind: ts.ModuleKind, configFileSourceFiles: readonly ts.SourceFile[],
        getInputFiles: () => readonly string[],
        compilerHost: ts.CompilerHost,
        compilerOptions: ts.CompilerOptions): CompileProjectFilesResult {

        const program = ts.createProgram(getInputFiles(), compilerOptions, compilerHost);
        const errors = ts.getPreEmitDiagnostics(program);

        const { sourceMaps: sourceMapData, diagnostics: emitDiagnostics } = program.emit();

        // Clean up source map data that will be used in baselining
        if (sourceMapData) {
            for (const data of sourceMapData) {
                data.sourceMap = {
                    ...data.sourceMap,
                    sources: data.sourceMap.sources.map(source => this.cleanProjectUrl(source)),
                    sourceRoot: data.sourceMap.sourceRoot && this.cleanProjectUrl(data.sourceMap.sourceRoot)
                };
            }
        }

        return {
            configFileSourceFiles,
            moduleKind,
            program,
            errors: ts.concatenate(errors, emitDiagnostics),
            sourceMapData
        };
    }

    private compileDeclarations(compilerResult: BatchCompileProjectTestCaseResult) {
        if (!compilerResult.program) {
            return;
        }

        const compilerOptions = compilerResult.program.getCompilerOptions();
        const allInputFiles: documents.TextDocument[] = [];
        const rootFiles: string[] = [];
        ts.forEach(compilerResult.program.getSourceFiles(), sourceFile => {
            if (sourceFile.isDeclarationFile) {
                if (!vpath.isDefaultLibrary(sourceFile.fileName)) {
                    allInputFiles.unshift(new documents.TextDocument(sourceFile.fileName, sourceFile.text));
                }
                rootFiles.unshift(sourceFile.fileName);
            }
            else if (!(compilerOptions.outFile || compilerOptions.out)) {
                let emitOutputFilePathWithoutExtension: string | undefined;
                if (compilerOptions.outDir) {
                    let sourceFilePath = ts.getNormalizedAbsolutePath(sourceFile.fileName, compilerResult.program!.getCurrentDirectory());
                    sourceFilePath = sourceFilePath.replace(compilerResult.program!.getCommonSourceDirectory(), "");
                    emitOutputFilePathWithoutExtension = ts.removeFileExtension(ts.combinePaths(compilerOptions.outDir, sourceFilePath));
                }
                else {
                    emitOutputFilePathWithoutExtension = ts.removeFileExtension(sourceFile.fileName);
                }

                const outputDtsFileName = emitOutputFilePathWithoutExtension + ts.Extension.Dts;
                const file = findOutputDtsFile(outputDtsFileName);
                if (file) {
                    allInputFiles.unshift(file);
                    rootFiles.unshift(file.meta.get("fileName") || file.file);
                }
            }
            else {
                const outputDtsFileName = ts.removeFileExtension(compilerOptions.outFile || compilerOptions.out!) + ts.Extension.Dts;
                const outputDtsFile = findOutputDtsFile(outputDtsFileName)!;
                if (!ts.contains(allInputFiles, outputDtsFile)) {
                    allInputFiles.unshift(outputDtsFile);
                    rootFiles.unshift(outputDtsFile.meta.get("fileName") || outputDtsFile.file);
                }
            }
        });

        const _vfs = vfs.createFromFileSystem(Harness.IO, /*ignoreCase*/ false, {
            documents: allInputFiles,
            cwd: vpath.combine(vfs.srcFolder, this.testCase.projectRoot)
        });

        // Dont allow config files since we are compiling existing source options
        const compilerHost = new ProjectCompilerHost(_vfs, compilerResult.compilerOptions!, this.testCaseJustName, this.testCase, compilerResult.moduleKind);
        return this.compileProjectFiles(compilerResult.moduleKind, compilerResult.configFileSourceFiles, () => rootFiles, compilerHost, compilerResult.compilerOptions!);

        function findOutputDtsFile(fileName: string) {
            return ts.forEach(compilerResult.outputFiles, outputFile => outputFile.meta.get("fileName") === fileName ? outputFile : undefined);
        }
    }
}

function moduleNameToString(moduleKind: ts.ModuleKind) {
    return moduleKind === ts.ModuleKind.AMD ? "amd" :
        moduleKind === ts.ModuleKind.CommonJS ? "node" : "none";
}

function getErrorsBaseline(compilerResult: CompileProjectFilesResult) {
    const inputSourceFiles = compilerResult.configFileSourceFiles.slice();
    if (compilerResult.program) {
        for (const sourceFile of compilerResult.program.getSourceFiles()) {
            if (!Harness.isDefaultLibraryFile(sourceFile.fileName)) {
                inputSourceFiles.push(sourceFile);
            }
        }
    }

    const inputFiles = inputSourceFiles.map<Harness.Compiler.TestFile>(sourceFile => ({
        unitName: ts.isRootedDiskPath(sourceFile.fileName) ?
            Harness.RunnerBase.removeFullPaths(sourceFile.fileName) :
            sourceFile.fileName,
        content: sourceFile.text
    }));

    return Harness.Compiler.getErrorBaseline(inputFiles, compilerResult.errors);
}

function createCompilerOptions(testCase: ProjectRunnerTestCase & ts.CompilerOptions, moduleKind: ts.ModuleKind) {
    // Set the special options that depend on other testcase options
    const compilerOptions: ts.CompilerOptions = {
        noErrorTruncation: false,
        skipDefaultLibCheck: false,
        moduleResolution: ts.ModuleResolutionKind.Classic,
        module: moduleKind,
        newLine: ts.NewLineKind.CarriageReturnLineFeed,
        mapRoot: testCase.resolveMapRoot && testCase.mapRoot
            ? vpath.resolve(vfs.srcFolder, testCase.mapRoot)
            : testCase.mapRoot,

        sourceRoot: testCase.resolveSourceRoot && testCase.sourceRoot
            ? vpath.resolve(vfs.srcFolder, testCase.sourceRoot)
            : testCase.sourceRoot
    };

    // Set the values specified using json
    const optionNameMap = ts.arrayToMap(ts.optionDeclarations, option => option.name);
    for (const name in testCase) {
        if (name !== "mapRoot" && name !== "sourceRoot") {
            const option = optionNameMap.get(name);
            if (option) {
                const optType = option.type;
                let value = testCase[name] as any;
                if (!ts.isString(optType)) {
                    const key = value.toLowerCase();
                    const optTypeValue = optType.get(key);
                    if (optTypeValue) {
                        value = optTypeValue;
                    }
                }
                compilerOptions[option.name] = value;
            }
        }
    }

    return compilerOptions;
}