• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import {
2    FileReference, isKeyword, lastOrUndefined, length, noop, PragmaContext, PreProcessedFileInfo, processCommentPragmas,
3    processPragmasIntoFields, scanner, ScriptTarget, SyntaxKind,
4} from "./_namespaces/ts";
5
6export function preProcessFile(sourceText: string, readImportFiles = true, detectJavaScriptImports = false): PreProcessedFileInfo {
7    const pragmaContext: PragmaContext = {
8        languageVersion: ScriptTarget.ES5, // controls whether the token scanner considers unicode identifiers or not - shouldn't matter, since we're only using it for trivia
9        pragmas: undefined,
10        checkJsDirective: undefined,
11        referencedFiles: [],
12        typeReferenceDirectives: [],
13        libReferenceDirectives: [],
14        amdDependencies: [],
15        hasNoDefaultLib: undefined,
16        moduleName: undefined
17    };
18    const importedFiles: FileReference[] = [];
19    let ambientExternalModules: { ref: FileReference, depth: number }[] | undefined;
20    let lastToken: SyntaxKind;
21    let currentToken: SyntaxKind;
22    let braceNesting = 0;
23    // assume that text represent an external module if it contains at least one top level import/export
24    // ambient modules that are found inside external modules are interpreted as module augmentations
25    let externalModule = false;
26
27    function nextToken() {
28        lastToken = currentToken;
29        currentToken = scanner.scan();
30        if (currentToken === SyntaxKind.OpenBraceToken) {
31            braceNesting++;
32        }
33        else if (currentToken === SyntaxKind.CloseBraceToken) {
34            braceNesting--;
35        }
36        return currentToken;
37    }
38
39    function getFileReference() {
40        const fileName = scanner.getTokenValue();
41        const pos = scanner.getTokenPos();
42        return { fileName, pos, end: pos + fileName.length };
43    }
44
45    function recordAmbientExternalModule(): void {
46        if (!ambientExternalModules) {
47            ambientExternalModules = [];
48        }
49        ambientExternalModules.push({ ref: getFileReference(), depth: braceNesting });
50    }
51
52    function recordModuleName() {
53        importedFiles.push(getFileReference());
54
55        markAsExternalModuleIfTopLevel();
56    }
57
58    function markAsExternalModuleIfTopLevel() {
59        if (braceNesting === 0) {
60            externalModule = true;
61        }
62    }
63
64    /**
65     * Returns true if at least one token was consumed from the stream
66     */
67    function tryConsumeDeclare(): boolean {
68        let token = scanner.getToken();
69        if (token === SyntaxKind.DeclareKeyword) {
70            // declare module "mod"
71            token = nextToken();
72            if (token === SyntaxKind.ModuleKeyword) {
73                token = nextToken();
74                if (token === SyntaxKind.StringLiteral) {
75                    recordAmbientExternalModule();
76                }
77            }
78            return true;
79        }
80
81        return false;
82    }
83
84    /**
85     * Returns true if at least one token was consumed from the stream
86     */
87    function tryConsumeImport(): boolean {
88        if (lastToken === SyntaxKind.DotToken) {
89            return false;
90        }
91        let token = scanner.getToken();
92        if (token === SyntaxKind.ImportKeyword) {
93            token = nextToken();
94            if (token === SyntaxKind.OpenParenToken) {
95                token = nextToken();
96                if (token === SyntaxKind.StringLiteral || token === SyntaxKind.NoSubstitutionTemplateLiteral) {
97                    // import("mod");
98                    recordModuleName();
99                    return true;
100                }
101            }
102            else if (token === SyntaxKind.StringLiteral) {
103                // import "mod";
104                recordModuleName();
105                return true;
106            }
107            else {
108                if (token === SyntaxKind.TypeKeyword) {
109                    const skipTypeKeyword = scanner.lookAhead(() => {
110                        const token = scanner.scan();
111                        return token !== SyntaxKind.FromKeyword && (
112                            token === SyntaxKind.AsteriskToken ||
113                            token === SyntaxKind.OpenBraceToken ||
114                            token === SyntaxKind.Identifier ||
115                            isKeyword(token)
116                        );
117                    });
118                    if (skipTypeKeyword) {
119                        token = nextToken();
120                    }
121                }
122
123                if (token === SyntaxKind.Identifier || isKeyword(token)) {
124                    token = nextToken();
125                    if (token === SyntaxKind.FromKeyword) {
126                        token = nextToken();
127                        if (token === SyntaxKind.StringLiteral) {
128                            // import d from "mod";
129                            recordModuleName();
130                            return true;
131                        }
132                    }
133                    else if (token === SyntaxKind.EqualsToken) {
134                        if (tryConsumeRequireCall(/*skipCurrentToken*/ true)) {
135                            return true;
136                        }
137                    }
138                    else if (token === SyntaxKind.CommaToken) {
139                        // consume comma and keep going
140                        token = nextToken();
141                    }
142                    else {
143                        // unknown syntax
144                        return true;
145                    }
146                }
147
148                if (token === SyntaxKind.OpenBraceToken) {
149                    token = nextToken();
150                    // consume "{ a as B, c, d as D}" clauses
151                    // make sure that it stops on EOF
152                    while (token !== SyntaxKind.CloseBraceToken && token !== SyntaxKind.EndOfFileToken) {
153                        token = nextToken();
154                    }
155
156                    if (token === SyntaxKind.CloseBraceToken) {
157                        token = nextToken();
158                        if (token === SyntaxKind.FromKeyword) {
159                            token = nextToken();
160                            if (token === SyntaxKind.StringLiteral) {
161                                // import {a as A} from "mod";
162                                // import d, {a, b as B} from "mod"
163                                recordModuleName();
164                            }
165                        }
166                    }
167                }
168                else if (token === SyntaxKind.AsteriskToken) {
169                    token = nextToken();
170                    if (token === SyntaxKind.AsKeyword) {
171                        token = nextToken();
172                        if (token === SyntaxKind.Identifier || isKeyword(token)) {
173                            token = nextToken();
174                            if (token === SyntaxKind.FromKeyword) {
175                                token = nextToken();
176                                if (token === SyntaxKind.StringLiteral) {
177                                    // import * as NS from "mod"
178                                    // import d, * as NS from "mod"
179                                    recordModuleName();
180                                }
181                            }
182                        }
183                    }
184                }
185            }
186
187            return true;
188        }
189
190        return false;
191    }
192
193    function tryConsumeExport(): boolean {
194        let token = scanner.getToken();
195        if (token === SyntaxKind.ExportKeyword) {
196            markAsExternalModuleIfTopLevel();
197            token = nextToken();
198            if (token === SyntaxKind.TypeKeyword) {
199                const skipTypeKeyword = scanner.lookAhead(() => {
200                    const token = scanner.scan();
201                    return token === SyntaxKind.AsteriskToken ||
202                        token === SyntaxKind.OpenBraceToken;
203                });
204                if (skipTypeKeyword) {
205                    token = nextToken();
206                }
207            }
208            if (token === SyntaxKind.OpenBraceToken) {
209                token = nextToken();
210                // consume "{ a as B, c, d as D}" clauses
211                // make sure it stops on EOF
212                while (token !== SyntaxKind.CloseBraceToken && token !== SyntaxKind.EndOfFileToken) {
213                    token = nextToken();
214                }
215
216                if (token === SyntaxKind.CloseBraceToken) {
217                    token = nextToken();
218                    if (token === SyntaxKind.FromKeyword) {
219                        token = nextToken();
220                        if (token === SyntaxKind.StringLiteral) {
221                            // export {a as A} from "mod";
222                            // export {a, b as B} from "mod"
223                            recordModuleName();
224                        }
225                    }
226                }
227            }
228            else if (token === SyntaxKind.AsteriskToken) {
229                token = nextToken();
230                if (token === SyntaxKind.FromKeyword) {
231                    token = nextToken();
232                    if (token === SyntaxKind.StringLiteral) {
233                        // export * from "mod"
234                        recordModuleName();
235                    }
236                }
237            }
238            else if (token === SyntaxKind.ImportKeyword) {
239                token = nextToken();
240                if (token === SyntaxKind.TypeKeyword) {
241                    const skipTypeKeyword = scanner.lookAhead(() => {
242                        const token = scanner.scan();
243                        return token === SyntaxKind.Identifier ||
244                            isKeyword(token);
245                    });
246                    if (skipTypeKeyword) {
247                        token = nextToken();
248                    }
249                }
250                if (token === SyntaxKind.Identifier || isKeyword(token)) {
251                    token = nextToken();
252                    if (token === SyntaxKind.EqualsToken) {
253                        if (tryConsumeRequireCall(/*skipCurrentToken*/ true)) {
254                            return true;
255                        }
256                    }
257                }
258            }
259
260            return true;
261        }
262
263        return false;
264    }
265
266    function tryConsumeRequireCall(skipCurrentToken: boolean, allowTemplateLiterals = false): boolean {
267        let token = skipCurrentToken ? nextToken() : scanner.getToken();
268        if (token === SyntaxKind.RequireKeyword) {
269            token = nextToken();
270            if (token === SyntaxKind.OpenParenToken) {
271                token = nextToken();
272                if (token === SyntaxKind.StringLiteral ||
273                    allowTemplateLiterals && token === SyntaxKind.NoSubstitutionTemplateLiteral) {
274                    //  require("mod");
275                    recordModuleName();
276                }
277            }
278            return true;
279        }
280        return false;
281    }
282
283    function tryConsumeDefine(): boolean {
284        let token = scanner.getToken();
285        if (token === SyntaxKind.Identifier && scanner.getTokenValue() === "define") {
286            token = nextToken();
287            if (token !== SyntaxKind.OpenParenToken) {
288                return true;
289            }
290
291            token = nextToken();
292            if (token === SyntaxKind.StringLiteral || token === SyntaxKind.NoSubstitutionTemplateLiteral) {
293                // looks like define ("modname", ... - skip string literal and comma
294                token = nextToken();
295                if (token === SyntaxKind.CommaToken) {
296                    token = nextToken();
297                }
298                else {
299                    // unexpected token
300                    return true;
301                }
302            }
303
304            // should be start of dependency list
305            if (token !== SyntaxKind.OpenBracketToken) {
306                return true;
307            }
308
309            // skip open bracket
310            token = nextToken();
311            // scan until ']' or EOF
312            while (token !== SyntaxKind.CloseBracketToken && token !== SyntaxKind.EndOfFileToken) {
313                // record string literals as module names
314                if (token === SyntaxKind.StringLiteral || token === SyntaxKind.NoSubstitutionTemplateLiteral) {
315                    recordModuleName();
316                }
317
318                token = nextToken();
319            }
320            return true;
321
322        }
323        return false;
324    }
325
326    function processImports(): void {
327        scanner.setText(sourceText);
328        nextToken();
329        // Look for:
330        //    import "mod";
331        //    import d from "mod"
332        //    import {a as A } from "mod";
333        //    import * as NS from "mod"
334        //    import d, {a, b as B} from "mod"
335        //    import i = require("mod");
336        //    import("mod");
337
338        //    export * from "mod"
339        //    export {a as b} from "mod"
340        //    export import i = require("mod")
341        //    (for JavaScript files) require("mod")
342
343        // Do not look for:
344        //    AnySymbol.import("mod")
345        //    AnySymbol.nested.import("mod")
346
347        while (true) {
348            if (scanner.getToken() === SyntaxKind.EndOfFileToken) {
349                break;
350            }
351
352            if (scanner.getToken() === SyntaxKind.TemplateHead) {
353                const stack = [scanner.getToken()];
354                loop: while (length(stack)) {
355                    const token = scanner.scan();
356                    switch (token) {
357                        case SyntaxKind.EndOfFileToken:
358                            break loop;
359                        case SyntaxKind.ImportKeyword:
360                            tryConsumeImport();
361                            break;
362                        case SyntaxKind.TemplateHead:
363                            stack.push(token);
364                            break;
365                        case SyntaxKind.OpenBraceToken:
366                            if (length(stack)) {
367                                stack.push(token);
368                            }
369                            break;
370                        case SyntaxKind.CloseBraceToken:
371                            if (length(stack)) {
372                                if (lastOrUndefined(stack) === SyntaxKind.TemplateHead) {
373                                    if (scanner.reScanTemplateToken(/* isTaggedTemplate */ false) === SyntaxKind.TemplateTail) {
374                                        stack.pop();
375                                    }
376                                }
377                                else {
378                                    stack.pop();
379                                }
380                            }
381                            break;
382                    }
383                }
384                nextToken();
385            }
386
387            // check if at least one of alternative have moved scanner forward
388            if (tryConsumeDeclare() ||
389                tryConsumeImport() ||
390                tryConsumeExport() ||
391                (detectJavaScriptImports && (
392                    tryConsumeRequireCall(/*skipCurrentToken*/ false, /*allowTemplateLiterals*/ true) ||
393                    tryConsumeDefine()
394                ))) {
395                continue;
396            }
397            else {
398                nextToken();
399            }
400        }
401
402        scanner.setText(undefined);
403    }
404
405    if (readImportFiles) {
406        processImports();
407    }
408    processCommentPragmas(pragmaContext, sourceText);
409    processPragmasIntoFields(pragmaContext, noop);
410    if (externalModule) {
411        // for external modules module all nested ambient modules are augmentations
412        if (ambientExternalModules) {
413            // move all detected ambient modules to imported files since they need to be resolved
414            for (const decl of ambientExternalModules) {
415                importedFiles.push(decl.ref);
416            }
417        }
418        return { referencedFiles: pragmaContext.referencedFiles, typeReferenceDirectives: pragmaContext.typeReferenceDirectives, libReferenceDirectives: pragmaContext.libReferenceDirectives, importedFiles, isLibFile: !!pragmaContext.hasNoDefaultLib, ambientExternalModules: undefined };
419    }
420    else {
421        // for global scripts ambient modules still can have augmentations - look for ambient modules with depth > 0
422        let ambientModuleNames: string[] | undefined;
423        if (ambientExternalModules) {
424            for (const decl of ambientExternalModules) {
425                if (decl.depth === 0) {
426                    if (!ambientModuleNames) {
427                        ambientModuleNames = [];
428                    }
429                    ambientModuleNames.push(decl.ref.fileName);
430                }
431                else {
432                    importedFiles.push(decl.ref);
433                }
434            }
435        }
436        return { referencedFiles: pragmaContext.referencedFiles, typeReferenceDirectives: pragmaContext.typeReferenceDirectives, libReferenceDirectives: pragmaContext.libReferenceDirectives, importedFiles, isLibFile: !!pragmaContext.hasNoDefaultLib, ambientExternalModules: ambientModuleNames };
437    }
438}
439