1 /*
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "options.h"
17
18 #include <fstream>
19 #include <set>
20 #include <sstream>
21 #include <utility>
22
23 #if defined(PANDA_TARGET_WINDOWS)
24 #include <io.h>
25 #else
26 #include <dirent.h>
27 #endif
28
29 #include "os/file.h"
30
31 #include "mergeProgram.h"
32 #include "util/helpers.h"
33 #include "utils/pandargs.h"
34
35 namespace panda::es2panda::aot {
36 constexpr char PROCESS_AS_LIST_MARK = '@';
37 const std::string LIST_ITEM_SEPERATOR = ";";
38 const std::set<std::string> VALID_EXTENSIONS = { "js", "ts", "as" };
39
40 template <class T>
RemoveExtension(T const & filename)41 T RemoveExtension(T const &filename)
42 {
43 typename T::size_type const P(filename.find_last_of('.'));
44 return P > 0 && P != T::npos ? filename.substr(0, P) : filename;
45 }
46
GetScriptExtensionFromStr(const std::string & extension)47 static es2panda::ScriptExtension GetScriptExtensionFromStr(const std::string &extension)
48 {
49 if (extension == "js") {
50 return es2panda::ScriptExtension::JS;
51 } else if (extension == "ts") {
52 return es2panda::ScriptExtension::TS;
53 } else if (extension == "as") {
54 return es2panda::ScriptExtension::AS;
55 } else {
56 return es2panda::ScriptExtension::JS;
57 }
58 }
59
GetScriptExtension(const std::string & filename,const std::string & inputExtension)60 static es2panda::ScriptExtension GetScriptExtension(const std::string &filename, const std::string &inputExtension)
61 {
62 std::string fileExtension = "";
63 std::string::size_type pos(filename.find_last_of('.'));
64 if (pos > 0 && pos != std::string::npos) {
65 fileExtension = filename.substr(pos + 1);
66 }
67
68 if (VALID_EXTENSIONS.find(fileExtension) != VALID_EXTENSIONS.end()) {
69 return GetScriptExtensionFromStr(fileExtension);
70 }
71
72 return GetScriptExtensionFromStr(inputExtension);
73 }
74
GetStringItems(std::string & input,const std::string & delimiter)75 static std::vector<std::string> GetStringItems(std::string &input, const std::string &delimiter)
76 {
77 std::vector<std::string> items;
78 size_t pos = 0;
79 std::string token;
80 while ((pos = input.find(delimiter)) != std::string::npos) {
81 token = input.substr(0, pos);
82 if (!token.empty()) {
83 items.push_back(token);
84 }
85 input.erase(0, pos + delimiter.length());
86 }
87 if (!input.empty()) {
88 items.push_back(input);
89 }
90 return items;
91 }
92
93 // Options
CollectInputFilesFromFileList(const std::string & input,const std::string & inputExtension)94 bool Options::CollectInputFilesFromFileList(const std::string &input, const std::string &inputExtension)
95 {
96 std::ifstream ifs;
97 std::string line;
98 ifs.open(panda::os::file::File::GetExtendedFilePath(input));
99 if (!ifs.is_open()) {
100 std::cerr << "Failed to open source list: " << input << std::endl;
101 return false;
102 }
103
104 constexpr size_t ITEM_COUNT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile, pkgName]
105 constexpr size_t ITEM_COUNT_NOT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile; outputfile]
106 while (std::getline(ifs, line)) {
107 std::vector<std::string> itemList = GetStringItems(line, LIST_ITEM_SEPERATOR);
108 if ((compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_MERGE) ||
109 (!compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_NOT_MERGE)) {
110 std::cerr << "Failed to parse input file" << std::endl;
111 return false;
112 }
113
114 std::string fileName = itemList[0];
115 std::string recordName = compilerOptions_.mergeAbc ? itemList[1] : "";
116 parser::ScriptKind scriptKind;
117 if (itemList[2] == "script") {
118 scriptKind = parser::ScriptKind::SCRIPT;
119 } else if (itemList[2] == "commonjs") {
120 scriptKind = parser::ScriptKind::COMMONJS;
121 } else {
122 scriptKind = parser::ScriptKind::MODULE;
123 }
124
125 es2panda::SourceFile src(fileName, recordName, scriptKind, GetScriptExtension(fileName, inputExtension));
126 src.sourcefile = itemList[3];
127 if (compilerOptions_.mergeAbc) {
128 src.pkgName = itemList[4];
129 }
130
131 sourceFiles_.push_back(src);
132 if (!compilerOptions_.mergeAbc) {
133 outputFiles_.insert({fileName, itemList[4]});
134 }
135 }
136 return true;
137 }
138
CollectInputFilesFromFileDirectory(const std::string & input,const std::string & extension)139 bool Options::CollectInputFilesFromFileDirectory(const std::string &input, const std::string &extension)
140 {
141 std::vector<std::string> files;
142 if (!proto::MergeProgram::GetProtoFiles(input, extension, files)) {
143 return false;
144 }
145
146 for (const auto &f : files) {
147 es2panda::SourceFile src(f, RemoveExtension(f.substr(input.length() + 1)),
148 scriptKind_, GetScriptExtensionFromStr(extension));
149 sourceFiles_.push_back(src);
150 }
151
152 return true;
153 }
154
ParseCacheFileOption(const std::string & cacheInput)155 void Options::ParseCacheFileOption(const std::string &cacheInput)
156 {
157 if (cacheInput[0] != PROCESS_AS_LIST_MARK) {
158 compilerOptions_.cacheFiles.insert({sourceFile_, cacheInput});
159 return;
160 }
161
162 std::ifstream ifs;
163 std::string line;
164 ifs.open(panda::os::file::File::GetExtendedFilePath(cacheInput.substr(1)));
165 if (!ifs.is_open()) {
166 std::cerr << "Failed to open cache file list: " << cacheInput << std::endl;
167 return;
168 }
169
170 constexpr int cacheListItemCount = 2;
171 while (std::getline(ifs, line)) {
172 std::vector<std::string> itemList = GetStringItems(line, LIST_ITEM_SEPERATOR);
173 if (itemList.size() != cacheListItemCount) {
174 continue;
175 }
176 compilerOptions_.cacheFiles.insert({itemList[0], itemList[1]});
177 }
178 }
179
Options()180 Options::Options() : argparser_(new panda::PandArgParser()) {}
181
~Options()182 Options::~Options()
183 {
184 delete argparser_;
185 }
186
Parse(int argc,const char ** argv)187 bool Options::Parse(int argc, const char **argv)
188 {
189 panda::PandArg<bool> opHelp("help", false, "Print this message and exit");
190
191 // parser
192 panda::PandArg<std::string> inputExtension("extension", "js",
193 "Parse the input as the given extension (options: js | ts | as)");
194 panda::PandArg<bool> opModule("module", false, "Parse the input as module");
195 panda::PandArg<bool> opCommonjs("commonjs", false, "Parse the input as commonjs");
196 panda::PandArg<bool> opParseOnly("parse-only", false, "Parse the input only");
197 panda::PandArg<bool> opEnableTypeCheck("enable-type-check", false, "Check the type in ts after parse");
198 panda::PandArg<bool> opDumpAst("dump-ast", false, "Dump the parsed AST");
199 panda::PandArg<bool> opDumpTransformedAst("dump-transformed-ast", false, "Dump the parsed AST after transform");
200 panda::PandArg<bool> opCheckTransformedAstStructure("check-transformed-ast-structure", false,
201 "Check the AST structure after transform");
202
203 // type extractor
204 panda::PandArg<bool> opTypeExtractor("type-extractor", false, "Enable type extractor for typescript");
205 panda::PandArg<bool> opTypeDtsBuiltin("type-dts-builtin", false, "Enable builtin type extractor for .d.ts file");
206
207 // compiler
208 panda::PandArg<bool> opDumpAssembly("dump-assembly", false, "Dump pandasm");
209 panda::PandArg<bool> opDebugInfo("debug-info", false, "Compile with debug info");
210 panda::PandArg<bool> opDumpDebugInfo("dump-debug-info", false, "Dump debug info");
211 panda::PandArg<int> opOptLevel("opt-level", 2,
212 "Compiler optimization level (options: 0 | 1 | 2). In debug and base64Input mode, optimizer is disabled");
213 panda::PandArg<int> opFunctionThreadCount("function-threads", 0, "Number of worker threads to compile function");
214 panda::PandArg<int> opFileThreadCount("file-threads", 0, "Number of worker threads to compile file");
215 panda::PandArg<bool> opSizeStat("dump-size-stat", false, "Dump size statistics");
216 panda::PandArg<bool> opDumpLiteralBuffer("dump-literal-buffer", false, "Dump literal buffer");
217 panda::PandArg<std::string> outputFile("output", "", "Compiler binary output (.abc)");
218 panda::PandArg<std::string> recordName("record-name", "", "Specify the record name");
219 panda::PandArg<bool> debuggerEvaluateExpression("debugger-evaluate-expression", false,
220 "evaluate expression in debugger mode");
221 panda::PandArg<std::string> base64Input("base64Input", "", "base64 input of js content");
222 panda::PandArg<bool> base64Output("base64Output", false, "output panda file content as base64 to std out");
223 panda::PandArg<std::string> sourceFile("source-file", "",
224 "specify the file path info recorded in generated abc");
225 panda::PandArg<std::string> outputProto("outputProto", "",
226 "specify the output name for serializd protobuf file (.protoBin)");
227 panda::PandArg<std::string> opCacheFile("cache-file", "", "cache file for incremental compile");
228 panda::PandArg<std::string> opNpmModuleEntryList("npm-module-entry-list", "", "entry list file for module compile");
229 panda::PandArg<bool> opMergeAbc("merge-abc", false, "Compile as merge abc");
230
231 // patchfix && hotreload
232 panda::PandArg<std::string> opDumpSymbolTable("dump-symbol-table", "", "dump symbol table to file");
233 panda::PandArg<std::string> opInputSymbolTable("input-symbol-table", "", "input symbol table file");
234 panda::PandArg<bool> opGeneratePatch("generate-patch", false, "generate patch abc, default as hotfix mode unless "\
235 "the cold-fix argument is set");
236 panda::PandArg<bool> opHotReload("hot-reload", false, "compile as hot-reload mode");
237 panda::PandArg<bool> opColdFix("cold-fix", false, "generate patch abc as cold-fix mode");
238
239 // version
240 panda::PandArg<bool> bcVersion("bc-version", false, "Print ark bytecode version");
241 panda::PandArg<bool> bcMinVersion("bc-min-version", false, "Print ark bytecode minimum supported version");
242
243 // tail arguments
244 panda::PandArg<std::string> inputFile("input", "", "input file");
245
246 argparser_->Add(&opHelp);
247 argparser_->Add(&opModule);
248 argparser_->Add(&opCommonjs);
249 argparser_->Add(&opDumpAst);
250 argparser_->Add(&opDumpTransformedAst);
251 argparser_->Add(&opCheckTransformedAstStructure);
252 argparser_->Add(&opParseOnly);
253 argparser_->Add(&opEnableTypeCheck);
254 argparser_->Add(&opTypeExtractor);
255 argparser_->Add(&opTypeDtsBuiltin);
256 argparser_->Add(&opDumpAssembly);
257 argparser_->Add(&opDebugInfo);
258 argparser_->Add(&opDumpDebugInfo);
259 argparser_->Add(&debuggerEvaluateExpression);
260 argparser_->Add(&base64Input);
261 argparser_->Add(&base64Output);
262
263 argparser_->Add(&opOptLevel);
264 argparser_->Add(&opFunctionThreadCount);
265 argparser_->Add(&opFileThreadCount);
266 argparser_->Add(&opSizeStat);
267 argparser_->Add(&opDumpLiteralBuffer);
268
269 argparser_->Add(&inputExtension);
270 argparser_->Add(&outputFile);
271 argparser_->Add(&sourceFile);
272 argparser_->Add(&recordName);
273 argparser_->Add(&outputProto);
274 argparser_->Add(&opCacheFile);
275 argparser_->Add(&opNpmModuleEntryList);
276 argparser_->Add(&opMergeAbc);
277
278 argparser_->Add(&opDumpSymbolTable);
279 argparser_->Add(&opInputSymbolTable);
280 argparser_->Add(&opGeneratePatch);
281 argparser_->Add(&opHotReload);
282 argparser_->Add(&opColdFix);
283
284 argparser_->Add(&bcVersion);
285 argparser_->Add(&bcMinVersion);
286
287 argparser_->PushBackTail(&inputFile);
288 argparser_->EnableTail();
289 argparser_->EnableRemainder();
290
291 bool parseStatus = argparser_->Parse(argc, argv);
292 if (parseStatus && (bcVersion.GetValue() || bcMinVersion.GetValue())) {
293 compilerOptions_.bcVersion = bcVersion.GetValue();
294 compilerOptions_.bcMinVersion = bcMinVersion.GetValue();
295 return true;
296 }
297
298 if (!parseStatus || opHelp.GetValue() || (inputFile.GetValue().empty() && base64Input.GetValue().empty())) {
299 std::stringstream ss;
300
301 ss << argparser_->GetErrorString() << std::endl;
302 ss << "Usage: "
303 << "es2panda"
304 << " [OPTIONS] [input file] -- [arguments]" << std::endl;
305 ss << std::endl;
306 ss << "optional arguments:" << std::endl;
307 ss << argparser_->GetHelpString() << std::endl;
308
309 errorMsg_ = ss.str();
310 return false;
311 }
312
313 bool inputIsEmpty = inputFile.GetValue().empty();
314 bool base64InputIsEmpty = base64Input.GetValue().empty();
315 bool outputIsEmpty = outputFile.GetValue().empty();
316
317 if (!inputIsEmpty && !base64InputIsEmpty) {
318 errorMsg_ = "--input and --base64Input can not be used simultaneously";
319 return false;
320 }
321
322 if (!outputIsEmpty && base64Output.GetValue()) {
323 errorMsg_ = "--output and --base64Output can not be used simultaneously";
324 return false;
325 }
326
327 if (opModule.GetValue() && opCommonjs.GetValue()) {
328 errorMsg_ = "[--module] and [--commonjs] can not be used simultaneously";
329 return false;
330 }
331
332 if (opModule.GetValue()) {
333 scriptKind_ = es2panda::parser::ScriptKind::MODULE;
334 } else if (opCommonjs.GetValue()) {
335 scriptKind_ = es2panda::parser::ScriptKind::COMMONJS;
336 } else {
337 scriptKind_ = es2panda::parser::ScriptKind::SCRIPT;
338 }
339
340 auto parseTypeExtractor = [&opTypeExtractor, &opTypeDtsBuiltin, this]() {
341 compilerOptions_.typeExtractor = opTypeExtractor.GetValue();
342 if (compilerOptions_.typeExtractor) {
343 compilerOptions_.typeDtsBuiltin = opTypeDtsBuiltin.GetValue();
344 DCOUT << "[LOG]TypeExtractor is enabled, type-dts-builtin: " <<
345 compilerOptions_.typeDtsBuiltin << std::endl;
346 }
347 };
348 parseTypeExtractor(); // Type Extractor is only enabled for TypeScript
349
350 std::string extension = inputExtension.GetValue();
351 if (!extension.empty()) {
352 if (VALID_EXTENSIONS.find(extension) == VALID_EXTENSIONS.end()) {
353 errorMsg_ = "Invalid extension (available options: js, ts, as)";
354 return false;
355 }
356 }
357
358 bool isInputFileList = false;
359 if (!inputIsEmpty) {
360 std::string rawInput = inputFile.GetValue();
361 isInputFileList = rawInput[0] == PROCESS_AS_LIST_MARK;
362 std::string input = isInputFileList ? rawInput.substr(1) : rawInput;
363 sourceFile_ = input;
364 }
365
366 if (base64Output.GetValue()) {
367 compilerOutput_ = "";
368 } else if (!outputIsEmpty) {
369 compilerOutput_ = outputFile.GetValue();
370 } else if (outputIsEmpty && !inputIsEmpty) {
371 compilerOutput_ = RemoveExtension(util::Helpers::BaseName(sourceFile_)).append(".abc");
372 }
373
374 if (opMergeAbc.GetValue()) {
375 recordName_ = recordName.GetValue();
376 if (recordName_.empty()) {
377 recordName_ = compilerOutput_.empty() ? "Base64Output" :
378 RemoveExtension(util::Helpers::BaseName(compilerOutput_));
379 }
380 compilerOptions_.mergeAbc = opMergeAbc.GetValue();
381 }
382
383 if (!inputIsEmpty) {
384 // common mode
385 auto inputAbs = panda::os::file::File::GetAbsolutePath(sourceFile_);
386 if (!inputAbs) {
387 std::cerr << "Failed to find: " << sourceFile_ << std::endl;
388 return false;
389 }
390
391 auto fpath = inputAbs.Value();
392 if (isInputFileList) {
393 CollectInputFilesFromFileList(fpath, extension);
394 } else if (panda::os::file::File::IsDirectory(fpath)) {
395 CollectInputFilesFromFileDirectory(fpath, extension);
396 } else {
397 es2panda::SourceFile src(sourceFile_, recordName_, scriptKind_, GetScriptExtension(sourceFile_, extension));
398 sourceFiles_.push_back(src);
399 }
400 } else if (!base64InputIsEmpty) {
401 // input content is base64 string
402 base64Input_ = ExtractContentFromBase64Input(base64Input.GetValue());
403 if (base64Input_.empty()) {
404 errorMsg_ = "The input string is not a valid base64 data";
405 return false;
406 }
407
408 es2panda::SourceFile src("", recordName_, es2panda::parser::ScriptKind::SCRIPT,
409 GetScriptExtensionFromStr(extension));
410 src.source = base64Input_;
411 sourceFiles_.push_back(src);
412 }
413
414 if (!outputProto.GetValue().empty()) {
415 compilerProtoOutput_ = outputProto.GetValue();
416 }
417
418 optLevel_ = opOptLevel.GetValue();
419 functionThreadCount_ = opFunctionThreadCount.GetValue();
420 fileThreadCount_ = opFileThreadCount.GetValue();
421 npmModuleEntryList_ = opNpmModuleEntryList.GetValue();
422
423 if (!opCacheFile.GetValue().empty()) {
424 ParseCacheFileOption(opCacheFile.GetValue());
425 }
426
427 if (opParseOnly.GetValue()) {
428 options_ |= OptionFlags::PARSE_ONLY;
429 }
430
431 if (opSizeStat.GetValue()) {
432 options_ |= OptionFlags::SIZE_STAT;
433 }
434
435 compilerOptions_.dumpAsm = opDumpAssembly.GetValue();
436 compilerOptions_.dumpAst = opDumpAst.GetValue();
437 compilerOptions_.dumpTransformedAst = opDumpTransformedAst.GetValue();
438 compilerOptions_.checkTransformedAstStructure = opCheckTransformedAstStructure.GetValue();
439 compilerOptions_.dumpDebugInfo = opDumpDebugInfo.GetValue();
440 compilerOptions_.isDebug = opDebugInfo.GetValue();
441 compilerOptions_.parseOnly = opParseOnly.GetValue();
442 compilerOptions_.enableTypeCheck = opEnableTypeCheck.GetValue();
443 compilerOptions_.dumpLiteralBuffer = opDumpLiteralBuffer.GetValue();
444 compilerOptions_.isDebuggerEvaluateExpressionMode = debuggerEvaluateExpression.GetValue();
445
446 compilerOptions_.functionThreadCount = functionThreadCount_;
447 compilerOptions_.fileThreadCount = fileThreadCount_;
448 compilerOptions_.output = compilerOutput_;
449 compilerOptions_.debugInfoSourceFile = sourceFile.GetValue();
450 compilerOptions_.optLevel = (compilerOptions_.isDebug || !base64Input.GetValue().empty() ||
451 base64Output.GetValue()) ? 0 : opOptLevel.GetValue();
452 compilerOptions_.sourceFiles = sourceFiles_;
453 compilerOptions_.mergeAbc = opMergeAbc.GetValue();
454
455 compilerOptions_.patchFixOptions.dumpSymbolTable = opDumpSymbolTable.GetValue();
456 compilerOptions_.patchFixOptions.symbolTable = opInputSymbolTable.GetValue();
457
458 bool generatePatch = opGeneratePatch.GetValue();
459 bool hotReload = opHotReload.GetValue();
460 bool coldFix = opColdFix.GetValue();
461 if (generatePatch && hotReload) {
462 errorMsg_ = "--generate-patch and --hot-reload can not be used simultaneously";
463 return false;
464 }
465 if (coldFix && !generatePatch) {
466 errorMsg_ = "--cold-fix can not be used without --generate-patch";
467 return false;
468 }
469 compilerOptions_.patchFixOptions.generatePatch = generatePatch;
470 compilerOptions_.patchFixOptions.hotReload = hotReload;
471 compilerOptions_.patchFixOptions.coldFix = coldFix;
472
473 return true;
474 }
475
ExtractContentFromBase64Input(const std::string & inputBase64String)476 std::string Options::ExtractContentFromBase64Input(const std::string &inputBase64String)
477 {
478 std::string inputContent = util::Base64Decode(inputBase64String);
479 if (inputContent == "") {
480 return "";
481 }
482 bool validBase64Input = util::Base64Encode(inputContent) == inputBase64String;
483 if (!validBase64Input) {
484 return "";
485 }
486 return inputContent;
487 }
488 } // namespace panda::es2panda::aot
489