1 /*
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "options.h"
17
18 #include <fstream>
19 #include <set>
20 #include <sstream>
21 #include <utility>
22
23 #if defined(PANDA_TARGET_WINDOWS)
24 #include <io.h>
25 #else
26 #include <dirent.h>
27 #endif
28
29 #include "os/file.h"
30
31 #include "mergeProgram.h"
32 #include "util/helpers.h"
33 #include "utils/pandargs.h"
34
35 namespace panda::es2panda::aot {
36 constexpr char PROCESS_AS_LIST_MARK = '@';
37 const std::string LIST_ITEM_SEPERATOR = ";";
38 const std::set<std::string> VALID_EXTENSIONS = { "js", "ts", "as" };
39
40 template <class T>
RemoveExtension(T const & filename)41 T RemoveExtension(T const &filename)
42 {
43 typename T::size_type const P(filename.find_last_of('.'));
44 return P > 0 && P != T::npos ? filename.substr(0, P) : filename;
45 }
46
GetScriptExtensionFromStr(const std::string & extension)47 static es2panda::ScriptExtension GetScriptExtensionFromStr(const std::string &extension)
48 {
49 if (extension == "js") {
50 return es2panda::ScriptExtension::JS;
51 } else if (extension == "ts") {
52 return es2panda::ScriptExtension::TS;
53 } else if (extension == "as") {
54 return es2panda::ScriptExtension::AS;
55 } else {
56 return es2panda::ScriptExtension::JS;
57 }
58 }
59
GetScriptExtension(const std::string & filename,const std::string & inputExtension)60 static es2panda::ScriptExtension GetScriptExtension(const std::string &filename, const std::string &inputExtension)
61 {
62 std::string fileExtension = "";
63 std::string::size_type pos(filename.find_last_of('.'));
64 if (pos > 0 && pos != std::string::npos) {
65 fileExtension = filename.substr(pos + 1);
66 }
67
68 if (VALID_EXTENSIONS.find(fileExtension) != VALID_EXTENSIONS.end()) {
69 return GetScriptExtensionFromStr(fileExtension);
70 }
71
72 return GetScriptExtensionFromStr(inputExtension);
73 }
74
GetStringItems(std::string & input,const std::string & delimiter)75 static std::vector<std::string> GetStringItems(std::string &input, const std::string &delimiter)
76 {
77 std::vector<std::string> items;
78 size_t pos = 0;
79 std::string token;
80 while ((pos = input.find(delimiter)) != std::string::npos) {
81 token = input.substr(0, pos);
82 if (!token.empty()) {
83 items.push_back(token);
84 }
85 input.erase(0, pos + delimiter.length());
86 }
87 if (!input.empty()) {
88 items.push_back(input);
89 }
90 return items;
91 }
92
93 // Options
CollectInputFilesFromFileList(const std::string & input,const std::string & inputExtension)94 bool Options::CollectInputFilesFromFileList(const std::string &input, const std::string &inputExtension)
95 {
96 std::ifstream ifs;
97 std::string line;
98 ifs.open(panda::os::file::File::GetExtendedFilePath(input));
99 if (!ifs.is_open()) {
100 std::cerr << "Failed to open source list: " << input << std::endl;
101 return false;
102 }
103
104 constexpr size_t ITEM_COUNT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile, pkgName]
105 constexpr size_t ITEM_COUNT_NOT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile; outputfile]
106 while (std::getline(ifs, line)) {
107 std::vector<std::string> itemList = GetStringItems(line, LIST_ITEM_SEPERATOR);
108 if ((compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_MERGE) ||
109 (!compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_NOT_MERGE)) {
110 std::cerr << "Failed to parse input file" << std::endl;
111 return false;
112 }
113
114 std::string fileName = itemList[0];
115 std::string recordName = compilerOptions_.mergeAbc ? itemList[1] : "";
116 parser::ScriptKind scriptKind;
117 if (itemList[2] == "script") {
118 scriptKind = parser::ScriptKind::SCRIPT;
119 } else if (itemList[2] == "commonjs") {
120 scriptKind = parser::ScriptKind::COMMONJS;
121 } else {
122 scriptKind = parser::ScriptKind::MODULE;
123 }
124
125 es2panda::SourceFile src(fileName, recordName, scriptKind, GetScriptExtension(fileName, inputExtension));
126 src.sourcefile = itemList[3];
127 if (compilerOptions_.mergeAbc) {
128 src.pkgName = itemList[4];
129 }
130
131 sourceFiles_.push_back(src);
132 if (!compilerOptions_.mergeAbc) {
133 outputFiles_.insert({fileName, itemList[4]});
134 }
135 }
136 return true;
137 }
138
CollectInputFilesFromFileDirectory(const std::string & input,const std::string & extension)139 bool Options::CollectInputFilesFromFileDirectory(const std::string &input, const std::string &extension)
140 {
141 std::vector<std::string> files;
142 if (!proto::MergeProgram::GetProtoFiles(input, extension, files)) {
143 return false;
144 }
145
146 for (const auto &f : files) {
147 es2panda::SourceFile src(f, RemoveExtension(f.substr(input.length() + 1)),
148 scriptKind_, GetScriptExtensionFromStr(extension));
149 sourceFiles_.push_back(src);
150 }
151
152 return true;
153 }
154
ParseCacheFileOption(const std::string & cacheInput)155 void Options::ParseCacheFileOption(const std::string &cacheInput)
156 {
157 if (cacheInput[0] != PROCESS_AS_LIST_MARK) {
158 compilerOptions_.cacheFiles.insert({sourceFile_, cacheInput});
159 return;
160 }
161
162 std::ifstream ifs;
163 std::string line;
164 ifs.open(panda::os::file::File::GetExtendedFilePath(cacheInput.substr(1)));
165 if (!ifs.is_open()) {
166 std::cerr << "Failed to open cache file list: " << cacheInput << std::endl;
167 return;
168 }
169
170 constexpr int cacheListItemCount = 2;
171 while (std::getline(ifs, line)) {
172 std::vector<std::string> itemList = GetStringItems(line, LIST_ITEM_SEPERATOR);
173 if (itemList.size() != cacheListItemCount) {
174 continue;
175 }
176 compilerOptions_.cacheFiles.insert({itemList[0], itemList[1]});
177 }
178 }
179
Options()180 Options::Options() : argparser_(new panda::PandArgParser()) {}
181
~Options()182 Options::~Options()
183 {
184 delete argparser_;
185 }
186
Parse(int argc,const char ** argv)187 bool Options::Parse(int argc, const char **argv)
188 {
189 panda::PandArg<bool> opHelp("help", false, "Print this message and exit");
190
191 // parser
192 panda::PandArg<std::string> inputExtension("extension", "js",
193 "Parse the input as the given extension (options: js | ts | as)");
194 panda::PandArg<bool> opModule("module", false, "Parse the input as module");
195 panda::PandArg<bool> opCommonjs("commonjs", false, "Parse the input as commonjs");
196 panda::PandArg<bool> opParseOnly("parse-only", false, "Parse the input only");
197 panda::PandArg<bool> opEnableTypeCheck("enable-type-check", false, "Check the type in ts after parse");
198 panda::PandArg<bool> opDumpAst("dump-ast", false, "Dump the parsed AST");
199 panda::PandArg<bool> opDumpTransformedAst("dump-transformed-ast", false, "Dump the parsed AST after transform");
200 panda::PandArg<bool> opCheckTransformedAstStructure("check-transformed-ast-structure", false,
201 "Check the AST structure after transform");
202
203 // type extractor
204 panda::PandArg<bool> opTypeExtractor("type-extractor", false, "Enable type extractor for typescript");
205 panda::PandArg<bool> opTypeDtsBuiltin("type-dts-builtin", false, "Enable builtin type extractor for .d.ts file");
206
207 // compiler
208 panda::PandArg<bool> opDumpAssembly("dump-assembly", false, "Dump pandasm");
209 panda::PandArg<bool> opDebugInfo("debug-info", false, "Compile with debug info");
210 panda::PandArg<bool> opDumpDebugInfo("dump-debug-info", false, "Dump debug info");
211 panda::PandArg<int> opOptLevel("opt-level", 2,
212 "Compiler optimization level (options: 0 | 1 | 2). In debug and base64Input mode, optimizer is disabled");
213 panda::PandArg<int> opFunctionThreadCount("function-threads", 0, "Number of worker threads to compile function");
214 panda::PandArg<int> opFileThreadCount("file-threads", 0, "Number of worker threads to compile file");
215 panda::PandArg<bool> opSizeStat("dump-size-stat", false, "Dump size statistics");
216 panda::PandArg<bool> opDumpLiteralBuffer("dump-literal-buffer", false, "Dump literal buffer");
217 panda::PandArg<std::string> outputFile("output", "", "Compiler binary output (.abc)");
218 panda::PandArg<std::string> recordName("record-name", "", "Specify the record name");
219 panda::PandArg<bool> debuggerEvaluateExpression("debugger-evaluate-expression", false,
220 "evaluate expression in debugger mode");
221 panda::PandArg<std::string> base64Input("base64Input", "", "base64 input of js content");
222 panda::PandArg<bool> base64Output("base64Output", false, "output panda file content as base64 to std out");
223 panda::PandArg<std::string> sourceFile("source-file", "",
224 "specify the file path info recorded in generated abc");
225 panda::PandArg<std::string> outputProto("outputProto", "",
226 "specify the output name for serializd protobuf file (.protoBin)");
227 panda::PandArg<std::string> opCacheFile("cache-file", "", "cache file for incremental compile");
228 panda::PandArg<std::string> opNpmModuleEntryList("npm-module-entry-list", "", "entry list file for module compile");
229 panda::PandArg<bool> opMergeAbc("merge-abc", false, "Compile as merge abc");
230
231 // patchfix && hotreload
232 panda::PandArg<std::string> opDumpSymbolTable("dump-symbol-table", "", "dump symbol table to file");
233 panda::PandArg<std::string> opInputSymbolTable("input-symbol-table", "", "input symbol table file");
234 panda::PandArg<bool> opGeneratePatch("generate-patch", false, "generate patch abc, default as hotfix mode unless "\
235 "the cold-fix argument is set");
236 panda::PandArg<bool> opHotReload("hot-reload", false, "compile as hot-reload mode");
237 panda::PandArg<bool> opColdFix("cold-fix", false, "generate patch abc as cold-fix mode");
238
239 // version
240 panda::PandArg<bool> bcVersion("bc-version", false, "Print ark bytecode version");
241 panda::PandArg<bool> bcMinVersion("bc-min-version", false, "Print ark bytecode minimum supported version");
242
243 // tail arguments
244 panda::PandArg<std::string> inputFile("input", "", "input file");
245
246 argparser_->Add(&opHelp);
247 argparser_->Add(&opModule);
248 argparser_->Add(&opCommonjs);
249 argparser_->Add(&opDumpAst);
250 argparser_->Add(&opDumpTransformedAst);
251 argparser_->Add(&opCheckTransformedAstStructure);
252 argparser_->Add(&opParseOnly);
253 argparser_->Add(&opEnableTypeCheck);
254 argparser_->Add(&opTypeExtractor);
255 argparser_->Add(&opTypeDtsBuiltin);
256 argparser_->Add(&opDumpAssembly);
257 argparser_->Add(&opDebugInfo);
258 argparser_->Add(&opDumpDebugInfo);
259 argparser_->Add(&debuggerEvaluateExpression);
260 argparser_->Add(&base64Input);
261 argparser_->Add(&base64Output);
262
263 argparser_->Add(&opOptLevel);
264 argparser_->Add(&opFunctionThreadCount);
265 argparser_->Add(&opFileThreadCount);
266 argparser_->Add(&opSizeStat);
267 argparser_->Add(&opDumpLiteralBuffer);
268
269 argparser_->Add(&inputExtension);
270 argparser_->Add(&outputFile);
271 argparser_->Add(&sourceFile);
272 argparser_->Add(&recordName);
273 argparser_->Add(&outputProto);
274 argparser_->Add(&opCacheFile);
275 argparser_->Add(&opNpmModuleEntryList);
276 argparser_->Add(&opMergeAbc);
277
278 argparser_->Add(&opDumpSymbolTable);
279 argparser_->Add(&opInputSymbolTable);
280 argparser_->Add(&opGeneratePatch);
281 argparser_->Add(&opHotReload);
282 argparser_->Add(&opColdFix);
283
284 argparser_->Add(&bcVersion);
285 argparser_->Add(&bcMinVersion);
286
287 argparser_->PushBackTail(&inputFile);
288 argparser_->EnableTail();
289 argparser_->EnableRemainder();
290
291 bool parseStatus = argparser_->Parse(argc, argv);
292
293 if (parseStatus && (bcVersion.GetValue() || bcMinVersion.GetValue())) {
294 compilerOptions_.bcVersion = bcVersion.GetValue();
295 compilerOptions_.bcMinVersion = bcMinVersion.GetValue();
296 return true;
297 }
298
299 if (!parseStatus || opHelp.GetValue() || (inputFile.GetValue().empty() && base64Input.GetValue().empty())) {
300 std::stringstream ss;
301
302 ss << argparser_->GetErrorString() << std::endl;
303 ss << "Usage: "
304 << "es2panda"
305 << " [OPTIONS] [input file] -- [arguments]" << std::endl;
306 ss << std::endl;
307 ss << "optional arguments:" << std::endl;
308 ss << argparser_->GetHelpString() << std::endl;
309
310 errorMsg_ = ss.str();
311 return false;
312 }
313
314 bool inputIsEmpty = inputFile.GetValue().empty();
315 bool base64InputIsEmpty = base64Input.GetValue().empty();
316 bool outputIsEmpty = outputFile.GetValue().empty();
317
318 if (!inputIsEmpty && !base64InputIsEmpty) {
319 errorMsg_ = "--input and --base64Input can not be used simultaneously";
320 return false;
321 }
322
323 if (!outputIsEmpty && base64Output.GetValue()) {
324 errorMsg_ = "--output and --base64Output can not be used simultaneously";
325 return false;
326 }
327
328 if (opModule.GetValue() && opCommonjs.GetValue()) {
329 errorMsg_ = "[--module] and [--commonjs] can not be used simultaneously";
330 return false;
331 }
332
333 if (opModule.GetValue()) {
334 scriptKind_ = es2panda::parser::ScriptKind::MODULE;
335 } else if (opCommonjs.GetValue()) {
336 scriptKind_ = es2panda::parser::ScriptKind::COMMONJS;
337 } else {
338 scriptKind_ = es2panda::parser::ScriptKind::SCRIPT;
339 }
340
341 auto parseTypeExtractor = [&opTypeExtractor, &opTypeDtsBuiltin, this]() {
342 compilerOptions_.typeExtractor = opTypeExtractor.GetValue();
343 if (compilerOptions_.typeExtractor) {
344 compilerOptions_.typeDtsBuiltin = opTypeDtsBuiltin.GetValue();
345 DCOUT << "[LOG]TypeExtractor is enabled, type-dts-builtin: " <<
346 compilerOptions_.typeDtsBuiltin << std::endl;
347 }
348 };
349 parseTypeExtractor(); // Type Extractor is only enabled for TypeScript
350
351 std::string extension = inputExtension.GetValue();
352 if (!extension.empty()) {
353 if (VALID_EXTENSIONS.find(extension) == VALID_EXTENSIONS.end()) {
354 errorMsg_ = "Invalid extension (available options: js, ts, as)";
355 return false;
356 }
357 }
358
359 bool isInputFileList = false;
360 if (!inputIsEmpty) {
361 std::string rawInput = inputFile.GetValue();
362 isInputFileList = rawInput[0] == PROCESS_AS_LIST_MARK;
363 std::string input = isInputFileList ? rawInput.substr(1) : rawInput;
364 sourceFile_ = input;
365 }
366
367 if (base64Output.GetValue()) {
368 compilerOutput_ = "";
369 } else if (!outputIsEmpty) {
370 compilerOutput_ = outputFile.GetValue();
371 } else if (outputIsEmpty && !inputIsEmpty) {
372 compilerOutput_ = RemoveExtension(util::Helpers::BaseName(sourceFile_)).append(".abc");
373 }
374
375 if (opMergeAbc.GetValue()) {
376 recordName_ = recordName.GetValue();
377 if (recordName_.empty()) {
378 recordName_ = compilerOutput_.empty() ? "Base64Output" :
379 RemoveExtension(util::Helpers::BaseName(compilerOutput_));
380 }
381 compilerOptions_.mergeAbc = opMergeAbc.GetValue();
382 }
383
384 if (!inputIsEmpty) {
385 // common mode
386 auto inputAbs = panda::os::file::File::GetAbsolutePath(sourceFile_);
387 if (!inputAbs) {
388 std::cerr << "Failed to find: " << sourceFile_ << std::endl;
389 return false;
390 }
391
392 auto fpath = inputAbs.Value();
393 if (isInputFileList) {
394 CollectInputFilesFromFileList(fpath, extension);
395 } else if (panda::os::file::File::IsDirectory(fpath)) {
396 CollectInputFilesFromFileDirectory(fpath, extension);
397 } else {
398 es2panda::SourceFile src(sourceFile_, recordName_, scriptKind_, GetScriptExtension(sourceFile_, extension));
399 sourceFiles_.push_back(src);
400 }
401 } else if (!base64InputIsEmpty) {
402 // input content is base64 string
403 base64Input_ = ExtractContentFromBase64Input(base64Input.GetValue());
404 if (base64Input_.empty()) {
405 errorMsg_ = "The input string is not a valid base64 data";
406 return false;
407 }
408
409 es2panda::SourceFile src("", recordName_, es2panda::parser::ScriptKind::SCRIPT,
410 GetScriptExtensionFromStr(extension));
411 src.source = base64Input_;
412 sourceFiles_.push_back(src);
413 }
414
415 if (!outputProto.GetValue().empty()) {
416 compilerProtoOutput_ = outputProto.GetValue();
417 }
418
419 optLevel_ = opOptLevel.GetValue();
420 functionThreadCount_ = opFunctionThreadCount.GetValue();
421 fileThreadCount_ = opFileThreadCount.GetValue();
422 npmModuleEntryList_ = opNpmModuleEntryList.GetValue();
423
424 if (!opCacheFile.GetValue().empty()) {
425 ParseCacheFileOption(opCacheFile.GetValue());
426 }
427
428 if (opParseOnly.GetValue()) {
429 options_ |= OptionFlags::PARSE_ONLY;
430 }
431
432 if (opSizeStat.GetValue()) {
433 options_ |= OptionFlags::SIZE_STAT;
434 }
435
436 compilerOptions_.dumpAsm = opDumpAssembly.GetValue();
437 compilerOptions_.dumpAst = opDumpAst.GetValue();
438 compilerOptions_.dumpTransformedAst = opDumpTransformedAst.GetValue();
439 compilerOptions_.checkTransformedAstStructure = opCheckTransformedAstStructure.GetValue();
440 compilerOptions_.dumpDebugInfo = opDumpDebugInfo.GetValue();
441 compilerOptions_.isDebug = opDebugInfo.GetValue();
442 compilerOptions_.parseOnly = opParseOnly.GetValue();
443 compilerOptions_.enableTypeCheck = opEnableTypeCheck.GetValue();
444 compilerOptions_.dumpLiteralBuffer = opDumpLiteralBuffer.GetValue();
445 compilerOptions_.isDebuggerEvaluateExpressionMode = debuggerEvaluateExpression.GetValue();
446
447 compilerOptions_.functionThreadCount = functionThreadCount_;
448 compilerOptions_.fileThreadCount = fileThreadCount_;
449 compilerOptions_.output = compilerOutput_;
450 compilerOptions_.debugInfoSourceFile = sourceFile.GetValue();
451 compilerOptions_.optLevel = (compilerOptions_.isDebug || !base64Input.GetValue().empty() ||
452 base64Output.GetValue()) ? 0 : opOptLevel.GetValue();
453 compilerOptions_.sourceFiles = sourceFiles_;
454 compilerOptions_.mergeAbc = opMergeAbc.GetValue();
455
456 compilerOptions_.patchFixOptions.dumpSymbolTable = opDumpSymbolTable.GetValue();
457 compilerOptions_.patchFixOptions.symbolTable = opInputSymbolTable.GetValue();
458
459 bool generatePatch = opGeneratePatch.GetValue();
460 bool hotReload = opHotReload.GetValue();
461 bool coldFix = opColdFix.GetValue();
462 if (generatePatch && hotReload) {
463 errorMsg_ = "--generate-patch and --hot-reload can not be used simultaneously";
464 return false;
465 }
466 if (coldFix && !generatePatch) {
467 errorMsg_ = "--cold-fix can not be used without --generate-patch";
468 return false;
469 }
470 compilerOptions_.patchFixOptions.generatePatch = generatePatch;
471 compilerOptions_.patchFixOptions.hotReload = hotReload;
472 compilerOptions_.patchFixOptions.coldFix = coldFix;
473
474 return true;
475 }
476
ExtractContentFromBase64Input(const std::string & inputBase64String)477 std::string Options::ExtractContentFromBase64Input(const std::string &inputBase64String)
478 {
479 std::string inputContent = util::Base64Decode(inputBase64String);
480 if (inputContent == "") {
481 return "";
482 }
483 bool validBase64Input = util::Base64Encode(inputContent) == inputBase64String;
484 if (!validBase64Input) {
485 return "";
486 }
487 return inputContent;
488 }
489 } // namespace panda::es2panda::aot
490