1 /*
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "options.h"
17
18 #include <fstream>
19 #include <set>
20 #include <sstream>
21 #include <utility>
22
23 #if defined(PANDA_TARGET_WINDOWS)
24 #include <io.h>
25 #else
26 #include <dirent.h>
27 #endif
28
29 #include "os/file.h"
30
31 #include "mergeProgram.h"
32 #include "util/helpers.h"
33 #include "utils/pandargs.h"
34
35 namespace panda::es2panda::aot {
36 constexpr char PROCESS_AS_LIST_MARK = '@';
37 const std::string LIST_ITEM_SEPERATOR = ";";
38 const std::set<std::string> VALID_EXTENSIONS = { "js", "ts", "as" };
39
40 template <class T>
RemoveExtension(T const & filename)41 T RemoveExtension(T const &filename)
42 {
43 typename T::size_type const P(filename.find_last_of('.'));
44 return P > 0 && P != T::npos ? filename.substr(0, P) : filename;
45 }
46
GetScriptExtensionFromStr(const std::string & extension)47 static es2panda::ScriptExtension GetScriptExtensionFromStr(const std::string &extension)
48 {
49 if (extension == "js") {
50 return es2panda::ScriptExtension::JS;
51 } else if (extension == "ts") {
52 return es2panda::ScriptExtension::TS;
53 } else if (extension == "as") {
54 return es2panda::ScriptExtension::AS;
55 } else {
56 return es2panda::ScriptExtension::JS;
57 }
58 }
59
GetScriptExtension(const std::string & filename,const std::string & inputExtension)60 static es2panda::ScriptExtension GetScriptExtension(const std::string &filename, const std::string &inputExtension)
61 {
62 std::string fileExtension = "";
63 std::string::size_type pos(filename.find_last_of('.'));
64 if (pos > 0 && pos != std::string::npos) {
65 fileExtension = filename.substr(pos + 1);
66 }
67
68 if (VALID_EXTENSIONS.find(fileExtension) != VALID_EXTENSIONS.end()) {
69 return GetScriptExtensionFromStr(fileExtension);
70 }
71
72 return GetScriptExtensionFromStr(inputExtension);
73 }
74
GetStringItems(std::string & input,const std::string & delimiter)75 static std::vector<std::string> GetStringItems(std::string &input, const std::string &delimiter)
76 {
77 std::vector<std::string> items;
78 size_t pos = 0;
79 std::string token;
80 while ((pos = input.find(delimiter)) != std::string::npos) {
81 token = input.substr(0, pos);
82 if (!token.empty()) {
83 items.push_back(token);
84 }
85 input.erase(0, pos + delimiter.length());
86 }
87 if (!input.empty()) {
88 items.push_back(input);
89 }
90 return items;
91 }
92
93 // Options
CollectInputFilesFromFileList(const std::string & input,const std::string & inputExtension)94 bool Options::CollectInputFilesFromFileList(const std::string &input, const std::string &inputExtension)
95 {
96 std::ifstream ifs;
97 std::string line;
98 ifs.open(panda::os::file::File::GetExtendedFilePath(input));
99 if (!ifs.is_open()) {
100 std::cerr << "Failed to open source list: " << input << std::endl;
101 return false;
102 }
103
104 constexpr size_t ITEM_COUNT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile, pkgName]
105 constexpr size_t ITEM_COUNT_NOT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile; outputfile]
106 while (std::getline(ifs, line)) {
107 std::vector<std::string> itemList = GetStringItems(line, LIST_ITEM_SEPERATOR);
108 if ((compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_MERGE) ||
109 (!compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_NOT_MERGE)) {
110 std::cerr << "Failed to parse input file" << std::endl;
111 return false;
112 }
113
114 std::string fileName = itemList[0];
115 std::string recordName = compilerOptions_.mergeAbc ? itemList[1] : "";
116 parser::ScriptKind scriptKind;
117 if (itemList[2] == "script") {
118 scriptKind = parser::ScriptKind::SCRIPT;
119 } else if (itemList[2] == "commonjs") {
120 scriptKind = parser::ScriptKind::COMMONJS;
121 } else {
122 scriptKind = parser::ScriptKind::MODULE;
123 }
124
125 es2panda::SourceFile src(fileName, recordName, scriptKind, GetScriptExtension(fileName, inputExtension));
126 src.sourcefile = itemList[3];
127 if (compilerOptions_.mergeAbc) {
128 src.pkgName = itemList[4];
129 }
130
131 sourceFiles_.push_back(src);
132 if (!compilerOptions_.mergeAbc) {
133 outputFiles_.insert({fileName, itemList[4]});
134 }
135 }
136 return true;
137 }
138
CollectInputFilesFromFileDirectory(const std::string & input,const std::string & extension)139 bool Options::CollectInputFilesFromFileDirectory(const std::string &input, const std::string &extension)
140 {
141 std::vector<std::string> files;
142 if (!proto::MergeProgram::GetProtoFiles(input, extension, files)) {
143 return false;
144 }
145
146 for (const auto &f : files) {
147 es2panda::SourceFile src(f, RemoveExtension(f.substr(input.length() + 1)),
148 scriptKind_, GetScriptExtensionFromStr(extension));
149 sourceFiles_.push_back(src);
150 }
151
152 return true;
153 }
154
ParseCacheFileOption(const std::string & cacheInput)155 void Options::ParseCacheFileOption(const std::string &cacheInput)
156 {
157 if (cacheInput[0] != PROCESS_AS_LIST_MARK) {
158 compilerOptions_.cacheFiles.insert({sourceFile_, cacheInput});
159 return;
160 }
161
162 std::ifstream ifs;
163 std::string line;
164 ifs.open(panda::os::file::File::GetExtendedFilePath(cacheInput.substr(1)));
165 if (!ifs.is_open()) {
166 std::cerr << "Failed to open cache file list: " << cacheInput << std::endl;
167 return;
168 }
169
170 constexpr int cacheListItemCount = 2;
171 while (std::getline(ifs, line)) {
172 std::vector<std::string> itemList = GetStringItems(line, LIST_ITEM_SEPERATOR);
173 if (itemList.size() != cacheListItemCount) {
174 continue;
175 }
176 compilerOptions_.cacheFiles.insert({itemList[0], itemList[1]});
177 }
178 }
179
Options()180 Options::Options() : argparser_(new panda::PandArgParser()) {}
181
~Options()182 Options::~Options()
183 {
184 delete argparser_;
185 }
186
Parse(int argc,const char ** argv)187 bool Options::Parse(int argc, const char **argv)
188 {
189 panda::PandArg<bool> opHelp("help", false, "Print this message and exit");
190
191 // parser
192 panda::PandArg<std::string> inputExtension("extension", "js",
193 "Parse the input as the given extension (options: js | ts | as)");
194 panda::PandArg<bool> opModule("module", false, "Parse the input as module");
195 panda::PandArg<bool> opCommonjs("commonjs", false, "Parse the input as commonjs");
196 panda::PandArg<bool> opParseOnly("parse-only", false, "Parse the input only");
197 panda::PandArg<bool> opEnableTypeCheck("enable-type-check", false, "Check the type in ts after parse");
198 panda::PandArg<bool> opDumpAst("dump-ast", false, "Dump the parsed AST");
199 panda::PandArg<bool> opDumpTransformedAst("dump-transformed-ast", false, "Dump the parsed AST after transform");
200 panda::PandArg<bool> opCheckTransformedAstStructure("check-transformed-ast-structure", false,
201 "Check the AST structure after transform");
202 panda::PandArg<bool> opRecordSource("record-source", false, "Record all functions' source codes to support the "\
203 "using of [function].toString()");
204
205 // type extractor
206 panda::PandArg<bool> opTypeExtractor("type-extractor", false, "Enable type extractor for typescript");
207 panda::PandArg<bool> opTypeDtsBuiltin("type-dts-builtin", false, "Enable builtin type extractor for .d.ts file");
208
209 // compiler
210 panda::PandArg<bool> opDumpAssembly("dump-assembly", false, "Dump pandasm");
211 panda::PandArg<bool> opDebugInfo("debug-info", false, "Compile with debug info");
212 panda::PandArg<bool> opDumpDebugInfo("dump-debug-info", false, "Dump debug info");
213 panda::PandArg<int> opOptLevel("opt-level", 2,
214 "Compiler optimization level (options: 0 | 1 | 2). In debug and base64Input mode, optimizer is disabled");
215 panda::PandArg<int> opFunctionThreadCount("function-threads", 0, "Number of worker threads to compile function");
216 panda::PandArg<int> opFileThreadCount("file-threads", 0, "Number of worker threads to compile file");
217 panda::PandArg<bool> opSizeStat("dump-size-stat", false, "Dump size statistics");
218 panda::PandArg<bool> opDumpLiteralBuffer("dump-literal-buffer", false, "Dump literal buffer");
219 panda::PandArg<std::string> outputFile("output", "", "Compiler binary output (.abc)");
220 panda::PandArg<std::string> recordName("record-name", "", "Specify the record name");
221 panda::PandArg<bool> debuggerEvaluateExpression("debugger-evaluate-expression", false,
222 "evaluate expression in debugger mode");
223 panda::PandArg<std::string> base64Input("base64Input", "", "base64 input of js content");
224 panda::PandArg<bool> base64Output("base64Output", false, "output panda file content as base64 to std out");
225 panda::PandArg<std::string> sourceFile("source-file", "",
226 "specify the file path info recorded in generated abc");
227 panda::PandArg<std::string> outputProto("outputProto", "",
228 "specify the output name for serializd protobuf file (.protoBin)");
229 panda::PandArg<std::string> opCacheFile("cache-file", "", "cache file for incremental compile");
230 panda::PandArg<std::string> opNpmModuleEntryList("npm-module-entry-list", "", "entry list file for module compile");
231 panda::PandArg<bool> opMergeAbc("merge-abc", false, "Compile as merge abc");
232 panda::PandArg<bool> opuseDefineSemantic("use-define-semantic", false, "Compile ts class fields "\
233 "in accordance with ECMAScript2022");
234
235 // patchfix && hotreload
236 panda::PandArg<std::string> opDumpSymbolTable("dump-symbol-table", "", "dump symbol table to file");
237 panda::PandArg<std::string> opInputSymbolTable("input-symbol-table", "", "input symbol table file");
238 panda::PandArg<bool> opGeneratePatch("generate-patch", false, "generate patch abc, default as hotfix mode unless "\
239 "the cold-fix argument is set");
240 panda::PandArg<bool> opHotReload("hot-reload", false, "compile as hot-reload mode");
241 panda::PandArg<bool> opColdFix("cold-fix", false, "generate patch abc as cold-fix mode");
242
243 // version
244 panda::PandArg<bool> bcVersion("bc-version", false, "Print ark bytecode version");
245 panda::PandArg<bool> bcMinVersion("bc-min-version", false, "Print ark bytecode minimum supported version");
246 panda::PandArg<int> targetApiVersion("target-api-version", 11, "Specify the targeting api version for es2abc to "\
247 "generated the corresponding version of bytecode");
248
249 // tail arguments
250 panda::PandArg<std::string> inputFile("input", "", "input file");
251
252 argparser_->Add(&opHelp);
253 argparser_->Add(&opModule);
254 argparser_->Add(&opCommonjs);
255 argparser_->Add(&opDumpAst);
256 argparser_->Add(&opDumpTransformedAst);
257 argparser_->Add(&opCheckTransformedAstStructure);
258 argparser_->Add(&opRecordSource);
259 argparser_->Add(&opParseOnly);
260 argparser_->Add(&opEnableTypeCheck);
261 argparser_->Add(&opTypeExtractor);
262 argparser_->Add(&opTypeDtsBuiltin);
263 argparser_->Add(&opDumpAssembly);
264 argparser_->Add(&opDebugInfo);
265 argparser_->Add(&opDumpDebugInfo);
266 argparser_->Add(&debuggerEvaluateExpression);
267 argparser_->Add(&base64Input);
268 argparser_->Add(&base64Output);
269
270 argparser_->Add(&opOptLevel);
271 argparser_->Add(&opFunctionThreadCount);
272 argparser_->Add(&opFileThreadCount);
273 argparser_->Add(&opSizeStat);
274 argparser_->Add(&opDumpLiteralBuffer);
275
276 argparser_->Add(&inputExtension);
277 argparser_->Add(&outputFile);
278 argparser_->Add(&sourceFile);
279 argparser_->Add(&recordName);
280 argparser_->Add(&outputProto);
281 argparser_->Add(&opCacheFile);
282 argparser_->Add(&opNpmModuleEntryList);
283 argparser_->Add(&opMergeAbc);
284 argparser_->Add(&opuseDefineSemantic);
285
286 argparser_->Add(&opDumpSymbolTable);
287 argparser_->Add(&opInputSymbolTable);
288 argparser_->Add(&opGeneratePatch);
289 argparser_->Add(&opHotReload);
290 argparser_->Add(&opColdFix);
291
292 argparser_->Add(&bcVersion);
293 argparser_->Add(&bcMinVersion);
294 argparser_->Add(&targetApiVersion);
295
296 argparser_->PushBackTail(&inputFile);
297 argparser_->EnableTail();
298 argparser_->EnableRemainder();
299
300 bool parseStatus = argparser_->Parse(argc, argv);
301 if (parseStatus && (bcVersion.GetValue() || bcMinVersion.GetValue())) {
302 compilerOptions_.bcVersion = bcVersion.GetValue();
303 compilerOptions_.bcMinVersion = bcMinVersion.GetValue();
304 return true;
305 }
306
307 if (!parseStatus || opHelp.GetValue() || (inputFile.GetValue().empty() && base64Input.GetValue().empty())) {
308 std::stringstream ss;
309
310 ss << argparser_->GetErrorString() << std::endl;
311 ss << "Usage: "
312 << "es2panda"
313 << " [OPTIONS] [input file] -- [arguments]" << std::endl;
314 ss << std::endl;
315 ss << "optional arguments:" << std::endl;
316 ss << argparser_->GetHelpString() << std::endl;
317
318 errorMsg_ = ss.str();
319 return false;
320 }
321
322 bool inputIsEmpty = inputFile.GetValue().empty();
323 bool base64InputIsEmpty = base64Input.GetValue().empty();
324 bool outputIsEmpty = outputFile.GetValue().empty();
325
326 if (!inputIsEmpty && !base64InputIsEmpty) {
327 errorMsg_ = "--input and --base64Input can not be used simultaneously";
328 return false;
329 }
330
331 if (!outputIsEmpty && base64Output.GetValue()) {
332 errorMsg_ = "--output and --base64Output can not be used simultaneously";
333 return false;
334 }
335
336 if (opModule.GetValue() && opCommonjs.GetValue()) {
337 errorMsg_ = "[--module] and [--commonjs] can not be used simultaneously";
338 return false;
339 }
340
341 if (opModule.GetValue()) {
342 scriptKind_ = es2panda::parser::ScriptKind::MODULE;
343 } else if (opCommonjs.GetValue()) {
344 scriptKind_ = es2panda::parser::ScriptKind::COMMONJS;
345 } else {
346 scriptKind_ = es2panda::parser::ScriptKind::SCRIPT;
347 }
348
349 auto parseTypeExtractor = [&opTypeExtractor, &opTypeDtsBuiltin, this]() {
350 compilerOptions_.typeExtractor = opTypeExtractor.GetValue();
351 if (compilerOptions_.typeExtractor) {
352 compilerOptions_.typeDtsBuiltin = opTypeDtsBuiltin.GetValue();
353 DCOUT << "[LOG]TypeExtractor is enabled, type-dts-builtin: " <<
354 compilerOptions_.typeDtsBuiltin << std::endl;
355 }
356 };
357 parseTypeExtractor(); // Type Extractor is only enabled for TypeScript
358
359 std::string extension = inputExtension.GetValue();
360 if (!extension.empty()) {
361 if (VALID_EXTENSIONS.find(extension) == VALID_EXTENSIONS.end()) {
362 errorMsg_ = "Invalid extension (available options: js, ts, as)";
363 return false;
364 }
365 }
366
367 bool isInputFileList = false;
368 if (!inputIsEmpty) {
369 std::string rawInput = inputFile.GetValue();
370 isInputFileList = rawInput[0] == PROCESS_AS_LIST_MARK;
371 std::string input = isInputFileList ? rawInput.substr(1) : rawInput;
372 sourceFile_ = input;
373 }
374
375 if (base64Output.GetValue()) {
376 compilerOutput_ = "";
377 } else if (!outputIsEmpty) {
378 compilerOutput_ = outputFile.GetValue();
379 } else if (outputIsEmpty && !inputIsEmpty) {
380 compilerOutput_ = RemoveExtension(util::Helpers::BaseName(sourceFile_)).append(".abc");
381 }
382
383 if (opMergeAbc.GetValue()) {
384 recordName_ = recordName.GetValue();
385 if (recordName_.empty()) {
386 recordName_ = compilerOutput_.empty() ? "Base64Output" :
387 RemoveExtension(util::Helpers::BaseName(compilerOutput_));
388 }
389 compilerOptions_.mergeAbc = opMergeAbc.GetValue();
390 }
391
392 if (opuseDefineSemantic.GetValue()) {
393 compilerOptions_.useDefineSemantic = opuseDefineSemantic.GetValue();
394 }
395
396 if (!inputIsEmpty) {
397 // common mode
398 auto inputAbs = panda::os::file::File::GetAbsolutePath(sourceFile_);
399 if (!inputAbs) {
400 std::cerr << "Failed to find: " << sourceFile_ << std::endl;
401 return false;
402 }
403
404 auto fpath = inputAbs.Value();
405 if (isInputFileList) {
406 CollectInputFilesFromFileList(fpath, extension);
407 } else if (panda::os::file::File::IsDirectory(fpath)) {
408 CollectInputFilesFromFileDirectory(fpath, extension);
409 } else {
410 es2panda::SourceFile src(sourceFile_, recordName_, scriptKind_, GetScriptExtension(sourceFile_, extension));
411 sourceFiles_.push_back(src);
412 }
413 } else if (!base64InputIsEmpty) {
414 // input content is base64 string
415 base64Input_ = ExtractContentFromBase64Input(base64Input.GetValue());
416 if (base64Input_.empty()) {
417 errorMsg_ = "The input string is not a valid base64 data";
418 return false;
419 }
420
421 es2panda::SourceFile src("", recordName_, es2panda::parser::ScriptKind::SCRIPT,
422 GetScriptExtensionFromStr(extension));
423 src.source = base64Input_;
424 sourceFiles_.push_back(src);
425 }
426
427 if (!outputProto.GetValue().empty()) {
428 compilerProtoOutput_ = outputProto.GetValue();
429 }
430
431 optLevel_ = opOptLevel.GetValue();
432 functionThreadCount_ = opFunctionThreadCount.GetValue();
433 fileThreadCount_ = opFileThreadCount.GetValue();
434 npmModuleEntryList_ = opNpmModuleEntryList.GetValue();
435
436 if (!opCacheFile.GetValue().empty()) {
437 ParseCacheFileOption(opCacheFile.GetValue());
438 }
439
440 if (opParseOnly.GetValue()) {
441 options_ |= OptionFlags::PARSE_ONLY;
442 }
443
444 if (opSizeStat.GetValue()) {
445 options_ |= OptionFlags::SIZE_STAT;
446 }
447
448 compilerOptions_.recordSource = opRecordSource.GetValue();
449 compilerOptions_.dumpAsm = opDumpAssembly.GetValue();
450 compilerOptions_.dumpAst = opDumpAst.GetValue();
451 compilerOptions_.dumpTransformedAst = opDumpTransformedAst.GetValue();
452 compilerOptions_.checkTransformedAstStructure = opCheckTransformedAstStructure.GetValue();
453 compilerOptions_.dumpDebugInfo = opDumpDebugInfo.GetValue();
454 compilerOptions_.isDebug = opDebugInfo.GetValue();
455 compilerOptions_.parseOnly = opParseOnly.GetValue();
456 compilerOptions_.enableTypeCheck = opEnableTypeCheck.GetValue();
457 compilerOptions_.dumpLiteralBuffer = opDumpLiteralBuffer.GetValue();
458 compilerOptions_.isDebuggerEvaluateExpressionMode = debuggerEvaluateExpression.GetValue();
459
460 compilerOptions_.functionThreadCount = functionThreadCount_;
461 compilerOptions_.fileThreadCount = fileThreadCount_;
462 compilerOptions_.output = compilerOutput_;
463 compilerOptions_.debugInfoSourceFile = sourceFile.GetValue();
464 compilerOptions_.optLevel = (compilerOptions_.isDebug || !base64Input.GetValue().empty() ||
465 base64Output.GetValue()) ? 0 : opOptLevel.GetValue();
466 compilerOptions_.sourceFiles = sourceFiles_;
467 compilerOptions_.mergeAbc = opMergeAbc.GetValue();
468 compilerOptions_.targetApiVersion = targetApiVersion.GetValue();
469
470 compilerOptions_.patchFixOptions.dumpSymbolTable = opDumpSymbolTable.GetValue();
471 compilerOptions_.patchFixOptions.symbolTable = opInputSymbolTable.GetValue();
472
473 bool generatePatch = opGeneratePatch.GetValue();
474 bool hotReload = opHotReload.GetValue();
475 bool coldFix = opColdFix.GetValue();
476 if (generatePatch && hotReload) {
477 errorMsg_ = "--generate-patch and --hot-reload can not be used simultaneously";
478 return false;
479 }
480 if (coldFix && !generatePatch) {
481 errorMsg_ = "--cold-fix can not be used without --generate-patch";
482 return false;
483 }
484 compilerOptions_.patchFixOptions.generatePatch = generatePatch;
485 compilerOptions_.patchFixOptions.hotReload = hotReload;
486 compilerOptions_.patchFixOptions.coldFix = coldFix;
487
488 return true;
489 }
490
ExtractContentFromBase64Input(const std::string & inputBase64String)491 std::string Options::ExtractContentFromBase64Input(const std::string &inputBase64String)
492 {
493 std::string inputContent = util::Base64Decode(inputBase64String);
494 if (inputContent == "") {
495 return "";
496 }
497 bool validBase64Input = util::Base64Encode(inputContent) == inputBase64String;
498 if (!validBase64Input) {
499 return "";
500 }
501 return inputContent;
502 }
503 } // namespace panda::es2panda::aot
504