1 /*
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "options.h"
17
18 #include "mergeProgram.h"
19 #include "os/file.h"
20 #include <util/helpers.h>
21 #include <utils/pandargs.h>
22 #if defined(PANDA_TARGET_WINDOWS)
23 #include <io.h>
24 #else
25 #include <dirent.h>
26 #endif
27
28 #include <fstream>
29 #include <sstream>
30 #include <utility>
31
32 namespace panda::es2panda::aot {
33 constexpr char PROCESS_AS_LIST_MARK = '@';
34 const std::string LIST_ITEM_SEPERATOR = ";";
35
36 template <class T>
RemoveExtension(T const & filename)37 T RemoveExtension(T const &filename)
38 {
39 typename T::size_type const P(filename.find_last_of('.'));
40 return P > 0 && P != T::npos ? filename.substr(0, P) : filename;
41 }
42
GetStringItems(std::string & input,const std::string & delimiter)43 static std::vector<std::string> GetStringItems(std::string &input, const std::string &delimiter)
44 {
45 std::vector<std::string> items;
46 size_t pos = 0;
47 std::string token;
48 while ((pos = input.find(delimiter)) != std::string::npos) {
49 token = input.substr(0, pos);
50 if (!token.empty()) {
51 items.push_back(token);
52 }
53 input.erase(0, pos + delimiter.length());
54 }
55 if (!input.empty()) {
56 items.push_back(input);
57 }
58 return items;
59 }
60
61 // Options
CollectInputFilesFromFileList(const std::string & input)62 bool Options::CollectInputFilesFromFileList(const std::string &input)
63 {
64 std::ifstream ifs;
65 std::string line;
66 ifs.open(panda::os::file::File::GetExtendedFilePath(input));
67 if (!ifs.is_open()) {
68 std::cerr << "Failed to open source list: " << input << std::endl;
69 return false;
70 }
71
72 constexpr size_t ITEM_COUNT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile, pkgName]
73 constexpr size_t ITEM_COUNT_NOT_MERGE = 5; // item list: [filePath; recordName; moduleKind; sourceFile; outputfile]
74 while (std::getline(ifs, line)) {
75 std::vector<std::string> itemList = GetStringItems(line, LIST_ITEM_SEPERATOR);
76 if ((compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_MERGE) ||
77 (!compilerOptions_.mergeAbc && itemList.size() != ITEM_COUNT_NOT_MERGE)) {
78 std::cerr << "Failed to parse input file" << std::endl;
79 return false;
80 }
81
82 std::string fileName = itemList[0];
83 std::string recordName = compilerOptions_.mergeAbc ? itemList[1] : "";
84 parser::ScriptKind scriptKind;
85 if (itemList[2] == "script") {
86 scriptKind = parser::ScriptKind::SCRIPT;
87 } else if (itemList[2] == "commonjs") {
88 scriptKind = parser::ScriptKind::COMMONJS;
89 } else {
90 scriptKind = parser::ScriptKind::MODULE;
91 }
92
93 es2panda::SourceFile src(fileName, recordName, scriptKind);
94 src.sourcefile = itemList[3];
95 if (compilerOptions_.mergeAbc) {
96 src.pkgName = itemList[4];
97 }
98
99 sourceFiles_.push_back(src);
100 if (!compilerOptions_.mergeAbc) {
101 outputFiles_.insert({fileName, itemList[4]});
102 }
103 }
104 return true;
105 }
106
CollectInputFilesFromFileDirectory(const std::string & input,const std::string & extension)107 bool Options::CollectInputFilesFromFileDirectory(const std::string &input, const std::string &extension)
108 {
109 std::vector<std::string> files;
110 if (!proto::MergeProgram::GetProtoFiles(input, extension, files)) {
111 return false;
112 }
113 for (auto &f : files) {
114 es2panda::SourceFile src(f, util::Helpers::BaseName(f), scriptKind_);
115 sourceFiles_.push_back(src);
116 }
117
118 return true;
119 }
120
ParseCacheFileOption(const std::string & cacheInput)121 void Options::ParseCacheFileOption(const std::string &cacheInput)
122 {
123 if (cacheInput[0] != PROCESS_AS_LIST_MARK) {
124 compilerOptions_.cacheFiles.insert({sourceFile_, cacheInput});
125 return;
126 }
127
128 std::ifstream ifs;
129 std::string line;
130 ifs.open(panda::os::file::File::GetExtendedFilePath(cacheInput.substr(1)));
131 if (!ifs.is_open()) {
132 std::cerr << "Failed to open cache file list: " << cacheInput << std::endl;
133 return;
134 }
135
136 constexpr int cacheListItemCount = 2;
137 while (std::getline(ifs, line)) {
138 std::vector<std::string> itemList = GetStringItems(line, LIST_ITEM_SEPERATOR);
139 if (itemList.size() != cacheListItemCount) {
140 continue;
141 }
142 compilerOptions_.cacheFiles.insert({itemList[0], itemList[1]});
143 }
144 }
145
Options()146 Options::Options() : argparser_(new panda::PandArgParser()) {}
147
~Options()148 Options::~Options()
149 {
150 delete argparser_;
151 }
152
Parse(int argc,const char ** argv)153 bool Options::Parse(int argc, const char **argv)
154 {
155 panda::PandArg<bool> opHelp("help", false, "Print this message and exit");
156
157 // parser
158 panda::PandArg<std::string> inputExtension("extension", "js",
159 "Parse the input as the given extension (options: js | ts | as)");
160 panda::PandArg<bool> opModule("module", false, "Parse the input as module");
161 panda::PandArg<bool> opCommonjs("commonjs", false, "Parse the input as commonjs");
162 panda::PandArg<bool> opParseOnly("parse-only", false, "Parse the input only");
163 panda::PandArg<bool> opEnableTypeCheck("enable-type-check", false, "Check the type in ts after parse");
164 panda::PandArg<bool> opDumpAst("dump-ast", false, "Dump the parsed AST");
165
166 // type extractor
167 panda::PandArg<bool> opTypeExtractor("type-extractor", false, "Enable type extractor for typescript");
168 panda::PandArg<bool> opTypeDtsBuiltin("type-dts-builtin", false, "Enable builtin type extractor for .d.ts file");
169
170 // compiler
171 panda::PandArg<bool> opDumpAssembly("dump-assembly", false, "Dump pandasm");
172 panda::PandArg<bool> opDebugInfo("debug-info", false, "Compile with debug info");
173 panda::PandArg<bool> opDumpDebugInfo("dump-debug-info", false, "Dump debug info");
174 panda::PandArg<int> opOptLevel("opt-level", 2,
175 "Compiler optimization level (options: 0 | 1 | 2). In debug and base64Input mode, optimizer is disabled");
176 panda::PandArg<int> opFunctionThreadCount("function-threads", 0, "Number of worker threads to compile function");
177 panda::PandArg<int> opFileThreadCount("file-threads", 0, "Number of worker threads to compile file");
178 panda::PandArg<bool> opSizeStat("dump-size-stat", false, "Dump size statistics");
179 panda::PandArg<bool> opDumpLiteralBuffer("dump-literal-buffer", false, "Dump literal buffer");
180 panda::PandArg<std::string> outputFile("output", "", "Compiler binary output (.abc)");
181 panda::PandArg<std::string> recordName("record-name", "", "Specify the record name");
182 panda::PandArg<bool> debuggerEvaluateExpression("debugger-evaluate-expression", false,
183 "evaluate expression in debugger mode");
184 panda::PandArg<std::string> base64Input("base64Input", "", "base64 input of js content");
185 panda::PandArg<bool> base64Output("base64Output", false, "output panda file content as base64 to std out");
186 panda::PandArg<std::string> sourceFile("source-file", "",
187 "specify the file path info recorded in generated abc");
188 panda::PandArg<std::string> outputProto("outputProto", "",
189 "specify the output name for serializd protobuf file (.protoBin)");
190 panda::PandArg<std::string> opCacheFile("cache-file", "", "cache file for incremental compile");
191 panda::PandArg<std::string> opNpmModuleEntryList("npm-module-entry-list", "", "entry list file for module compile");
192 panda::PandArg<bool> opMergeAbc("merge-abc", false, "Compile as merge abc");
193
194 // hotfix && hotreload
195 panda::PandArg<std::string> opDumpSymbolTable("dump-symbol-table", "", "dump symbol table to file");
196 panda::PandArg<std::string> opInputSymbolTable("input-symbol-table", "", "input symbol table file");
197 panda::PandArg<bool> opGeneratePatch("generate-patch", false, "generate patch abc");
198 panda::PandArg<bool> opHotReload("hot-reload", false, "compile as hot-reload mode");
199
200 // version
201 panda::PandArg<bool> bcVersion("bc-version", false, "Print ark bytecode version");
202 panda::PandArg<bool> bcMinVersion("bc-min-version", false, "Print ark bytecode minimum supported version");
203
204 // tail arguments
205 panda::PandArg<std::string> inputFile("input", "", "input file");
206
207 argparser_->Add(&opHelp);
208 argparser_->Add(&opModule);
209 argparser_->Add(&opCommonjs);
210 argparser_->Add(&opDumpAst);
211 argparser_->Add(&opParseOnly);
212 argparser_->Add(&opEnableTypeCheck);
213 argparser_->Add(&opTypeExtractor);
214 argparser_->Add(&opTypeDtsBuiltin);
215 argparser_->Add(&opDumpAssembly);
216 argparser_->Add(&opDebugInfo);
217 argparser_->Add(&opDumpDebugInfo);
218 argparser_->Add(&debuggerEvaluateExpression);
219 argparser_->Add(&base64Input);
220 argparser_->Add(&base64Output);
221
222 argparser_->Add(&opOptLevel);
223 argparser_->Add(&opFunctionThreadCount);
224 argparser_->Add(&opFileThreadCount);
225 argparser_->Add(&opSizeStat);
226 argparser_->Add(&opDumpLiteralBuffer);
227
228 argparser_->Add(&inputExtension);
229 argparser_->Add(&outputFile);
230 argparser_->Add(&sourceFile);
231 argparser_->Add(&recordName);
232 argparser_->Add(&outputProto);
233 argparser_->Add(&opCacheFile);
234 argparser_->Add(&opNpmModuleEntryList);
235 argparser_->Add(&opMergeAbc);
236
237 argparser_->Add(&opDumpSymbolTable);
238 argparser_->Add(&opInputSymbolTable);
239 argparser_->Add(&opGeneratePatch);
240 argparser_->Add(&opHotReload);
241
242 argparser_->Add(&bcVersion);
243 argparser_->Add(&bcMinVersion);
244
245 argparser_->PushBackTail(&inputFile);
246 argparser_->EnableTail();
247 argparser_->EnableRemainder();
248
249 bool parseStatus = argparser_->Parse(argc, argv);
250
251 if (parseStatus && (bcVersion.GetValue() || bcMinVersion.GetValue())) {
252 compilerOptions_.bcVersion = bcVersion.GetValue();
253 compilerOptions_.bcMinVersion = bcMinVersion.GetValue();
254 return true;
255 }
256
257 if (!parseStatus || opHelp.GetValue() || (inputFile.GetValue().empty() && base64Input.GetValue().empty())) {
258 std::stringstream ss;
259
260 ss << argparser_->GetErrorString() << std::endl;
261 ss << "Usage: "
262 << "es2panda"
263 << " [OPTIONS] [input file] -- [arguments]" << std::endl;
264 ss << std::endl;
265 ss << "optional arguments:" << std::endl;
266 ss << argparser_->GetHelpString() << std::endl;
267
268 errorMsg_ = ss.str();
269 return false;
270 }
271
272 bool inputIsEmpty = inputFile.GetValue().empty();
273 bool base64InputIsEmpty = base64Input.GetValue().empty();
274 bool outputIsEmpty = outputFile.GetValue().empty();
275
276 if (!inputIsEmpty && !base64InputIsEmpty) {
277 errorMsg_ = "--input and --base64Input can not be used simultaneously";
278 return false;
279 }
280
281 if (!outputIsEmpty && base64Output.GetValue()) {
282 errorMsg_ = "--output and --base64Output can not be used simultaneously";
283 return false;
284 }
285
286 if (opModule.GetValue() && opCommonjs.GetValue()) {
287 errorMsg_ = "[--module] and [--commonjs] can not be used simultaneously";
288 return false;
289 }
290
291 if (opModule.GetValue()) {
292 scriptKind_ = es2panda::parser::ScriptKind::MODULE;
293 } else if (opCommonjs.GetValue()) {
294 scriptKind_ = es2panda::parser::ScriptKind::COMMONJS;
295 } else {
296 scriptKind_ = es2panda::parser::ScriptKind::SCRIPT;
297 }
298
299 auto parseTypeExtractor = [&opTypeExtractor, &opTypeDtsBuiltin, this]() {
300 compilerOptions_.typeExtractor = opTypeExtractor.GetValue();
301 if (compilerOptions_.typeExtractor) {
302 compilerOptions_.typeDtsBuiltin = opTypeDtsBuiltin.GetValue();
303 #ifndef NDEBUG
304 std::cout << "[LOG]TypeExtractor is enabled, type-dts-builtin: " <<
305 compilerOptions_.typeDtsBuiltin << std::endl;
306 #endif
307 }
308 };
309
310 std::string extension = inputExtension.GetValue();
311 if (!extension.empty()) {
312 if (extension == "js") {
313 extension_ = es2panda::ScriptExtension::JS;
314 } else if (extension == "ts") {
315 extension_ = es2panda::ScriptExtension::TS;
316 // Type Extractor is only enabled for TypeScript
317 parseTypeExtractor();
318 } else if (extension == "as") {
319 extension_ = es2panda::ScriptExtension::AS;
320 } else {
321 errorMsg_ = "Invalid extension (available options: js, ts, as)";
322 return false;
323 }
324 }
325
326 bool isInputFileList = false;
327 if (!inputIsEmpty) {
328 std::string rawInput = inputFile.GetValue();
329 isInputFileList = rawInput[0] == PROCESS_AS_LIST_MARK;
330 std::string input = isInputFileList ? rawInput.substr(1) : rawInput;
331 sourceFile_ = input;
332 }
333
334 if (base64Output.GetValue()) {
335 compilerOutput_ = "";
336 } else if (!outputIsEmpty) {
337 compilerOutput_ = outputFile.GetValue();
338 } else if (outputIsEmpty && !inputIsEmpty) {
339 compilerOutput_ = RemoveExtension(util::Helpers::BaseName(sourceFile_)).append(".abc");
340 }
341
342 if (opMergeAbc.GetValue()) {
343 recordName_ = recordName.GetValue();
344 if (recordName_.empty()) {
345 recordName_ = compilerOutput_.empty() ? "Base64Output" :
346 RemoveExtension(util::Helpers::BaseName(compilerOutput_));
347 }
348 compilerOptions_.mergeAbc = opMergeAbc.GetValue();
349 }
350
351 if (!inputIsEmpty) {
352 // common mode
353 auto inputAbs = panda::os::file::File::GetAbsolutePath(sourceFile_);
354 if (!inputAbs) {
355 std::cerr << "Failed to find: " << sourceFile_ << std::endl;
356 return false;
357 }
358
359 auto fpath = inputAbs.Value();
360 if (isInputFileList) {
361 CollectInputFilesFromFileList(fpath);
362 } else if (panda::os::file::File::IsDirectory(fpath)) {
363 CollectInputFilesFromFileDirectory(fpath, extension);
364 } else {
365 es2panda::SourceFile src(sourceFile_, recordName_, scriptKind_);
366 sourceFiles_.push_back(src);
367 }
368 } else if (!base64InputIsEmpty) {
369 // input content is base64 string
370 base64Input_ = ExtractContentFromBase64Input(base64Input.GetValue());
371 if (base64Input_.empty()) {
372 errorMsg_ = "The input string is not a valid base64 data";
373 return false;
374 }
375
376 es2panda::SourceFile src("", recordName_, es2panda::parser::ScriptKind::SCRIPT);
377 src.source = base64Input_;
378 sourceFiles_.push_back(src);
379 }
380
381 if (!outputProto.GetValue().empty()) {
382 compilerProtoOutput_ = outputProto.GetValue();
383 }
384
385 optLevel_ = opOptLevel.GetValue();
386 functionThreadCount_ = opFunctionThreadCount.GetValue();
387 fileThreadCount_ = opFileThreadCount.GetValue();
388 npmModuleEntryList_ = opNpmModuleEntryList.GetValue();
389
390 if (!opCacheFile.GetValue().empty()) {
391 ParseCacheFileOption(opCacheFile.GetValue());
392 }
393
394 if (opParseOnly.GetValue()) {
395 options_ |= OptionFlags::PARSE_ONLY;
396 }
397
398 if (opSizeStat.GetValue()) {
399 options_ |= OptionFlags::SIZE_STAT;
400 }
401
402 compilerOptions_.dumpAsm = opDumpAssembly.GetValue();
403 compilerOptions_.dumpAst = opDumpAst.GetValue();
404 compilerOptions_.dumpDebugInfo = opDumpDebugInfo.GetValue();
405 compilerOptions_.isDebug = opDebugInfo.GetValue();
406 compilerOptions_.parseOnly = opParseOnly.GetValue();
407 compilerOptions_.enableTypeCheck = opEnableTypeCheck.GetValue();
408 compilerOptions_.dumpLiteralBuffer = opDumpLiteralBuffer.GetValue();
409 compilerOptions_.isDebuggerEvaluateExpressionMode = debuggerEvaluateExpression.GetValue();
410
411 compilerOptions_.extension = extension_;
412 compilerOptions_.functionThreadCount = functionThreadCount_;
413 compilerOptions_.fileThreadCount = fileThreadCount_;
414 compilerOptions_.output = compilerOutput_;
415 compilerOptions_.debugInfoSourceFile = sourceFile.GetValue();
416 compilerOptions_.optLevel = (compilerOptions_.isDebug || !base64Input.GetValue().empty() ||
417 base64Output.GetValue()) ? 0 : opOptLevel.GetValue();
418 compilerOptions_.sourceFiles = sourceFiles_;
419 compilerOptions_.mergeAbc = opMergeAbc.GetValue();
420
421 compilerOptions_.hotfixOptions.dumpSymbolTable = opDumpSymbolTable.GetValue();
422 compilerOptions_.hotfixOptions.symbolTable = opInputSymbolTable.GetValue();
423 compilerOptions_.hotfixOptions.generatePatch = opGeneratePatch.GetValue();
424 compilerOptions_.hotfixOptions.hotReload = opHotReload.GetValue();
425
426 return true;
427 }
428
ExtractContentFromBase64Input(const std::string & inputBase64String)429 std::string Options::ExtractContentFromBase64Input(const std::string &inputBase64String)
430 {
431 std::string inputContent = util::Base64Decode(inputBase64String);
432 if (inputContent == "") {
433 return "";
434 }
435 bool validBase64Input = util::Base64Encode(inputContent) == inputBase64String;
436 if (!validBase64Input) {
437 return "";
438 }
439 return inputContent;
440 }
441 } // namespace panda::es2panda::aot
442