1 /*
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "compileQueue.h"
17
18 #include <binder/binder.h>
19 #include <binder/scope.h>
20 #include <compiler/core/compilerContext.h>
21 #include <compiler/core/emitter/emitter.h>
22 #include <compiler/core/function.h>
23 #include <compiler/core/pandagen.h>
24 #include <es2panda.h>
25 #include <mem/arena_allocator.h>
26 #include <mem/pool_manager.h>
27 #include <protobufSnapshotGenerator.h>
28 #include <util/commonUtil.h>
29 #include <util/dumper.h>
30 #include <util/helpers.h>
31
32 namespace panda::es2panda::compiler {
33
34 std::mutex CompileFileJob::globalMutex_;
35 std::mutex CompileAbcClassQueue::globalMutex_;
36
Run()37 void CompileFunctionJob::Run()
38 {
39 std::unique_lock<std::mutex> lock(m_);
40 cond_.wait(lock, [this] { return dependencies_ == 0; });
41
42 ArenaAllocator allocator(SpaceType::SPACE_TYPE_COMPILER, nullptr, true);
43 PandaGen pg(&allocator, context_, scope_);
44
45 Function::Compile(&pg);
46
47 FunctionEmitter funcEmitter(&allocator, &pg);
48 funcEmitter.Generate(context_->PatchFixHelper());
49
50 context_->GetEmitter()->AddFunction(&funcEmitter, context_);
51
52 for (auto *dependant : dependants_) {
53 dependant->Signal();
54 }
55 }
56
Run()57 void CompileModuleRecordJob::Run()
58 {
59 std::unique_lock<std::mutex> lock(m_);
60 cond_.wait(lock, [this] { return dependencies_ == 0; });
61
62 bool hasLazyImport = context_->Binder()->Program()->ModuleRecord()->HasLazyImport();
63 ModuleRecordEmitter moduleEmitter(context_->Binder()->Program()->ModuleRecord(), context_->NewLiteralIndex(),
64 hasLazyImport ? context_->NewLiteralIndex() : -1);
65 moduleEmitter.Generate();
66
67 context_->GetEmitter()->AddSourceTextModuleRecord(&moduleEmitter, context_);
68
69 for (auto *dependant : dependants_) {
70 dependant->Signal();
71 }
72 }
73
RetrieveProgramFromCacheFiles(const std::string & buffer)74 bool CompileFileJob::RetrieveProgramFromCacheFiles(const std::string &buffer)
75 {
76 if (options_->requireGlobalOptimization) {
77 return false;
78 }
79 auto cacheFileIter = options_->cacheFiles.find(src_->fileName);
80 // Disable the use of file caching when cross-program optimization is required, to prevent cached files from
81 // not being invalidated when their dependencies change, or from not being reanalyzed when their dependents
82 // are updated
83 if (cacheFileIter != options_->cacheFiles.end()) {
84 // cache is invalid when any one of source file infos being changed
85 auto bufToHash = buffer + src_->fileName + src_->recordName + src_->sourcefile + src_->pkgName;
86 src_->hash = GetHash32String(reinterpret_cast<const uint8_t *>(bufToHash.c_str()));
87
88 ArenaAllocator allocator(SpaceType::SPACE_TYPE_COMPILER, nullptr, true);
89 auto *cacheProgramInfo = proto::ProtobufSnapshotGenerator::GetCacheContext(cacheFileIter->second,
90 &allocator);
91
92 if (cacheProgramInfo != nullptr && cacheProgramInfo->hashCode == src_->hash) {
93 std::unique_lock<std::mutex> lock(globalMutex_);
94 auto *cache = allocator_->New<util::ProgramCache>(src_->hash, std::move(cacheProgramInfo->program));
95 progsInfo_.insert({src_->fileName, cache});
96 return true;
97 }
98 }
99 return false;
100 }
101
Run()102 void CompileFileJob::Run()
103 {
104 std::stringstream ss;
105 std::string buffer;
106 if (!src_->fileName.empty() && src_->isSourceMode) {
107 if (!util::Helpers::ReadFileToBuffer(src_->fileName, ss)) {
108 return;
109 }
110 buffer = ss.str();
111 src_->source = buffer;
112 if (RetrieveProgramFromCacheFiles(buffer)) {
113 return;
114 }
115 }
116
117 es2panda::Compiler compiler(src_->scriptExtension, options_->functionThreadCount);
118 panda::pandasm::Program *prog = nullptr;
119 if (src_->isSourceMode) {
120 prog = compiler.CompileFile(*options_, src_, symbolTable_);
121 } else if (!options_->mergeAbc) {
122 // If input is an abc file, in non merge-abc mode, compile classes one by one.
123 prog = compiler.CompileAbcFile(src_->fileName, *options_);
124 } else {
125 // If input is an abc file, in merge-abc mode, compile each class parallelly.
126 compiler.CompileAbcFileInParallel(src_, *options_, progsInfo_, allocator_);
127 return;
128 }
129 if (prog == nullptr) {
130 return;
131 }
132
133 bool requireOptimizationAfterAnalysis = false;
134 // When cross-program optimizations are required, skip program-local optimization at this stage
135 // and perform it later after the analysis of all programs has been completed
136 if (src_->isSourceMode && options_->transformLib.empty()) {
137 if (options_->requireGlobalOptimization) {
138 util::Helpers::AnalysisProgram(prog, src_->fileName);
139 requireOptimizationAfterAnalysis = true;
140 } else if (options_->optLevel != 0) {
141 util::Helpers::OptimizeProgram(prog, src_->fileName);
142 }
143 }
144
145 {
146 std::unique_lock<std::mutex> lock(globalMutex_);
147 auto *cache = allocator_->New<util::ProgramCache>(src_->hash, std::move(*prog), src_->isSourceMode);
148 progsInfo_.insert({src_->fileName, cache});
149 if (requireOptimizationAfterAnalysis) {
150 optimizationPendingProgs_.insert(src_->fileName);
151 }
152 }
153 }
154
Run()155 void CompileAbcClassJob::Run()
156 {
157 panda_file::File::EntityId recordId(classId_);
158 auto *program = new panda::pandasm::Program();
159 compiler_.CompileAbcClass(recordId, *program);
160
161 // Update version for abc input when needed
162 if (options_.updatePkgVersionForAbcInput && pkgVersionUpdateRequiredInAbc_) {
163 UpdatePackageVersion(program, options_);
164 // Remove redundant strings created due to version replacement
165 if (options_.removeRedundantFile && hasUpdatedVersion_) {
166 program->strings.clear();
167 for (const auto &[_, function] : program->function_table) {
168 const auto &funcStringSet = function.CollectStringsFromFunctionInsns();
169 program->strings.insert(funcStringSet.begin(), funcStringSet.end());
170 }
171 }
172 }
173
174 {
175 std::unique_lock<std::mutex> lock(CompileFileJob::globalMutex_);
176 ASSERT(compiler_.GetAbcFile().GetFilename().find(util::CHAR_VERTICAL_LINE) == std::string::npos);
177 ASSERT(program->record_table.size() == 1);
178 ASSERT(util::RecordNotGeneratedFromBytecode(program->record_table.begin()->first));
179 auto name = compiler_.GetAbcFile().GetFilename();
180 name += util::CHAR_VERTICAL_LINE + program->record_table.begin()->first;
181 auto *cache = allocator_->New<util::ProgramCache>(std::move(*program));
182 progsInfo_.emplace(name, cache);
183 }
184
185 delete program;
186 program = nullptr;
187 }
188
UpdateDynamicImportPackageVersion(panda::pandasm::Program * prog,const std::unordered_map<std::string,panda::es2panda::PkgInfo> & pkgContextInfo)189 void CompileAbcClassJob::UpdateDynamicImportPackageVersion(panda::pandasm::Program *prog,
190 const std::unordered_map<std::string, panda::es2panda::PkgInfo> &pkgContextInfo)
191 {
192 for (auto &[name, function] : prog->function_table) {
193 util::VisitDyanmicImports<false>(function, [this, &prog, pkgContextInfo](std::string &ohmurl) {
194 const auto &newOhmurl = util::UpdatePackageVersionIfNeeded(ohmurl, pkgContextInfo);
195 if (newOhmurl == ohmurl) {
196 return;
197 }
198 this->SetHasUpdatedVersion(true);
199 prog->strings.insert(newOhmurl);
200 ohmurl = newOhmurl;
201 });
202 }
203 }
204
UpdateStaticImportPackageVersion(panda::pandasm::Program * prog,const std::unordered_map<std::string,panda::es2panda::PkgInfo> & pkgContextInfo)205 void CompileAbcClassJob::UpdateStaticImportPackageVersion(panda::pandasm::Program *prog,
206 const std::unordered_map<std::string, panda::es2panda::PkgInfo> &pkgContextInfo)
207 {
208 for (auto &[recordName, record] : prog->record_table) {
209 util::VisitStaticImports<false>(*prog, record, [this, pkgContextInfo](std::string &ohmurl) {
210 const auto &newOhmurl = util::UpdatePackageVersionIfNeeded(ohmurl, pkgContextInfo);
211 if (newOhmurl == ohmurl) {
212 return;
213 }
214 this->SetHasUpdatedVersion(true);
215 ohmurl = newOhmurl;
216 });
217 }
218 }
219
UpdatePackageVersion(panda::pandasm::Program * prog,const panda::es2panda::CompilerOptions & options)220 void CompileAbcClassJob::UpdatePackageVersion(panda::pandasm::Program *prog,
221 const panda::es2panda::CompilerOptions &options)
222 {
223 bool isAccurateUpdateVersion = !options.compileContextInfo.updateVersionInfo.empty();
224 const std::unordered_map<std::string, panda::es2panda::PkgInfo> &pkgContextInfo = isAccurateUpdateVersion ?
225 options.compileContextInfo.updateVersionInfo.at(abcPkgName_) : options.compileContextInfo.pkgContextInfo;
226 // Replace for esm module static import
227 UpdateStaticImportPackageVersion(prog, pkgContextInfo);
228 // Replace for dynamic import
229 UpdateDynamicImportPackageVersion(prog, pkgContextInfo);
230 }
231
Run()232 void PostAnalysisOptimizeFileJob::Run()
233 {
234 util::Helpers::OptimizeProgram(program_, fileName_);
235 }
236
Schedule()237 void CompileFuncQueue::Schedule()
238 {
239 ASSERT(jobsCount_ == 0);
240 std::unique_lock<std::mutex> lock(m_);
241 const auto &functions = context_->Binder()->Functions();
242
243 for (auto *function : functions) {
244 auto *funcJob = new CompileFunctionJob(context_);
245 funcJob->SetFunctionScope(function);
246 jobs_.push_back(funcJob);
247 jobsCount_++;
248 }
249
250 if (context_->Binder()->Program()->Kind() == parser::ScriptKind::MODULE) {
251 auto *moduleRecordJob = new CompileModuleRecordJob(context_);
252 jobs_.push_back(moduleRecordJob);
253 jobsCount_++;
254 }
255
256 lock.unlock();
257 jobsAvailable_.notify_all();
258 }
259
Schedule()260 void CompileFileQueue::Schedule()
261 {
262 ASSERT(jobsCount_ == 0);
263 std::unique_lock<std::mutex> lock(m_);
264
265 for (auto &input: options_->sourceFiles) {
266 auto *fileJob = new CompileFileJob(&input, options_, progsInfo_, optimizationPendingProgs_,
267 symbolTable_, allocator_);
268 jobs_.push_back(fileJob);
269 jobsCount_++;
270 }
271
272 lock.unlock();
273 jobsAvailable_.notify_all();
274 }
275
NeedUpdateVersion()276 bool CompileAbcClassQueue::NeedUpdateVersion()
277 {
278 std::unordered_map<std::string, std::unordered_map<std::string, panda::es2panda::PkgInfo>> updateVersionInfo =
279 options_.compileContextInfo.updateVersionInfo;
280 auto iter = updateVersionInfo.find(src_->pkgName);
281 return updateVersionInfo.empty() || (iter != updateVersionInfo.end() && !iter->second.empty());
282 }
283
Schedule()284 void CompileAbcClassQueue::Schedule()
285 {
286 std::unique_lock<std::mutex> lock(m_);
287
288 auto classIds = compiler_.GetAbcFile().GetClasses();
289 size_t expectedProgsCountInAbcFile = 0;
290 bool needUpdateVersion = NeedUpdateVersion();
291 for (size_t i = 0; i != classIds.size(); ++i) {
292 if (!compiler_.CheckClassId(classIds[i], i)) {
293 continue;
294 }
295
296 auto *abcClassJob = new CompileAbcClassJob(classIds[i], options_, compiler_, progsInfo_, allocator_,
297 src_->pkgName, needUpdateVersion);
298
299 jobs_.push_back(abcClassJob);
300 jobsCount_++;
301 expectedProgsCountInAbcFile++;
302 }
303 {
304 std::unique_lock<std::mutex> lock(globalMutex_);
305 Compiler::SetExpectedProgsCount(Compiler::GetExpectedProgsCount() + expectedProgsCountInAbcFile - 1);
306 }
307
308 lock.unlock();
309 jobsAvailable_.notify_all();
310 }
311
Schedule()312 void PostAnalysisOptimizeFileQueue::Schedule()
313 {
314 ASSERT(jobsCount_ == 0);
315 std::unique_lock<std::mutex> lock(m_);
316
317 for (const auto &optimizationPendingProgName : optimizationPendingProgs_) {
318 auto progInfo = progsInfo_.find(optimizationPendingProgName);
319 if (progInfo == progsInfo_.end()) {
320 continue;
321 }
322 auto *optimizeJob = new PostAnalysisOptimizeFileJob(progInfo->first, &progInfo->second->program);
323 jobs_.push_back(optimizeJob);
324 jobsCount_++;
325 }
326 }
327
328 } // namespace panda::es2panda::compiler
329