1 /**
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "file_item_container.h"
17 #include <cstdint>
18 #include <type_traits>
19 #include "macros.h"
20 #include "file_format_version.h"
21 #include "pgo.h"
22
23 namespace ark::panda_file {
24
25 class ItemDeduper {
26 public:
27 template <class T>
Deduplicate(T * item)28 T *Deduplicate(T *item)
29 {
30 static_assert(std::is_base_of_v<BaseItem, T>);
31
32 if (auto iter = alreadyDedupedItems_.find(item); iter != alreadyDedupedItems_.end()) {
33 ASSERT(item->GetItemType() == iter->second->GetItemType());
34 return static_cast<T *>(iter->second);
35 }
36
37 ItemData itemData(item);
38 auto it = items_.find(itemData);
39 if (it == items_.cend()) {
40 items_.insert(itemData);
41 return item;
42 }
43
44 auto resultItem = it->GetItem();
45 ASSERT(item->GetItemType() == resultItem->GetItemType());
46 auto result = static_cast<T *>(resultItem);
47 if (item != result) {
48 alreadyDedupedItems_.emplace(item, result);
49 if constexpr (!std::is_same_v<T, LineNumberProgramItem>) {
50 item->SetNeedsEmit(false);
51 }
52 }
53
54 return result;
55 }
56
GetUniqueCount() const57 size_t GetUniqueCount() const
58 {
59 return items_.size();
60 }
61
62 private:
63 class ItemWriter : public Writer {
64 public:
ItemWriter(std::vector<uint8_t> * buf,size_t offset)65 ItemWriter(std::vector<uint8_t> *buf, size_t offset) : buf_(buf), offset_(offset) {}
66 ~ItemWriter() override = default;
67
68 NO_COPY_SEMANTIC(ItemWriter);
69 NO_MOVE_SEMANTIC(ItemWriter);
70
WriteByte(uint8_t byte)71 bool WriteByte(uint8_t byte) override
72 {
73 buf_->push_back(byte);
74 ++offset_;
75 return true;
76 }
77
WriteBytes(const std::vector<uint8_t> & bytes)78 bool WriteBytes(const std::vector<uint8_t> &bytes) override
79 {
80 buf_->insert(buf_->end(), bytes.cbegin(), bytes.cend());
81 offset_ += bytes.size();
82 return true;
83 }
84
GetOffset() const85 size_t GetOffset() const override
86 {
87 return offset_;
88 }
89
90 private:
91 std::vector<uint8_t> *buf_;
92 size_t offset_;
93 };
94
95 class ItemData {
96 public:
ItemData(BaseItem * item)97 explicit ItemData(BaseItem *item) : item_(item)
98 {
99 Initialize();
100 }
101
102 ~ItemData() = default;
103
104 DEFAULT_COPY_SEMANTIC(ItemData);
105 NO_MOVE_SEMANTIC(ItemData);
106
GetItem() const107 BaseItem *GetItem() const
108 {
109 return item_;
110 }
111
GetHash() const112 uint32_t GetHash() const
113 {
114 ASSERT(IsInitialized());
115 return hash_;
116 }
117
operator ==(const ItemData & itemData) const118 bool operator==(const ItemData &itemData) const noexcept
119 {
120 ASSERT(IsInitialized());
121 return data_ == itemData.data_;
122 }
123
124 private:
IsInitialized() const125 bool IsInitialized() const
126 {
127 return !data_.empty();
128 }
129
Initialize()130 void Initialize()
131 {
132 ASSERT(item_->NeedsEmit());
133
134 ItemWriter writer(&data_, item_->GetOffset());
135 [[maybe_unused]] auto res = item_->Write(&writer);
136 ASSERT(res);
137 ASSERT(data_.size() == item_->GetSize());
138
139 hash_ = GetHash32(data_.data(), data_.size());
140 }
141
142 BaseItem *item_;
143 uint32_t hash_ {0};
144 std::vector<uint8_t> data_;
145 };
146
147 struct ItemHash {
operator ()ark::panda_file::ItemDeduper::ItemHash148 size_t operator()(const ItemData &itemData) const noexcept
149 {
150 return itemData.GetHash();
151 }
152 };
153
154 std::unordered_set<ItemData, ItemHash> items_;
155 std::unordered_map<BaseItem *, BaseItem *> alreadyDedupedItems_;
156 };
157
158 // NOTE(nsizov): make method for items deletion
159 template <class T, class C, class I, class P, class E, class... Args>
160 // CC-OFFNXT(G.FUN.01-CPP) solid logic
GetOrInsert(C & map,I & items,const P & pos,const E & key,bool isForeign,Args &&...args)161 static T *GetOrInsert(C &map, I &items, const P &pos, const E &key, bool isForeign, Args &&...args)
162 {
163 auto it = map.find(key);
164 if (it != map.cend()) {
165 auto *item = it->second;
166 if (item->IsForeign() == isForeign) {
167 return static_cast<T *>(item);
168 }
169
170 UNREACHABLE();
171 return nullptr;
172 }
173
174 auto ii = items.insert(pos, std::make_unique<T>(std::forward<Args>(args)...));
175 auto *item = static_cast<T *>(ii->get());
176
177 [[maybe_unused]] auto res = map.insert({key, item});
178 ASSERT(res.second);
179 return item;
180 }
181
ItemContainer()182 ItemContainer::ItemContainer()
183 {
184 itemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
185 annotationItemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
186 codeItemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
187 debugItemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
188 end_ = debugItemsEnd_->get();
189 }
190
GetOrCreateClassItem(const std::string & str)191 ClassItem *ItemContainer::GetOrCreateClassItem(const std::string &str)
192 {
193 return GetOrInsert<ClassItem>(classMap_, items_, itemsEnd_, str, false, str);
194 }
195
GetOrCreateForeignClassItem(const std::string & str)196 ForeignClassItem *ItemContainer::GetOrCreateForeignClassItem(const std::string &str)
197 {
198 return GetOrInsert<ForeignClassItem>(classMap_, foreignItems_, foreignItems_.end(), str, true, str);
199 }
200
GetOrCreateStringItem(const std::string & str)201 StringItem *ItemContainer::GetOrCreateStringItem(const std::string &str)
202 {
203 auto it = classMap_.find(str);
204 if (it != classMap_.cend()) {
205 return it->second->GetNameItem();
206 }
207
208 return GetOrInsert<StringItem>(stringMap_, items_, itemsEnd_, str, false, str);
209 }
210
GetOrCreateLiteralArrayItem(const std::string & id)211 LiteralArrayItem *ItemContainer::GetOrCreateLiteralArrayItem(const std::string &id)
212 {
213 return GetOrInsert<LiteralArrayItem>(literalarrayMap_, items_, itemsEnd_, id, false);
214 }
215
GetOrCreateIntegerValueItem(uint32_t v)216 ScalarValueItem *ItemContainer::GetOrCreateIntegerValueItem(uint32_t v)
217 {
218 return GetOrInsert<ScalarValueItem>(intValueMap_, items_, itemsEnd_, v, false, v);
219 }
220
GetOrCreateLongValueItem(uint64_t v)221 ScalarValueItem *ItemContainer::GetOrCreateLongValueItem(uint64_t v)
222 {
223 return GetOrInsert<ScalarValueItem>(longValueMap_, items_, itemsEnd_, v, false, v);
224 }
225
GetOrCreateFloatValueItem(float v)226 ScalarValueItem *ItemContainer::GetOrCreateFloatValueItem(float v)
227 {
228 return GetOrInsert<ScalarValueItem>(floatValueMap_, items_, itemsEnd_, bit_cast<uint32_t>(v), false, v);
229 }
230
GetOrCreateDoubleValueItem(double v)231 ScalarValueItem *ItemContainer::GetOrCreateDoubleValueItem(double v)
232 {
233 return GetOrInsert<ScalarValueItem>(doubleValueMap_, items_, itemsEnd_, bit_cast<uint64_t>(v), false, v);
234 }
235
GetOrCreateIdValueItem(BaseItem * v)236 ScalarValueItem *ItemContainer::GetOrCreateIdValueItem(BaseItem *v)
237 {
238 return GetOrInsert<ScalarValueItem>(idValueMap_, items_, itemsEnd_, v, false, v);
239 }
240
GetOrCreateProtoItem(TypeItem * retType,const std::vector<MethodParamItem> & params)241 ProtoItem *ItemContainer::GetOrCreateProtoItem(TypeItem *retType, const std::vector<MethodParamItem> ¶ms)
242 {
243 ProtoKey key(retType, params);
244 return GetOrInsert<ProtoItem>(protoMap_, items_, itemsEnd_, key, false, retType, params);
245 }
246
GetOrCreatePrimitiveTypeItem(Type type)247 PrimitiveTypeItem *ItemContainer::GetOrCreatePrimitiveTypeItem(Type type)
248 {
249 return GetOrCreatePrimitiveTypeItem(type.GetId());
250 }
251
GetOrCreatePrimitiveTypeItem(Type::TypeId type)252 PrimitiveTypeItem *ItemContainer::GetOrCreatePrimitiveTypeItem(Type::TypeId type)
253 {
254 return GetOrInsert<PrimitiveTypeItem>(primitiveTypeMap_, items_, itemsEnd_, type, false, type);
255 }
256
CreateLineNumberProgramItem()257 LineNumberProgramItem *ItemContainer::CreateLineNumberProgramItem()
258 {
259 auto it = items_.insert(debugItemsEnd_, std::make_unique<LineNumberProgramItem>());
260 auto *item = static_cast<LineNumberProgramItem *>(it->get());
261 [[maybe_unused]] auto res = lineNumberProgramIndexItem_.Add(item);
262 ASSERT(res);
263 return item;
264 }
265
IncRefLineNumberProgramItem(LineNumberProgramItem * it)266 void ItemContainer::IncRefLineNumberProgramItem(LineNumberProgramItem *it)
267 {
268 lineNumberProgramIndexItem_.IncRefCount(it);
269 }
270
DeduplicateLineNumberProgram(DebugInfoItem * item,ItemDeduper * deduper)271 void ItemContainer::DeduplicateLineNumberProgram(DebugInfoItem *item, ItemDeduper *deduper)
272 {
273 auto *lineNumberProgram = item->GetLineNumberProgram();
274 auto *deduplicated = deduper->Deduplicate(lineNumberProgram);
275 if (deduplicated != lineNumberProgram) {
276 item->SetLineNumberProgram(deduplicated);
277 lineNumberProgramIndexItem_.IncRefCount(deduplicated);
278 lineNumberProgramIndexItem_.DecRefCount(lineNumberProgram);
279 }
280 }
281
DeduplicateDebugInfo(MethodItem * method,ItemDeduper * debugInfoDeduper,ItemDeduper * lineNumberProgramDeduper)282 void ItemContainer::DeduplicateDebugInfo(MethodItem *method, ItemDeduper *debugInfoDeduper,
283 ItemDeduper *lineNumberProgramDeduper)
284 {
285 auto *debugItem = method->GetDebugInfo();
286 if (debugItem == nullptr) {
287 return;
288 }
289
290 DeduplicateLineNumberProgram(debugItem, lineNumberProgramDeduper);
291
292 auto *deduplicated = debugInfoDeduper->Deduplicate(debugItem);
293 if (deduplicated != debugItem) {
294 method->SetDebugInfo(deduplicated);
295 lineNumberProgramIndexItem_.DecRefCount(debugItem->GetLineNumberProgram());
296 }
297 }
298
DeduplicateCode(MethodItem * method,ItemDeduper * codeDeduper)299 static void DeduplicateCode(MethodItem *method, ItemDeduper *codeDeduper)
300 {
301 auto *codeItem = method->GetCode();
302 if (codeItem == nullptr) {
303 return;
304 }
305
306 auto *deduplicated = codeDeduper->Deduplicate(codeItem);
307 if (deduplicated != codeItem) {
308 method->SetCode(deduplicated);
309 deduplicated->AddMethod(method); // we need it for Profile-Guided optimization
310 }
311 }
312
DeduplicateCodeAndDebugInfo()313 void ItemContainer::DeduplicateCodeAndDebugInfo()
314 {
315 ItemDeduper lineNumberProgramDeduper;
316 ItemDeduper debugDeduper;
317 ItemDeduper codeDeduper;
318
319 for (auto &p : classMap_) {
320 auto *item = p.second;
321 if (item->IsForeign()) {
322 continue;
323 }
324
325 auto *classItem = static_cast<ClassItem *>(item);
326
327 classItem->VisitMethods([this, &debugDeduper, &lineNumberProgramDeduper, &codeDeduper](BaseItem *paramItem) {
328 auto *methodItem = static_cast<MethodItem *>(paramItem);
329 DeduplicateDebugInfo(methodItem, &debugDeduper, &lineNumberProgramDeduper);
330 DeduplicateCode(methodItem, &codeDeduper);
331 return true;
332 });
333 }
334 }
335
DeduplicateAnnotationValue(AnnotationItem * annotationItem,ItemDeduper * deduper)336 static void DeduplicateAnnotationValue(AnnotationItem *annotationItem, ItemDeduper *deduper)
337 {
338 auto *elems = annotationItem->GetElements();
339 const auto &tags = annotationItem->GetTags();
340
341 for (size_t i = 0; i < elems->size(); i++) {
342 auto tag = tags[i];
343
344 // try to dedupe only ArrayValueItems
345 switch (tag.GetItem()) {
346 case 'K':
347 case 'L':
348 case 'M':
349 case 'N':
350 case 'O':
351 case 'P':
352 case 'Q':
353 case 'R':
354 case 'S':
355 case 'T':
356 case 'U':
357 case 'V':
358 case 'W':
359 case 'X':
360 case 'Y':
361 case 'Z':
362 case '@':
363 break;
364 default:
365 continue;
366 }
367
368 auto &elem = (*elems)[i];
369 auto *value = elem.GetValue();
370 auto *deduplicated = deduper->Deduplicate(value);
371 if (deduplicated != value) {
372 elem.SetValue(deduplicated);
373 }
374 }
375 }
376
DeduplicateAnnotations(std::vector<AnnotationItem * > * items,ItemDeduper * annotationDeduper,ItemDeduper * valueDeduper)377 static void DeduplicateAnnotations(std::vector<AnnotationItem *> *items, ItemDeduper *annotationDeduper,
378 ItemDeduper *valueDeduper)
379 {
380 for (auto &item : *items) {
381 DeduplicateAnnotationValue(item, valueDeduper);
382 auto *deduplicated = annotationDeduper->Deduplicate(item);
383 if (deduplicated != item) {
384 item = deduplicated;
385 }
386 }
387 }
388
DeduplicateAnnotations()389 void ItemContainer::DeduplicateAnnotations()
390 {
391 ItemDeduper valueDeduper;
392 ItemDeduper annotationDeduper;
393
394 for (auto &p : classMap_) {
395 auto *item = p.second;
396 if (item->IsForeign()) {
397 continue;
398 }
399
400 auto *classItem = static_cast<ClassItem *>(item);
401
402 panda_file::DeduplicateAnnotations(classItem->GetRuntimeAnnotations(), &annotationDeduper, &valueDeduper);
403 panda_file::DeduplicateAnnotations(classItem->GetAnnotations(), &annotationDeduper, &valueDeduper);
404 panda_file::DeduplicateAnnotations(classItem->GetRuntimeTypeAnnotations(), &annotationDeduper, &valueDeduper);
405 panda_file::DeduplicateAnnotations(classItem->GetTypeAnnotations(), &annotationDeduper, &valueDeduper);
406
407 classItem->VisitMethods([&annotationDeduper, &valueDeduper](BaseItem *paramItem) {
408 auto *methodItem = static_cast<MethodItem *>(paramItem);
409 panda_file::DeduplicateAnnotations(methodItem->GetRuntimeAnnotations(), &annotationDeduper, &valueDeduper);
410 panda_file::DeduplicateAnnotations(methodItem->GetAnnotations(), &annotationDeduper, &valueDeduper);
411 panda_file::DeduplicateAnnotations(methodItem->GetRuntimeTypeAnnotations(), &annotationDeduper,
412 &valueDeduper);
413 panda_file::DeduplicateAnnotations(methodItem->GetTypeAnnotations(), &annotationDeduper, &valueDeduper);
414 return true;
415 });
416
417 classItem->VisitFields([&annotationDeduper, &valueDeduper](BaseItem *paramItem) {
418 auto *fieldItem = static_cast<FieldItem *>(paramItem);
419 panda_file::DeduplicateAnnotations(fieldItem->GetRuntimeAnnotations(), &annotationDeduper, &valueDeduper);
420 panda_file::DeduplicateAnnotations(fieldItem->GetAnnotations(), &annotationDeduper, &valueDeduper);
421 panda_file::DeduplicateAnnotations(fieldItem->GetRuntimeTypeAnnotations(), &annotationDeduper,
422 &valueDeduper);
423 panda_file::DeduplicateAnnotations(fieldItem->GetTypeAnnotations(), &annotationDeduper, &valueDeduper);
424 return true;
425 });
426 }
427 }
428
DeduplicateItems(bool computeLayout)429 void ItemContainer::DeduplicateItems(bool computeLayout)
430 {
431 if (computeLayout) {
432 ComputeLayout();
433 }
434 DeduplicateCodeAndDebugInfo();
435 DeduplicateAnnotations();
436 }
437
ComputeLayout()438 uint32_t ItemContainer::ComputeLayout()
439 {
440 uint32_t numClasses = classMap_.size();
441 uint32_t numLiteralarrays = literalarrayMap_.size();
442 uint32_t classIdxOffset = sizeof(File::Header);
443 uint32_t curOffset = classIdxOffset + (numClasses + numLiteralarrays) * ID_SIZE;
444
445 UpdateOrderIndexes();
446 UpdateLiteralIndexes();
447
448 RebuildRegionSection();
449 RebuildLineNumberProgramIndex();
450
451 regionSectionItem_.SetOffset(curOffset);
452 regionSectionItem_.ComputeLayout();
453 curOffset += regionSectionItem_.GetSize();
454
455 for (auto &item : foreignItems_) {
456 curOffset = RoundUp(curOffset, item->Alignment());
457 item->SetOffset(curOffset);
458 item->ComputeLayout();
459 curOffset += item->GetSize();
460 }
461
462 for (auto &item : items_) {
463 if (!item->NeedsEmit()) {
464 continue;
465 }
466
467 curOffset = RoundUp(curOffset, item->Alignment());
468 item->SetOffset(curOffset);
469 item->ComputeLayout();
470 curOffset += item->GetSize();
471 }
472
473 // Line number program should be last because it's size is known only after deduplication
474 curOffset = RoundUp(curOffset, lineNumberProgramIndexItem_.Alignment());
475 lineNumberProgramIndexItem_.SetOffset(curOffset);
476 lineNumberProgramIndexItem_.ComputeLayout();
477 curOffset += lineNumberProgramIndexItem_.GetSize();
478
479 end_->SetOffset(curOffset);
480
481 return curOffset;
482 }
483
RebuildLineNumberProgramIndex()484 void ItemContainer::RebuildLineNumberProgramIndex()
485 {
486 lineNumberProgramIndexItem_.Reset();
487 lineNumberProgramIndexItem_.UpdateItems(nullptr, nullptr);
488 }
489
RebuildRegionSection()490 void ItemContainer::RebuildRegionSection()
491 {
492 regionSectionItem_.Reset();
493
494 for (auto &item : foreignItems_) {
495 ProcessIndexDependecies(item.get());
496 }
497
498 for (auto &item : items_) {
499 if (!item->NeedsEmit()) {
500 continue;
501 }
502
503 ProcessIndexDependecies(item.get());
504 }
505
506 if (!regionSectionItem_.IsEmpty()) {
507 regionSectionItem_.GetCurrentHeader()->SetEnd(end_);
508 }
509
510 regionSectionItem_.UpdateItems();
511 }
512
UpdateOrderIndexes()513 void ItemContainer::UpdateOrderIndexes()
514 {
515 size_t idx = 0;
516
517 for (auto &item : foreignItems_) {
518 item->SetOrderIndex(idx++);
519 item->Visit([&idx](BaseItem *paramItem) {
520 paramItem->SetOrderIndex(idx++);
521 return true;
522 });
523 }
524
525 for (auto &item : items_) {
526 if (!item->NeedsEmit()) {
527 continue;
528 }
529
530 item->SetOrderIndex(idx++);
531 item->Visit([&idx](BaseItem *paramItem) {
532 paramItem->SetOrderIndex(idx++);
533 return true;
534 });
535 }
536
537 end_->SetOrderIndex(idx++);
538 }
539
UpdateLiteralIndexes()540 void ItemContainer::UpdateLiteralIndexes()
541 {
542 size_t idx = 0;
543
544 for (auto &it : literalarrayMap_) {
545 it.second->SetIndex(idx++);
546 }
547 }
548
ReorderItems(ark::panda_file::pgo::ProfileOptimizer * profileOpt)549 void ItemContainer::ReorderItems(ark::panda_file::pgo::ProfileOptimizer *profileOpt)
550 {
551 profileOpt->ProfileGuidedRelayout(items_);
552 }
553
ProcessIndexDependecies(BaseItem * item)554 void ItemContainer::ProcessIndexDependecies(BaseItem *item)
555 {
556 auto deps = item->GetIndexDependencies();
557
558 item->Visit([&deps](BaseItem *paramItem) {
559 const auto &itemDeps = paramItem->GetIndexDependencies();
560 deps.insert(deps.end(), itemDeps.cbegin(), itemDeps.cend());
561 return true;
562 });
563
564 if (regionSectionItem_.IsEmpty()) {
565 regionSectionItem_.AddHeader();
566 regionSectionItem_.GetCurrentHeader()->SetStart(item);
567 }
568
569 if (regionSectionItem_.GetCurrentHeader()->Add(deps)) {
570 return;
571 }
572
573 regionSectionItem_.GetCurrentHeader()->SetEnd(item);
574 regionSectionItem_.AddHeader();
575 regionSectionItem_.GetCurrentHeader()->SetStart(item);
576
577 if (!regionSectionItem_.GetCurrentHeader()->Add(deps)) {
578 LOG(FATAL, PANDAFILE) << "Cannot add " << deps.size() << " items to index";
579 }
580 }
581
WriteHeaderIndexInfo(Writer * writer)582 bool ItemContainer::WriteHeaderIndexInfo(Writer *writer)
583 {
584 if (!writer->Write<uint32_t>(classMap_.size())) {
585 return false;
586 }
587
588 if (!writer->Write<uint32_t>(sizeof(File::Header))) {
589 return false;
590 }
591
592 if (!writer->Write<uint32_t>(lineNumberProgramIndexItem_.GetNumItems())) {
593 return false;
594 }
595
596 if (!writer->Write<uint32_t>(lineNumberProgramIndexItem_.GetOffset())) {
597 return false;
598 }
599
600 if (!writer->Write<uint32_t>(literalarrayMap_.size())) {
601 return false;
602 }
603
604 uint32_t literalarrayIdxOffset = sizeof(File::Header) + classMap_.size() * ID_SIZE;
605 if (!writer->Write<uint32_t>(literalarrayIdxOffset)) {
606 return false;
607 }
608
609 if (!writer->Write<uint32_t>(regionSectionItem_.GetNumHeaders())) {
610 return false;
611 }
612
613 size_t indexSectionOff = literalarrayIdxOffset + literalarrayMap_.size() * ID_SIZE;
614 return writer->Write<uint32_t>(indexSectionOff);
615 }
616
WriteHeader(Writer * writer,ssize_t * checksumOffset)617 bool ItemContainer::WriteHeader(Writer *writer, ssize_t *checksumOffset)
618 {
619 uint32_t fileSize = ComputeLayout();
620
621 std::vector<uint8_t> magic;
622 magic.assign(File::MAGIC.cbegin(), File::MAGIC.cend());
623 if (!writer->WriteBytes(magic)) {
624 return false;
625 }
626
627 *checksumOffset = static_cast<ssize_t>(writer->GetOffset());
628 uint32_t checksum = 0;
629 if (!writer->Write(checksum)) {
630 return false;
631 }
632 writer->CountChecksum(true);
633
634 std::vector<uint8_t> versionVec(std::begin(VERSION), std::end(VERSION));
635 if (!writer->WriteBytes(versionVec)) {
636 return false;
637 }
638
639 if (!writer->Write(fileSize)) {
640 return false;
641 }
642
643 uint32_t foreignOffset = GetForeignOffset();
644 if (!writer->Write(foreignOffset)) {
645 return false;
646 }
647
648 uint32_t foreignSize = GetForeignSize();
649 if (!writer->Write(foreignSize)) {
650 return false;
651 }
652
653 if (!writer->Write<uint32_t>(static_cast<uint32_t>(isQuickened_))) {
654 return false;
655 }
656
657 return WriteHeaderIndexInfo(writer);
658 }
659
Write(Writer * writer,bool deduplicateItems,bool computeLayout)660 bool ItemContainer::Write(Writer *writer, bool deduplicateItems, bool computeLayout)
661 {
662 if (deduplicateItems) {
663 DeduplicateItems(computeLayout);
664 }
665
666 ssize_t checksumOffset = -1;
667 if (!WriteHeader(writer, &checksumOffset)) {
668 return false;
669 }
670 ASSERT(checksumOffset != -1);
671
672 // Write class idx
673
674 for (auto &entry : classMap_) {
675 if (!writer->Write(entry.second->GetOffset())) {
676 return false;
677 }
678 }
679
680 // Write literalArray idx
681
682 for (auto &entry : literalarrayMap_) {
683 if (!writer->Write(entry.second->GetOffset())) {
684 return false;
685 }
686 }
687
688 // Write index section
689
690 if (!regionSectionItem_.Write(writer)) {
691 return false;
692 }
693
694 for (auto &item : foreignItems_) {
695 if (!writer->Align(item->Alignment()) || !item->Write(writer)) {
696 return false;
697 }
698 }
699
700 for (auto &item : items_) {
701 if (!item->NeedsEmit()) {
702 continue;
703 }
704
705 if (!writer->Align(item->Alignment()) || !item->Write(writer)) {
706 return false;
707 }
708 }
709
710 if (!writer->Align(lineNumberProgramIndexItem_.Alignment())) {
711 return false;
712 }
713
714 // Write line number program idx
715
716 if (!lineNumberProgramIndexItem_.Write(writer)) {
717 return false;
718 }
719
720 writer->CountChecksum(false);
721 writer->WriteChecksum(checksumOffset);
722
723 return true;
724 }
725
GetStat()726 std::map<std::string, size_t> ItemContainer::GetStat()
727 {
728 std::map<std::string, size_t> stat;
729
730 DeduplicateItems();
731 ComputeLayout();
732
733 stat["header_item"] = sizeof(File::Header);
734 stat["class_idx_item"] = classMap_.size() * ID_SIZE;
735 stat["line_number_program_idx_item"] = lineNumberProgramIndexItem_.GetNumItems() * ID_SIZE;
736 stat["literalarray_idx"] = literalarrayMap_.size() * ID_SIZE;
737
738 stat["region_section_item"] = regionSectionItem_.GetSize();
739 stat["foreign_item"] = GetForeignSize();
740
741 size_t numIns = 0;
742 size_t codesize = 0;
743 for (auto &item : items_) {
744 if (!item->NeedsEmit()) {
745 continue;
746 }
747
748 const auto &name = item->GetName();
749 size_t size = item->GetSize();
750 auto it = stat.find(name);
751 if (it != stat.cend()) {
752 stat[name] += size;
753 } else if (size != 0) {
754 stat[name] = size;
755 }
756 if (name == "code_item") {
757 numIns += static_cast<CodeItem *>(item.get())->GetNumInstructions();
758 codesize += static_cast<CodeItem *>(item.get())->GetCodeSize();
759 }
760 }
761 stat["instructions_number"] = numIns;
762 stat["codesize"] = codesize;
763
764 return stat;
765 }
766
DumpItemsStat(std::ostream & os) const767 void ItemContainer::DumpItemsStat(std::ostream &os) const
768 {
769 struct Stat {
770 size_t n;
771 size_t totalSize;
772 };
773
774 std::map<std::string, Stat> stat;
775
776 auto collectStat = [&stat](auto &items) {
777 for (auto &item : items) {
778 if (!item->NeedsEmit()) {
779 continue;
780 }
781
782 const auto &name = item->GetName();
783 size_t size = item->GetSize();
784 auto it = stat.find(name);
785 if (it != stat.cend()) {
786 stat[name].n += 1;
787 stat[name].totalSize += size;
788 } else if (size != 0) {
789 stat[name] = {1, size};
790 }
791 }
792 };
793
794 collectStat(foreignItems_);
795 collectStat(items_);
796
797 for (auto &[name, elem] : stat) {
798 os << name << ":" << std::endl;
799 os << " n = " << elem.n << std::endl;
800 os << " total size = " << elem.totalSize << std::endl;
801 }
802 }
803
GetForeignOffset() const804 size_t ItemContainer::GetForeignOffset() const
805 {
806 if (foreignItems_.empty()) {
807 return 0;
808 }
809
810 return foreignItems_.front()->GetOffset();
811 }
812
GetForeignSize() const813 size_t ItemContainer::GetForeignSize() const
814 {
815 if (foreignItems_.empty()) {
816 return 0;
817 }
818
819 size_t begin = foreignItems_.front()->GetOffset();
820 size_t end = foreignItems_.back()->GetOffset() + foreignItems_.back()->GetSize();
821
822 return end - begin;
823 }
824
Write(Writer * writer)825 bool ItemContainer::RegionHeaderItem::Write(Writer *writer)
826 {
827 ASSERT(GetOffset() == writer->GetOffset());
828 ASSERT(start_ != nullptr);
829 ASSERT(start_->GetOffset() != 0);
830 ASSERT(end_ != nullptr);
831 ASSERT(end_->GetOffset() != 0);
832
833 if (!writer->Write<uint32_t>(start_->GetOffset())) {
834 return false;
835 }
836
837 if (!writer->Write<uint32_t>(end_->GetOffset())) {
838 return false;
839 }
840
841 for (auto *indexItem : indexes_) {
842 if (!writer->Write<uint32_t>(indexItem->GetNumItems())) {
843 return false;
844 }
845
846 ASSERT(indexItem->GetOffset() != 0);
847 if (!writer->Write<uint32_t>(indexItem->GetOffset())) {
848 return false;
849 }
850 }
851
852 return true;
853 }
854
Add(const std::list<IndexedItem * > & items)855 bool ItemContainer::RegionHeaderItem::Add(const std::list<IndexedItem *> &items)
856 {
857 std::list<IndexedItem *> addedItems;
858
859 for (auto *item : items) {
860 auto type = item->GetIndexType();
861 ASSERT(type != IndexType::NONE);
862
863 auto *indexItem = GetIndexByType(type);
864
865 if (indexItem->Has(item)) {
866 continue;
867 }
868
869 if (!indexItem->Add(item)) {
870 Remove(addedItems);
871 return false;
872 }
873
874 addedItems.push_back(item);
875 }
876
877 return true;
878 }
879
Remove(const std::list<IndexedItem * > & items)880 void ItemContainer::RegionHeaderItem::Remove(const std::list<IndexedItem *> &items)
881 {
882 for (auto *item : items) {
883 auto type = item->GetIndexType();
884 ASSERT(type != IndexType::NONE);
885
886 auto *indexItem = GetIndexByType(type);
887 indexItem->Remove(item);
888 }
889 }
890
Write(Writer * writer)891 bool ItemContainer::IndexItem::Write(Writer *writer)
892 {
893 ASSERT(GetOffset() == writer->GetOffset());
894
895 for (auto *item : index_) {
896 if (!writer->Write<uint32_t>(item->GetOffset())) {
897 return false;
898 }
899 }
900
901 return true;
902 }
903
GetItemType() const904 ItemTypes ItemContainer::IndexItem::GetItemType() const
905 {
906 switch (type_) {
907 case IndexType::CLASS:
908 return ItemTypes::CLASS_INDEX_ITEM;
909 case IndexType::METHOD:
910 return ItemTypes::METHOD_INDEX_ITEM;
911 case IndexType::FIELD:
912 return ItemTypes::FIELD_INDEX_ITEM;
913 case IndexType::PROTO:
914 return ItemTypes::PROTO_INDEX_ITEM;
915 case IndexType::LINE_NUMBER_PROG:
916 return ItemTypes::LINE_NUMBER_PROGRAM_INDEX_ITEM;
917 default:
918 break;
919 }
920
921 UNREACHABLE();
922 }
923
Add(IndexedItem * item)924 bool ItemContainer::IndexItem::Add(IndexedItem *item)
925 {
926 auto size = index_.size();
927 ASSERT(size <= maxIndex_);
928
929 if (size == maxIndex_) {
930 return false;
931 }
932
933 auto res = index_.insert(item);
934 ASSERT(res.second);
935
936 return res.second;
937 }
938
AddHeader()939 void ItemContainer::RegionSectionItem::AddHeader()
940 {
941 std::vector<IndexItem *> indexItems;
942 for (size_t i = 0; i < INDEX_COUNT_16; i++) {
943 auto type = static_cast<IndexType>(i);
944 indexes_.emplace_back(type, MAX_INDEX_16);
945 indexItems.push_back(&indexes_.back());
946 }
947 headers_.emplace_back(indexItems);
948 }
949
CalculateSize() const950 size_t ItemContainer::RegionSectionItem::CalculateSize() const
951 {
952 size_t size = headers_.size() * sizeof(File::RegionHeader);
953 for (auto &indexItem : indexes_) {
954 size += indexItem.GetSize();
955 }
956 return size;
957 }
958
ComputeLayout()959 void ItemContainer::RegionSectionItem::ComputeLayout()
960 {
961 size_t offset = GetOffset();
962
963 for (auto &header : headers_) {
964 header.SetOffset(offset);
965 header.ComputeLayout();
966 offset += header.GetSize();
967 }
968
969 for (auto &index : indexes_) {
970 index.SetOffset(offset);
971 index.ComputeLayout();
972 offset += index.GetSize();
973 }
974 }
975
Write(Writer * writer)976 bool ItemContainer::RegionSectionItem::Write(Writer *writer)
977 {
978 ASSERT(GetOffset() == writer->GetOffset());
979
980 for (auto &header : headers_) {
981 if (!header.Write(writer)) {
982 return false;
983 }
984 }
985
986 for (auto &index : indexes_) {
987 if (!index.Write(writer)) {
988 return false;
989 }
990 }
991
992 return true;
993 }
994
ProtoKey(TypeItem * retType,const std::vector<MethodParamItem> & params)995 ItemContainer::ProtoKey::ProtoKey(TypeItem *retType, const std::vector<MethodParamItem> ¶ms)
996 {
997 Add(retType);
998 for (const auto ¶m : params) {
999 Add(param.GetType());
1000 }
1001 size_t shortyHash = std::hash<std::string>()(shorty_);
1002 size_t retTypeHash = std::hash<TypeItem *>()(retType);
1003 // combine hashes of shorty and ref_types
1004 hash_ = ark::MergeHashes(shortyHash, retTypeHash);
1005 // combine hashes of all param types
1006 for (const auto &item : params) {
1007 size_t paramTypeHash = std::hash<TypeItem *>()(item.GetType());
1008 hash_ = ark::MergeHashes(hash_, paramTypeHash);
1009 }
1010 }
1011
Add(TypeItem * item)1012 void ItemContainer::ProtoKey::Add(TypeItem *item)
1013 {
1014 auto type = item->GetType();
1015 shorty_.append(Type::GetSignatureByTypeId(type));
1016 if (type.IsReference()) {
1017 refTypes_.push_back(item);
1018 }
1019 }
1020
1021 } // namespace ark::panda_file
1022