1 /**
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "file_item_container.h"
17 #include <cstdint>
18 #include <type_traits>
19 #include "macros.h"
20 #include "file_format_version.h"
21 #include "pgo.h"
22
23 namespace ark::panda_file {
24
25 class ItemDeduper {
26 public:
27 template <class T>
Deduplicate(T * item)28 T *Deduplicate(T *item)
29 {
30 static_assert(std::is_base_of_v<BaseItem, T>);
31
32 if (auto iter = alreadyDedupedItems_.find(item); iter != alreadyDedupedItems_.end()) {
33 ASSERT(item->GetItemType() == iter->second->GetItemType());
34 return static_cast<T *>(iter->second);
35 }
36
37 ItemData itemData(item);
38 auto it = items_.find(itemData);
39 if (it == items_.cend()) {
40 items_.insert(itemData);
41 return item;
42 }
43
44 auto resultItem = it->GetItem();
45 ASSERT(item->GetItemType() == resultItem->GetItemType());
46 auto result = static_cast<T *>(resultItem);
47 if (item != result) {
48 alreadyDedupedItems_.emplace(item, result);
49 if constexpr (!std::is_same_v<T, LineNumberProgramItem>) {
50 item->SetNeedsEmit(false);
51 }
52 }
53
54 return result;
55 }
56
GetUniqueCount() const57 size_t GetUniqueCount() const
58 {
59 return items_.size();
60 }
61
62 private:
63 class ItemWriter : public Writer {
64 public:
ItemWriter(std::vector<uint8_t> * buf,size_t offset)65 ItemWriter(std::vector<uint8_t> *buf, size_t offset) : buf_(buf), offset_(offset) {}
66 ~ItemWriter() override = default;
67
68 NO_COPY_SEMANTIC(ItemWriter);
69 NO_MOVE_SEMANTIC(ItemWriter);
70
WriteByte(uint8_t byte)71 bool WriteByte(uint8_t byte) override
72 {
73 buf_->push_back(byte);
74 ++offset_;
75 return true;
76 }
77
WriteBytes(const std::vector<uint8_t> & bytes)78 bool WriteBytes(const std::vector<uint8_t> &bytes) override
79 {
80 buf_->insert(buf_->end(), bytes.cbegin(), bytes.cend());
81 offset_ += bytes.size();
82 return true;
83 }
84
GetOffset() const85 size_t GetOffset() const override
86 {
87 return offset_;
88 }
89
90 private:
91 std::vector<uint8_t> *buf_;
92 size_t offset_;
93 };
94
95 class ItemData {
96 public:
ItemData(BaseItem * item)97 explicit ItemData(BaseItem *item) : item_(item)
98 {
99 Initialize();
100 }
101
102 ~ItemData() = default;
103
104 DEFAULT_COPY_SEMANTIC(ItemData);
105 NO_MOVE_SEMANTIC(ItemData);
106
GetItem() const107 BaseItem *GetItem() const
108 {
109 return item_;
110 }
111
GetHash() const112 uint32_t GetHash() const
113 {
114 ASSERT(IsInitialized());
115 return hash_;
116 }
117
operator ==(const ItemData & itemData) const118 bool operator==(const ItemData &itemData) const noexcept
119 {
120 ASSERT(IsInitialized());
121 return data_ == itemData.data_;
122 }
123
124 private:
IsInitialized() const125 bool IsInitialized() const
126 {
127 return !data_.empty();
128 }
129
Initialize()130 void Initialize()
131 {
132 ASSERT(item_->NeedsEmit());
133
134 ItemWriter writer(&data_, item_->GetOffset());
135 [[maybe_unused]] auto res = item_->Write(&writer);
136 ASSERT(res);
137 ASSERT(data_.size() == item_->GetSize());
138
139 hash_ = GetHash32(data_.data(), data_.size());
140 }
141
142 BaseItem *item_;
143 uint32_t hash_ {0};
144 std::vector<uint8_t> data_;
145 };
146
147 struct ItemHash {
operator ()ark::panda_file::ItemDeduper::ItemHash148 size_t operator()(const ItemData &itemData) const noexcept
149 {
150 return itemData.GetHash();
151 }
152 };
153
154 std::unordered_set<ItemData, ItemHash> items_;
155 std::unordered_map<BaseItem *, BaseItem *> alreadyDedupedItems_;
156 };
157
158 // NOTE(nsizov): make method for items deletion
159 template <class T, class C, class I, class P, class E, class... Args>
GetOrInsert(C & map,I & items,const P & pos,const E & key,bool isForeign,Args &&...args)160 static T *GetOrInsert(C &map, I &items, const P &pos, const E &key, bool isForeign, Args &&...args)
161 {
162 auto it = map.find(key);
163 if (it != map.cend()) {
164 auto *item = it->second;
165 if (item->IsForeign() == isForeign) {
166 return static_cast<T *>(item);
167 }
168
169 UNREACHABLE();
170 return nullptr;
171 }
172
173 auto ii = items.insert(pos, std::make_unique<T>(std::forward<Args>(args)...));
174 auto *item = static_cast<T *>(ii->get());
175
176 [[maybe_unused]] auto res = map.insert({key, item});
177 ASSERT(res.second);
178 return item;
179 }
180
ItemContainer()181 ItemContainer::ItemContainer()
182 {
183 itemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
184 annotationItemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
185 codeItemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
186 debugItemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
187 end_ = debugItemsEnd_->get();
188 }
189
GetOrCreateClassItem(const std::string & str)190 ClassItem *ItemContainer::GetOrCreateClassItem(const std::string &str)
191 {
192 return GetOrInsert<ClassItem>(classMap_, items_, itemsEnd_, str, false, str);
193 }
194
GetOrCreateForeignClassItem(const std::string & str)195 ForeignClassItem *ItemContainer::GetOrCreateForeignClassItem(const std::string &str)
196 {
197 return GetOrInsert<ForeignClassItem>(classMap_, foreignItems_, foreignItems_.end(), str, true, str);
198 }
199
GetOrCreateStringItem(const std::string & str)200 StringItem *ItemContainer::GetOrCreateStringItem(const std::string &str)
201 {
202 auto it = classMap_.find(str);
203 if (it != classMap_.cend()) {
204 return it->second->GetNameItem();
205 }
206
207 return GetOrInsert<StringItem>(stringMap_, items_, itemsEnd_, str, false, str);
208 }
209
GetOrCreateLiteralArrayItem(const std::string & id)210 LiteralArrayItem *ItemContainer::GetOrCreateLiteralArrayItem(const std::string &id)
211 {
212 return GetOrInsert<LiteralArrayItem>(literalarrayMap_, items_, itemsEnd_, id, false);
213 }
214
GetOrCreateIntegerValueItem(uint32_t v)215 ScalarValueItem *ItemContainer::GetOrCreateIntegerValueItem(uint32_t v)
216 {
217 return GetOrInsert<ScalarValueItem>(intValueMap_, items_, itemsEnd_, v, false, v);
218 }
219
GetOrCreateLongValueItem(uint64_t v)220 ScalarValueItem *ItemContainer::GetOrCreateLongValueItem(uint64_t v)
221 {
222 return GetOrInsert<ScalarValueItem>(longValueMap_, items_, itemsEnd_, v, false, v);
223 }
224
GetOrCreateFloatValueItem(float v)225 ScalarValueItem *ItemContainer::GetOrCreateFloatValueItem(float v)
226 {
227 return GetOrInsert<ScalarValueItem>(floatValueMap_, items_, itemsEnd_, bit_cast<uint32_t>(v), false, v);
228 }
229
GetOrCreateDoubleValueItem(double v)230 ScalarValueItem *ItemContainer::GetOrCreateDoubleValueItem(double v)
231 {
232 return GetOrInsert<ScalarValueItem>(doubleValueMap_, items_, itemsEnd_, bit_cast<uint64_t>(v), false, v);
233 }
234
GetOrCreateIdValueItem(BaseItem * v)235 ScalarValueItem *ItemContainer::GetOrCreateIdValueItem(BaseItem *v)
236 {
237 return GetOrInsert<ScalarValueItem>(idValueMap_, items_, itemsEnd_, v, false, v);
238 }
239
GetOrCreateProtoItem(TypeItem * retType,const std::vector<MethodParamItem> & params)240 ProtoItem *ItemContainer::GetOrCreateProtoItem(TypeItem *retType, const std::vector<MethodParamItem> ¶ms)
241 {
242 ProtoKey key(retType, params);
243 return GetOrInsert<ProtoItem>(protoMap_, items_, itemsEnd_, key, false, retType, params);
244 }
245
GetOrCreatePrimitiveTypeItem(Type type)246 PrimitiveTypeItem *ItemContainer::GetOrCreatePrimitiveTypeItem(Type type)
247 {
248 return GetOrCreatePrimitiveTypeItem(type.GetId());
249 }
250
GetOrCreatePrimitiveTypeItem(Type::TypeId type)251 PrimitiveTypeItem *ItemContainer::GetOrCreatePrimitiveTypeItem(Type::TypeId type)
252 {
253 return GetOrInsert<PrimitiveTypeItem>(primitiveTypeMap_, items_, itemsEnd_, type, false, type);
254 }
255
CreateLineNumberProgramItem()256 LineNumberProgramItem *ItemContainer::CreateLineNumberProgramItem()
257 {
258 auto it = items_.insert(debugItemsEnd_, std::make_unique<LineNumberProgramItem>());
259 auto *item = static_cast<LineNumberProgramItem *>(it->get());
260 [[maybe_unused]] auto res = lineNumberProgramIndexItem_.Add(item);
261 ASSERT(res);
262 return item;
263 }
264
IncRefLineNumberProgramItem(LineNumberProgramItem * it)265 void ItemContainer::IncRefLineNumberProgramItem(LineNumberProgramItem *it)
266 {
267 lineNumberProgramIndexItem_.IncRefCount(it);
268 }
269
DeduplicateLineNumberProgram(DebugInfoItem * item,ItemDeduper * deduper)270 void ItemContainer::DeduplicateLineNumberProgram(DebugInfoItem *item, ItemDeduper *deduper)
271 {
272 auto *lineNumberProgram = item->GetLineNumberProgram();
273 auto *deduplicated = deduper->Deduplicate(lineNumberProgram);
274 if (deduplicated != lineNumberProgram) {
275 item->SetLineNumberProgram(deduplicated);
276 lineNumberProgramIndexItem_.IncRefCount(deduplicated);
277 lineNumberProgramIndexItem_.DecRefCount(lineNumberProgram);
278 }
279 }
280
DeduplicateDebugInfo(MethodItem * method,ItemDeduper * debugInfoDeduper,ItemDeduper * lineNumberProgramDeduper)281 void ItemContainer::DeduplicateDebugInfo(MethodItem *method, ItemDeduper *debugInfoDeduper,
282 ItemDeduper *lineNumberProgramDeduper)
283 {
284 auto *debugItem = method->GetDebugInfo();
285 if (debugItem == nullptr) {
286 return;
287 }
288
289 DeduplicateLineNumberProgram(debugItem, lineNumberProgramDeduper);
290
291 auto *deduplicated = debugInfoDeduper->Deduplicate(debugItem);
292 if (deduplicated != debugItem) {
293 method->SetDebugInfo(deduplicated);
294 lineNumberProgramIndexItem_.DecRefCount(debugItem->GetLineNumberProgram());
295 }
296 }
297
DeduplicateCode(MethodItem * method,ItemDeduper * codeDeduper)298 static void DeduplicateCode(MethodItem *method, ItemDeduper *codeDeduper)
299 {
300 auto *codeItem = method->GetCode();
301 if (codeItem == nullptr) {
302 return;
303 }
304
305 auto *deduplicated = codeDeduper->Deduplicate(codeItem);
306 if (deduplicated != codeItem) {
307 method->SetCode(deduplicated);
308 deduplicated->AddMethod(method); // we need it for Profile-Guided optimization
309 }
310 }
311
DeduplicateCodeAndDebugInfo()312 void ItemContainer::DeduplicateCodeAndDebugInfo()
313 {
314 ItemDeduper lineNumberProgramDeduper;
315 ItemDeduper debugDeduper;
316 ItemDeduper codeDeduper;
317
318 for (auto &p : classMap_) {
319 auto *item = p.second;
320 if (item->IsForeign()) {
321 continue;
322 }
323
324 auto *classItem = static_cast<ClassItem *>(item);
325
326 classItem->VisitMethods([this, &debugDeduper, &lineNumberProgramDeduper, &codeDeduper](BaseItem *paramItem) {
327 auto *methodItem = static_cast<MethodItem *>(paramItem);
328 DeduplicateDebugInfo(methodItem, &debugDeduper, &lineNumberProgramDeduper);
329 DeduplicateCode(methodItem, &codeDeduper);
330 return true;
331 });
332 }
333 }
334
DeduplicateAnnotationValue(AnnotationItem * annotationItem,ItemDeduper * deduper)335 static void DeduplicateAnnotationValue(AnnotationItem *annotationItem, ItemDeduper *deduper)
336 {
337 auto *elems = annotationItem->GetElements();
338 const auto &tags = annotationItem->GetTags();
339
340 for (size_t i = 0; i < elems->size(); i++) {
341 auto tag = tags[i];
342
343 // try to dedupe only ArrayValueItems
344 switch (tag.GetItem()) {
345 case 'K':
346 case 'L':
347 case 'M':
348 case 'N':
349 case 'O':
350 case 'P':
351 case 'Q':
352 case 'R':
353 case 'S':
354 case 'T':
355 case 'U':
356 case 'V':
357 case 'W':
358 case 'X':
359 case 'Y':
360 case 'Z':
361 case '@':
362 break;
363 default:
364 continue;
365 }
366
367 auto &elem = (*elems)[i];
368 auto *value = elem.GetValue();
369 auto *deduplicated = deduper->Deduplicate(value);
370 if (deduplicated != value) {
371 elem.SetValue(deduplicated);
372 }
373 }
374 }
375
DeduplicateAnnotations(std::vector<AnnotationItem * > * items,ItemDeduper * annotationDeduper,ItemDeduper * valueDeduper)376 static void DeduplicateAnnotations(std::vector<AnnotationItem *> *items, ItemDeduper *annotationDeduper,
377 ItemDeduper *valueDeduper)
378 {
379 for (auto &item : *items) {
380 DeduplicateAnnotationValue(item, valueDeduper);
381 auto *deduplicated = annotationDeduper->Deduplicate(item);
382 if (deduplicated != item) {
383 item = deduplicated;
384 }
385 }
386 }
387
DeduplicateAnnotations()388 void ItemContainer::DeduplicateAnnotations()
389 {
390 ItemDeduper valueDeduper;
391 ItemDeduper annotationDeduper;
392
393 for (auto &p : classMap_) {
394 auto *item = p.second;
395 if (item->IsForeign()) {
396 continue;
397 }
398
399 auto *classItem = static_cast<ClassItem *>(item);
400
401 panda_file::DeduplicateAnnotations(classItem->GetRuntimeAnnotations(), &annotationDeduper, &valueDeduper);
402 panda_file::DeduplicateAnnotations(classItem->GetAnnotations(), &annotationDeduper, &valueDeduper);
403 panda_file::DeduplicateAnnotations(classItem->GetRuntimeTypeAnnotations(), &annotationDeduper, &valueDeduper);
404 panda_file::DeduplicateAnnotations(classItem->GetTypeAnnotations(), &annotationDeduper, &valueDeduper);
405
406 classItem->VisitMethods([&annotationDeduper, &valueDeduper](BaseItem *paramItem) {
407 auto *methodItem = static_cast<MethodItem *>(paramItem);
408 panda_file::DeduplicateAnnotations(methodItem->GetRuntimeAnnotations(), &annotationDeduper, &valueDeduper);
409 panda_file::DeduplicateAnnotations(methodItem->GetAnnotations(), &annotationDeduper, &valueDeduper);
410 panda_file::DeduplicateAnnotations(methodItem->GetRuntimeTypeAnnotations(), &annotationDeduper,
411 &valueDeduper);
412 panda_file::DeduplicateAnnotations(methodItem->GetTypeAnnotations(), &annotationDeduper, &valueDeduper);
413 return true;
414 });
415
416 classItem->VisitFields([&annotationDeduper, &valueDeduper](BaseItem *paramItem) {
417 auto *fieldItem = static_cast<FieldItem *>(paramItem);
418 panda_file::DeduplicateAnnotations(fieldItem->GetRuntimeAnnotations(), &annotationDeduper, &valueDeduper);
419 panda_file::DeduplicateAnnotations(fieldItem->GetAnnotations(), &annotationDeduper, &valueDeduper);
420 panda_file::DeduplicateAnnotations(fieldItem->GetRuntimeTypeAnnotations(), &annotationDeduper,
421 &valueDeduper);
422 panda_file::DeduplicateAnnotations(fieldItem->GetTypeAnnotations(), &annotationDeduper, &valueDeduper);
423 return true;
424 });
425 }
426 }
427
DeduplicateItems(bool computeLayout)428 void ItemContainer::DeduplicateItems(bool computeLayout)
429 {
430 if (computeLayout) {
431 ComputeLayout();
432 }
433 DeduplicateCodeAndDebugInfo();
434 DeduplicateAnnotations();
435 }
436
ComputeLayout()437 uint32_t ItemContainer::ComputeLayout()
438 {
439 uint32_t numClasses = classMap_.size();
440 uint32_t numLiteralarrays = literalarrayMap_.size();
441 uint32_t classIdxOffset = sizeof(File::Header);
442 uint32_t curOffset = classIdxOffset + (numClasses + numLiteralarrays) * ID_SIZE;
443
444 UpdateOrderIndexes();
445 UpdateLiteralIndexes();
446
447 RebuildRegionSection();
448 RebuildLineNumberProgramIndex();
449
450 regionSectionItem_.SetOffset(curOffset);
451 regionSectionItem_.ComputeLayout();
452 curOffset += regionSectionItem_.GetSize();
453
454 for (auto &item : foreignItems_) {
455 curOffset = RoundUp(curOffset, item->Alignment());
456 item->SetOffset(curOffset);
457 item->ComputeLayout();
458 curOffset += item->GetSize();
459 }
460
461 for (auto &item : items_) {
462 if (!item->NeedsEmit()) {
463 continue;
464 }
465
466 curOffset = RoundUp(curOffset, item->Alignment());
467 item->SetOffset(curOffset);
468 item->ComputeLayout();
469 curOffset += item->GetSize();
470 }
471
472 // Line number program should be last because it's size is known only after deduplication
473 curOffset = RoundUp(curOffset, lineNumberProgramIndexItem_.Alignment());
474 lineNumberProgramIndexItem_.SetOffset(curOffset);
475 lineNumberProgramIndexItem_.ComputeLayout();
476 curOffset += lineNumberProgramIndexItem_.GetSize();
477
478 end_->SetOffset(curOffset);
479
480 return curOffset;
481 }
482
RebuildLineNumberProgramIndex()483 void ItemContainer::RebuildLineNumberProgramIndex()
484 {
485 lineNumberProgramIndexItem_.Reset();
486 lineNumberProgramIndexItem_.UpdateItems(nullptr, nullptr);
487 }
488
RebuildRegionSection()489 void ItemContainer::RebuildRegionSection()
490 {
491 regionSectionItem_.Reset();
492
493 for (auto &item : foreignItems_) {
494 ProcessIndexDependecies(item.get());
495 }
496
497 for (auto &item : items_) {
498 if (!item->NeedsEmit()) {
499 continue;
500 }
501
502 ProcessIndexDependecies(item.get());
503 }
504
505 if (!regionSectionItem_.IsEmpty()) {
506 regionSectionItem_.GetCurrentHeader()->SetEnd(end_);
507 }
508
509 regionSectionItem_.UpdateItems();
510 }
511
UpdateOrderIndexes()512 void ItemContainer::UpdateOrderIndexes()
513 {
514 size_t idx = 0;
515
516 for (auto &item : foreignItems_) {
517 item->SetOrderIndex(idx++);
518 item->Visit([&idx](BaseItem *paramItem) {
519 paramItem->SetOrderIndex(idx++);
520 return true;
521 });
522 }
523
524 for (auto &item : items_) {
525 if (!item->NeedsEmit()) {
526 continue;
527 }
528
529 item->SetOrderIndex(idx++);
530 item->Visit([&idx](BaseItem *paramItem) {
531 paramItem->SetOrderIndex(idx++);
532 return true;
533 });
534 }
535
536 end_->SetOrderIndex(idx++);
537 }
538
UpdateLiteralIndexes()539 void ItemContainer::UpdateLiteralIndexes()
540 {
541 size_t idx = 0;
542
543 for (auto &it : literalarrayMap_) {
544 it.second->SetIndex(idx++);
545 }
546 }
547
ReorderItems(ark::panda_file::pgo::ProfileOptimizer * profileOpt)548 void ItemContainer::ReorderItems(ark::panda_file::pgo::ProfileOptimizer *profileOpt)
549 {
550 profileOpt->ProfileGuidedRelayout(items_);
551 }
552
ProcessIndexDependecies(BaseItem * item)553 void ItemContainer::ProcessIndexDependecies(BaseItem *item)
554 {
555 auto deps = item->GetIndexDependencies();
556
557 item->Visit([&deps](BaseItem *paramItem) {
558 const auto &itemDeps = paramItem->GetIndexDependencies();
559 deps.insert(deps.end(), itemDeps.cbegin(), itemDeps.cend());
560 return true;
561 });
562
563 if (regionSectionItem_.IsEmpty()) {
564 regionSectionItem_.AddHeader();
565 regionSectionItem_.GetCurrentHeader()->SetStart(item);
566 }
567
568 if (regionSectionItem_.GetCurrentHeader()->Add(deps)) {
569 return;
570 }
571
572 regionSectionItem_.GetCurrentHeader()->SetEnd(item);
573 regionSectionItem_.AddHeader();
574 regionSectionItem_.GetCurrentHeader()->SetStart(item);
575
576 if (!regionSectionItem_.GetCurrentHeader()->Add(deps)) {
577 LOG(FATAL, PANDAFILE) << "Cannot add " << deps.size() << " items to index";
578 }
579 }
580
WriteHeaderIndexInfo(Writer * writer)581 bool ItemContainer::WriteHeaderIndexInfo(Writer *writer)
582 {
583 if (!writer->Write<uint32_t>(classMap_.size())) {
584 return false;
585 }
586
587 if (!writer->Write<uint32_t>(sizeof(File::Header))) {
588 return false;
589 }
590
591 if (!writer->Write<uint32_t>(lineNumberProgramIndexItem_.GetNumItems())) {
592 return false;
593 }
594
595 if (!writer->Write<uint32_t>(lineNumberProgramIndexItem_.GetOffset())) {
596 return false;
597 }
598
599 if (!writer->Write<uint32_t>(literalarrayMap_.size())) {
600 return false;
601 }
602
603 uint32_t literalarrayIdxOffset = sizeof(File::Header) + classMap_.size() * ID_SIZE;
604 if (!writer->Write<uint32_t>(literalarrayIdxOffset)) {
605 return false;
606 }
607
608 if (!writer->Write<uint32_t>(regionSectionItem_.GetNumHeaders())) {
609 return false;
610 }
611
612 size_t indexSectionOff = literalarrayIdxOffset + literalarrayMap_.size() * ID_SIZE;
613 return writer->Write<uint32_t>(indexSectionOff);
614 }
615
WriteHeader(Writer * writer,ssize_t * checksumOffset)616 bool ItemContainer::WriteHeader(Writer *writer, ssize_t *checksumOffset)
617 {
618 uint32_t fileSize = ComputeLayout();
619
620 std::vector<uint8_t> magic;
621 magic.assign(File::MAGIC.cbegin(), File::MAGIC.cend());
622 if (!writer->WriteBytes(magic)) {
623 return false;
624 }
625
626 *checksumOffset = static_cast<ssize_t>(writer->GetOffset());
627 uint32_t checksum = 0;
628 if (!writer->Write(checksum)) {
629 return false;
630 }
631 writer->CountChecksum(true);
632
633 std::vector<uint8_t> versionVec(std::begin(VERSION), std::end(VERSION));
634 if (!writer->WriteBytes(versionVec)) {
635 return false;
636 }
637
638 if (!writer->Write(fileSize)) {
639 return false;
640 }
641
642 uint32_t foreignOffset = GetForeignOffset();
643 if (!writer->Write(foreignOffset)) {
644 return false;
645 }
646
647 uint32_t foreignSize = GetForeignSize();
648 if (!writer->Write(foreignSize)) {
649 return false;
650 }
651
652 if (!writer->Write<uint32_t>(static_cast<uint32_t>(isQuickened_))) {
653 return false;
654 }
655
656 return WriteHeaderIndexInfo(writer);
657 }
658
Write(Writer * writer,bool deduplicateItems,bool computeLayout)659 bool ItemContainer::Write(Writer *writer, bool deduplicateItems, bool computeLayout)
660 {
661 if (deduplicateItems) {
662 DeduplicateItems(computeLayout);
663 }
664
665 ssize_t checksumOffset = -1;
666 if (!WriteHeader(writer, &checksumOffset)) {
667 return false;
668 }
669 ASSERT(checksumOffset != -1);
670
671 // Write class idx
672
673 for (auto &entry : classMap_) {
674 if (!writer->Write(entry.second->GetOffset())) {
675 return false;
676 }
677 }
678
679 // Write literalArray idx
680
681 for (auto &entry : literalarrayMap_) {
682 if (!writer->Write(entry.second->GetOffset())) {
683 return false;
684 }
685 }
686
687 // Write index section
688
689 if (!regionSectionItem_.Write(writer)) {
690 return false;
691 }
692
693 for (auto &item : foreignItems_) {
694 if (!writer->Align(item->Alignment()) || !item->Write(writer)) {
695 return false;
696 }
697 }
698
699 for (auto &item : items_) {
700 if (!item->NeedsEmit()) {
701 continue;
702 }
703
704 if (!writer->Align(item->Alignment()) || !item->Write(writer)) {
705 return false;
706 }
707 }
708
709 if (!writer->Align(lineNumberProgramIndexItem_.Alignment())) {
710 return false;
711 }
712
713 // Write line number program idx
714
715 if (!lineNumberProgramIndexItem_.Write(writer)) {
716 return false;
717 }
718
719 writer->CountChecksum(false);
720 writer->WriteChecksum(checksumOffset);
721
722 return true;
723 }
724
GetStat()725 std::map<std::string, size_t> ItemContainer::GetStat()
726 {
727 std::map<std::string, size_t> stat;
728
729 DeduplicateItems();
730 ComputeLayout();
731
732 stat["header_item"] = sizeof(File::Header);
733 stat["class_idx_item"] = classMap_.size() * ID_SIZE;
734 stat["line_number_program_idx_item"] = lineNumberProgramIndexItem_.GetNumItems() * ID_SIZE;
735 stat["literalarray_idx"] = literalarrayMap_.size() * ID_SIZE;
736
737 stat["region_section_item"] = regionSectionItem_.GetSize();
738 stat["foreign_item"] = GetForeignSize();
739
740 size_t numIns = 0;
741 size_t codesize = 0;
742 for (auto &item : items_) {
743 if (!item->NeedsEmit()) {
744 continue;
745 }
746
747 const auto &name = item->GetName();
748 size_t size = item->GetSize();
749 auto it = stat.find(name);
750 if (it != stat.cend()) {
751 stat[name] += size;
752 } else if (size != 0) {
753 stat[name] = size;
754 }
755 if (name == "code_item") {
756 numIns += static_cast<CodeItem *>(item.get())->GetNumInstructions();
757 codesize += static_cast<CodeItem *>(item.get())->GetCodeSize();
758 }
759 }
760 stat["instructions_number"] = numIns;
761 stat["codesize"] = codesize;
762
763 return stat;
764 }
765
DumpItemsStat(std::ostream & os) const766 void ItemContainer::DumpItemsStat(std::ostream &os) const
767 {
768 struct Stat {
769 size_t n;
770 size_t totalSize;
771 };
772
773 std::map<std::string, Stat> stat;
774
775 auto collectStat = [&stat](auto &items) {
776 for (auto &item : items) {
777 if (!item->NeedsEmit()) {
778 continue;
779 }
780
781 const auto &name = item->GetName();
782 size_t size = item->GetSize();
783 auto it = stat.find(name);
784 if (it != stat.cend()) {
785 stat[name].n += 1;
786 stat[name].totalSize += size;
787 } else if (size != 0) {
788 stat[name] = {1, size};
789 }
790 }
791 };
792
793 collectStat(foreignItems_);
794 collectStat(items_);
795
796 for (auto &[name, elem] : stat) {
797 os << name << ":" << std::endl;
798 os << " n = " << elem.n << std::endl;
799 os << " total size = " << elem.totalSize << std::endl;
800 }
801 }
802
GetForeignOffset() const803 size_t ItemContainer::GetForeignOffset() const
804 {
805 if (foreignItems_.empty()) {
806 return 0;
807 }
808
809 return foreignItems_.front()->GetOffset();
810 }
811
GetForeignSize() const812 size_t ItemContainer::GetForeignSize() const
813 {
814 if (foreignItems_.empty()) {
815 return 0;
816 }
817
818 size_t begin = foreignItems_.front()->GetOffset();
819 size_t end = foreignItems_.back()->GetOffset() + foreignItems_.back()->GetSize();
820
821 return end - begin;
822 }
823
Write(Writer * writer)824 bool ItemContainer::RegionHeaderItem::Write(Writer *writer)
825 {
826 ASSERT(GetOffset() == writer->GetOffset());
827 ASSERT(start_ != nullptr);
828 ASSERT(start_->GetOffset() != 0);
829 ASSERT(end_ != nullptr);
830 ASSERT(end_->GetOffset() != 0);
831
832 if (!writer->Write<uint32_t>(start_->GetOffset())) {
833 return false;
834 }
835
836 if (!writer->Write<uint32_t>(end_->GetOffset())) {
837 return false;
838 }
839
840 for (auto *indexItem : indexes_) {
841 if (!writer->Write<uint32_t>(indexItem->GetNumItems())) {
842 return false;
843 }
844
845 ASSERT(indexItem->GetOffset() != 0);
846 if (!writer->Write<uint32_t>(indexItem->GetOffset())) {
847 return false;
848 }
849 }
850
851 return true;
852 }
853
Add(const std::list<IndexedItem * > & items)854 bool ItemContainer::RegionHeaderItem::Add(const std::list<IndexedItem *> &items)
855 {
856 std::list<IndexedItem *> addedItems;
857
858 for (auto *item : items) {
859 auto type = item->GetIndexType();
860 ASSERT(type != IndexType::NONE);
861
862 auto *indexItem = GetIndexByType(type);
863
864 if (indexItem->Has(item)) {
865 continue;
866 }
867
868 if (!indexItem->Add(item)) {
869 Remove(addedItems);
870 return false;
871 }
872
873 addedItems.push_back(item);
874 }
875
876 return true;
877 }
878
Remove(const std::list<IndexedItem * > & items)879 void ItemContainer::RegionHeaderItem::Remove(const std::list<IndexedItem *> &items)
880 {
881 for (auto *item : items) {
882 auto type = item->GetIndexType();
883 ASSERT(type != IndexType::NONE);
884
885 auto *indexItem = GetIndexByType(type);
886 indexItem->Remove(item);
887 }
888 }
889
Write(Writer * writer)890 bool ItemContainer::IndexItem::Write(Writer *writer)
891 {
892 ASSERT(GetOffset() == writer->GetOffset());
893
894 for (auto *item : index_) {
895 if (!writer->Write<uint32_t>(item->GetOffset())) {
896 return false;
897 }
898 }
899
900 return true;
901 }
902
GetItemType() const903 ItemTypes ItemContainer::IndexItem::GetItemType() const
904 {
905 switch (type_) {
906 case IndexType::CLASS:
907 return ItemTypes::CLASS_INDEX_ITEM;
908 case IndexType::METHOD:
909 return ItemTypes::METHOD_INDEX_ITEM;
910 case IndexType::FIELD:
911 return ItemTypes::FIELD_INDEX_ITEM;
912 case IndexType::PROTO:
913 return ItemTypes::PROTO_INDEX_ITEM;
914 case IndexType::LINE_NUMBER_PROG:
915 return ItemTypes::LINE_NUMBER_PROGRAM_INDEX_ITEM;
916 default:
917 break;
918 }
919
920 UNREACHABLE();
921 }
922
Add(IndexedItem * item)923 bool ItemContainer::IndexItem::Add(IndexedItem *item)
924 {
925 auto size = index_.size();
926 ASSERT(size <= maxIndex_);
927
928 if (size == maxIndex_) {
929 return false;
930 }
931
932 auto res = index_.insert(item);
933 ASSERT(res.second);
934
935 return res.second;
936 }
937
AddHeader()938 void ItemContainer::RegionSectionItem::AddHeader()
939 {
940 std::vector<IndexItem *> indexItems;
941 for (size_t i = 0; i < INDEX_COUNT_16; i++) {
942 auto type = static_cast<IndexType>(i);
943 indexes_.emplace_back(type, MAX_INDEX_16);
944 indexItems.push_back(&indexes_.back());
945 }
946 headers_.emplace_back(indexItems);
947 }
948
CalculateSize() const949 size_t ItemContainer::RegionSectionItem::CalculateSize() const
950 {
951 size_t size = headers_.size() * sizeof(File::RegionHeader);
952 for (auto &indexItem : indexes_) {
953 size += indexItem.GetSize();
954 }
955 return size;
956 }
957
ComputeLayout()958 void ItemContainer::RegionSectionItem::ComputeLayout()
959 {
960 size_t offset = GetOffset();
961
962 for (auto &header : headers_) {
963 header.SetOffset(offset);
964 header.ComputeLayout();
965 offset += header.GetSize();
966 }
967
968 for (auto &index : indexes_) {
969 index.SetOffset(offset);
970 index.ComputeLayout();
971 offset += index.GetSize();
972 }
973 }
974
Write(Writer * writer)975 bool ItemContainer::RegionSectionItem::Write(Writer *writer)
976 {
977 ASSERT(GetOffset() == writer->GetOffset());
978
979 for (auto &header : headers_) {
980 if (!header.Write(writer)) {
981 return false;
982 }
983 }
984
985 for (auto &index : indexes_) {
986 if (!index.Write(writer)) {
987 return false;
988 }
989 }
990
991 return true;
992 }
993
ProtoKey(TypeItem * retType,const std::vector<MethodParamItem> & params)994 ItemContainer::ProtoKey::ProtoKey(TypeItem *retType, const std::vector<MethodParamItem> ¶ms)
995 {
996 Add(retType);
997 for (const auto ¶m : params) {
998 Add(param.GetType());
999 }
1000 size_t shortyHash = std::hash<std::string>()(shorty_);
1001 size_t retTypeHash = std::hash<TypeItem *>()(retType);
1002 // combine hashes of shorty and ref_types
1003 hash_ = ark::MergeHashes(shortyHash, retTypeHash);
1004 // combine hashes of all param types
1005 for (const auto &item : params) {
1006 size_t paramTypeHash = std::hash<TypeItem *>()(item.GetType());
1007 hash_ = ark::MergeHashes(hash_, paramTypeHash);
1008 }
1009 }
1010
Add(TypeItem * item)1011 void ItemContainer::ProtoKey::Add(TypeItem *item)
1012 {
1013 auto type = item->GetType();
1014 shorty_.append(Type::GetSignatureByTypeId(type));
1015 if (type.IsReference()) {
1016 refTypes_.push_back(item);
1017 }
1018 }
1019
1020 } // namespace ark::panda_file
1021