1 /**
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "file_item_container.h"
17 #include <cstdint>
18 #include <type_traits>
19 #include "macros.h"
20 #include "file_format_version.h"
21 #include "pgo.h"
22
23 namespace panda::panda_file {
24
25 class ItemDeduper {
26 public:
27 template <class T>
Deduplicate(T * item)28 T *Deduplicate(T *item)
29 {
30 static_assert(std::is_base_of_v<BaseItem, T>);
31
32 if (auto iter = alreadyDedupedItems_.find(item); iter != alreadyDedupedItems_.end()) {
33 ASSERT(item->GetItemType() == iter->second->GetItemType());
34 return static_cast<T *>(iter->second);
35 }
36
37 ItemData itemData(item);
38 auto it = items_.find(itemData);
39 if (it == items_.cend()) {
40 items_.insert(itemData);
41 return item;
42 }
43
44 auto resultItem = it->GetItem();
45 ASSERT(item->GetItemType() == resultItem->GetItemType());
46 auto result = static_cast<T *>(resultItem);
47 if (item != result) {
48 alreadyDedupedItems_.emplace(item, result);
49 if constexpr (!std::is_same_v<T, LineNumberProgramItem>) {
50 item->SetNeedsEmit(false);
51 }
52 }
53
54 return result;
55 }
56
GetUniqueCount() const57 size_t GetUniqueCount() const
58 {
59 return items_.size();
60 }
61
62 private:
63 class ItemWriter : public Writer {
64 public:
ItemWriter(std::vector<uint8_t> * buf,size_t offset)65 ItemWriter(std::vector<uint8_t> *buf, size_t offset) : buf_(buf), offset_(offset) {}
66 ~ItemWriter() override = default;
67
68 NO_COPY_SEMANTIC(ItemWriter);
69 NO_MOVE_SEMANTIC(ItemWriter);
70
WriteByte(uint8_t byte)71 bool WriteByte(uint8_t byte) override
72 {
73 buf_->push_back(byte);
74 ++offset_;
75 return true;
76 }
77
WriteBytes(const std::vector<uint8_t> & bytes)78 bool WriteBytes(const std::vector<uint8_t> &bytes) override
79 {
80 buf_->insert(buf_->end(), bytes.cbegin(), bytes.cend());
81 offset_ += bytes.size();
82 return true;
83 }
84
GetOffset() const85 size_t GetOffset() const override
86 {
87 return offset_;
88 }
89
90 private:
91 std::vector<uint8_t> *buf_;
92 size_t offset_;
93 };
94
95 class ItemData {
96 public:
ItemData(BaseItem * item)97 explicit ItemData(BaseItem *item) : item_(item)
98 {
99 Initialize();
100 }
101
102 ~ItemData() = default;
103
104 DEFAULT_COPY_SEMANTIC(ItemData);
105 NO_MOVE_SEMANTIC(ItemData);
106
GetItem() const107 BaseItem *GetItem() const
108 {
109 return item_;
110 }
111
GetHash() const112 uint32_t GetHash() const
113 {
114 ASSERT(IsInitialized());
115 return hash_;
116 }
117
operator ==(const ItemData & itemData) const118 bool operator==(const ItemData &itemData) const noexcept
119 {
120 ASSERT(IsInitialized());
121 return data_ == itemData.data_;
122 }
123
124 private:
IsInitialized() const125 bool IsInitialized() const
126 {
127 return !data_.empty();
128 }
129
Initialize()130 void Initialize()
131 {
132 ASSERT(item_->NeedsEmit());
133
134 ItemWriter writer(&data_, item_->GetOffset());
135 [[maybe_unused]] auto res = item_->Write(&writer);
136 ASSERT(res);
137 ASSERT(data_.size() == item_->GetSize());
138
139 hash_ = GetHash32(data_.data(), data_.size());
140 }
141
142 BaseItem *item_;
143 uint32_t hash_ {0};
144 std::vector<uint8_t> data_;
145 };
146
147 struct ItemHash {
operator ()panda::panda_file::ItemDeduper::ItemHash148 size_t operator()(const ItemData &itemData) const noexcept
149 {
150 return itemData.GetHash();
151 }
152 };
153
154 std::unordered_set<ItemData, ItemHash> items_;
155 std::unordered_map<BaseItem *, BaseItem *> alreadyDedupedItems_;
156 };
157
158 // NOTE(nsizov): make method for items deletion
159 template <class T, class C, class I, class P, class E, class... Args>
GetOrInsert(C & map,I & items,const P & pos,const E & key,bool isForeign,Args &&...args)160 static T *GetOrInsert(C &map, I &items, const P &pos, const E &key, bool isForeign, Args &&...args)
161 {
162 auto it = map.find(key);
163 if (it != map.cend()) {
164 auto *item = it->second;
165 if (item->IsForeign() == isForeign) {
166 return static_cast<T *>(item);
167 }
168
169 UNREACHABLE();
170 return nullptr;
171 }
172
173 auto ii = items.insert(pos, std::make_unique<T>(std::forward<Args>(args)...));
174 auto *item = static_cast<T *>(ii->get());
175
176 [[maybe_unused]] auto res = map.insert({key, item});
177 ASSERT(res.second);
178 return item;
179 }
180
ItemContainer()181 ItemContainer::ItemContainer()
182 {
183 itemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
184 annotationItemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
185 codeItemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
186 debugItemsEnd_ = items_.insert(items_.end(), std::make_unique<EndItem>());
187 end_ = debugItemsEnd_->get();
188 }
189
GetOrCreateClassItem(const std::string & str)190 ClassItem *ItemContainer::GetOrCreateClassItem(const std::string &str)
191 {
192 return GetOrInsert<ClassItem>(classMap_, items_, itemsEnd_, str, false, str);
193 }
194
GetOrCreateForeignClassItem(const std::string & str)195 ForeignClassItem *ItemContainer::GetOrCreateForeignClassItem(const std::string &str)
196 {
197 return GetOrInsert<ForeignClassItem>(classMap_, foreignItems_, foreignItems_.end(), str, true, str);
198 }
199
GetOrCreateStringItem(const std::string & str)200 StringItem *ItemContainer::GetOrCreateStringItem(const std::string &str)
201 {
202 auto it = classMap_.find(str);
203 if (it != classMap_.cend()) {
204 return it->second->GetNameItem();
205 }
206
207 return GetOrInsert<StringItem>(stringMap_, items_, itemsEnd_, str, false, str);
208 }
209
GetOrCreateLiteralArrayItem(const std::string & id)210 LiteralArrayItem *ItemContainer::GetOrCreateLiteralArrayItem(const std::string &id)
211 {
212 return GetOrInsert<LiteralArrayItem>(literalarrayMap_, items_, itemsEnd_, id, false);
213 }
214
GetOrCreateIntegerValueItem(uint32_t v)215 ScalarValueItem *ItemContainer::GetOrCreateIntegerValueItem(uint32_t v)
216 {
217 return GetOrInsert<ScalarValueItem>(intValueMap_, items_, itemsEnd_, v, false, v);
218 }
219
GetOrCreateLongValueItem(uint64_t v)220 ScalarValueItem *ItemContainer::GetOrCreateLongValueItem(uint64_t v)
221 {
222 return GetOrInsert<ScalarValueItem>(longValueMap_, items_, itemsEnd_, v, false, v);
223 }
224
GetOrCreateFloatValueItem(float v)225 ScalarValueItem *ItemContainer::GetOrCreateFloatValueItem(float v)
226 {
227 return GetOrInsert<ScalarValueItem>(floatValueMap_, items_, itemsEnd_, bit_cast<uint32_t>(v), false, v);
228 }
229
GetOrCreateDoubleValueItem(double v)230 ScalarValueItem *ItemContainer::GetOrCreateDoubleValueItem(double v)
231 {
232 return GetOrInsert<ScalarValueItem>(doubleValueMap_, items_, itemsEnd_, bit_cast<uint64_t>(v), false, v);
233 }
234
GetOrCreateIdValueItem(BaseItem * v)235 ScalarValueItem *ItemContainer::GetOrCreateIdValueItem(BaseItem *v)
236 {
237 return GetOrInsert<ScalarValueItem>(idValueMap_, items_, itemsEnd_, v, false, v);
238 }
239
GetOrCreateProtoItem(TypeItem * retType,const std::vector<MethodParamItem> & params)240 ProtoItem *ItemContainer::GetOrCreateProtoItem(TypeItem *retType, const std::vector<MethodParamItem> ¶ms)
241 {
242 ProtoKey key(retType, params);
243 return GetOrInsert<ProtoItem>(protoMap_, items_, itemsEnd_, key, false, retType, params);
244 }
245
GetOrCreatePrimitiveTypeItem(Type type)246 PrimitiveTypeItem *ItemContainer::GetOrCreatePrimitiveTypeItem(Type type)
247 {
248 return GetOrCreatePrimitiveTypeItem(type.GetId());
249 }
250
GetOrCreatePrimitiveTypeItem(Type::TypeId type)251 PrimitiveTypeItem *ItemContainer::GetOrCreatePrimitiveTypeItem(Type::TypeId type)
252 {
253 return GetOrInsert<PrimitiveTypeItem>(primitiveTypeMap_, items_, itemsEnd_, type, false, type);
254 }
255
CreateLineNumberProgramItem()256 LineNumberProgramItem *ItemContainer::CreateLineNumberProgramItem()
257 {
258 auto it = items_.insert(debugItemsEnd_, std::make_unique<LineNumberProgramItem>());
259 auto *item = static_cast<LineNumberProgramItem *>(it->get());
260 [[maybe_unused]] auto res = lineNumberProgramIndexItem_.Add(item);
261 ASSERT(res);
262 return item;
263 }
264
IncRefLineNumberProgramItem(LineNumberProgramItem * it)265 void ItemContainer::IncRefLineNumberProgramItem(LineNumberProgramItem *it)
266 {
267 lineNumberProgramIndexItem_.IncRefCount(it);
268 }
269
DeduplicateLineNumberProgram(DebugInfoItem * item,ItemDeduper * deduper)270 void ItemContainer::DeduplicateLineNumberProgram(DebugInfoItem *item, ItemDeduper *deduper)
271 {
272 auto *lineNumberProgram = item->GetLineNumberProgram();
273 auto *deduplicated = deduper->Deduplicate(lineNumberProgram);
274 if (deduplicated != lineNumberProgram) {
275 item->SetLineNumberProgram(deduplicated);
276 lineNumberProgramIndexItem_.IncRefCount(deduplicated);
277 lineNumberProgramIndexItem_.DecRefCount(lineNumberProgram);
278 }
279 }
280
DeduplicateDebugInfo(MethodItem * method,ItemDeduper * debugInfoDeduper,ItemDeduper * lineNumberProgramDeduper)281 void ItemContainer::DeduplicateDebugInfo(MethodItem *method, ItemDeduper *debugInfoDeduper,
282 ItemDeduper *lineNumberProgramDeduper)
283 {
284 auto *debugItem = method->GetDebugInfo();
285 if (debugItem == nullptr) {
286 return;
287 }
288
289 DeduplicateLineNumberProgram(debugItem, lineNumberProgramDeduper);
290
291 auto *deduplicated = debugInfoDeduper->Deduplicate(debugItem);
292 if (deduplicated != debugItem) {
293 method->SetDebugInfo(deduplicated);
294 lineNumberProgramIndexItem_.DecRefCount(debugItem->GetLineNumberProgram());
295 }
296 }
297
DeduplicateCode(MethodItem * method,ItemDeduper * codeDeduper)298 static void DeduplicateCode(MethodItem *method, ItemDeduper *codeDeduper)
299 {
300 auto *codeItem = method->GetCode();
301 if (codeItem == nullptr) {
302 return;
303 }
304
305 auto *deduplicated = codeDeduper->Deduplicate(codeItem);
306 if (deduplicated != codeItem) {
307 method->SetCode(deduplicated);
308 deduplicated->AddMethod(method); // we need it for Profile-Guided optimization
309 }
310 }
311
DeduplicateCodeAndDebugInfo()312 void ItemContainer::DeduplicateCodeAndDebugInfo()
313 {
314 ItemDeduper lineNumberProgramDeduper;
315 ItemDeduper debugDeduper;
316 ItemDeduper codeDeduper;
317
318 for (auto &p : classMap_) {
319 auto *item = p.second;
320 if (item->IsForeign()) {
321 continue;
322 }
323
324 auto *classItem = static_cast<ClassItem *>(item);
325
326 classItem->VisitMethods([this, &debugDeduper, &lineNumberProgramDeduper, &codeDeduper](BaseItem *paramItem) {
327 auto *methodItem = static_cast<MethodItem *>(paramItem);
328 DeduplicateDebugInfo(methodItem, &debugDeduper, &lineNumberProgramDeduper);
329 DeduplicateCode(methodItem, &codeDeduper);
330 return true;
331 });
332 }
333 }
334
DeduplicateAnnotationValue(AnnotationItem * annotationItem,ItemDeduper * deduper)335 static void DeduplicateAnnotationValue(AnnotationItem *annotationItem, ItemDeduper *deduper)
336 {
337 auto *elems = annotationItem->GetElements();
338 const auto &tags = annotationItem->GetTags();
339
340 for (size_t i = 0; i < elems->size(); i++) {
341 auto tag = tags[i];
342
343 // try to dedupe only ArrayValueItems
344 switch (tag.GetItem()) {
345 case 'K':
346 case 'L':
347 case 'M':
348 case 'N':
349 case 'O':
350 case 'P':
351 case 'Q':
352 case 'R':
353 case 'S':
354 case 'T':
355 case 'U':
356 case 'V':
357 case 'W':
358 case 'X':
359 case 'Y':
360 case 'Z':
361 case '@':
362 break;
363 default:
364 continue;
365 }
366
367 auto &elem = (*elems)[i];
368 auto *value = elem.GetValue();
369 auto *deduplicated = deduper->Deduplicate(value);
370 if (deduplicated != value) {
371 elem.SetValue(deduplicated);
372 }
373 }
374 }
375
DeduplicateAnnotations(std::vector<AnnotationItem * > * items,ItemDeduper * annotationDeduper,ItemDeduper * valueDeduper)376 static void DeduplicateAnnotations(std::vector<AnnotationItem *> *items, ItemDeduper *annotationDeduper,
377 ItemDeduper *valueDeduper)
378 {
379 for (auto &item : *items) {
380 DeduplicateAnnotationValue(item, valueDeduper);
381 auto *deduplicated = annotationDeduper->Deduplicate(item);
382 if (deduplicated != item) {
383 item = deduplicated;
384 }
385 }
386 }
387
DeduplicateAnnotations()388 void ItemContainer::DeduplicateAnnotations()
389 {
390 ItemDeduper valueDeduper;
391 ItemDeduper annotationDeduper;
392
393 for (auto &p : classMap_) {
394 auto *item = p.second;
395 if (item->IsForeign()) {
396 continue;
397 }
398
399 auto *classItem = static_cast<ClassItem *>(item);
400
401 panda_file::DeduplicateAnnotations(classItem->GetRuntimeAnnotations(), &annotationDeduper, &valueDeduper);
402 panda_file::DeduplicateAnnotations(classItem->GetAnnotations(), &annotationDeduper, &valueDeduper);
403 panda_file::DeduplicateAnnotations(classItem->GetRuntimeTypeAnnotations(), &annotationDeduper, &valueDeduper);
404 panda_file::DeduplicateAnnotations(classItem->GetTypeAnnotations(), &annotationDeduper, &valueDeduper);
405
406 classItem->VisitMethods([&annotationDeduper, &valueDeduper](BaseItem *paramItem) {
407 auto *methodItem = static_cast<MethodItem *>(paramItem);
408 panda_file::DeduplicateAnnotations(methodItem->GetRuntimeAnnotations(), &annotationDeduper, &valueDeduper);
409 panda_file::DeduplicateAnnotations(methodItem->GetAnnotations(), &annotationDeduper, &valueDeduper);
410 panda_file::DeduplicateAnnotations(methodItem->GetRuntimeTypeAnnotations(), &annotationDeduper,
411 &valueDeduper);
412 panda_file::DeduplicateAnnotations(methodItem->GetTypeAnnotations(), &annotationDeduper, &valueDeduper);
413 return true;
414 });
415
416 classItem->VisitFields([&annotationDeduper, &valueDeduper](BaseItem *paramItem) {
417 auto *fieldItem = static_cast<FieldItem *>(paramItem);
418 panda_file::DeduplicateAnnotations(fieldItem->GetRuntimeAnnotations(), &annotationDeduper, &valueDeduper);
419 panda_file::DeduplicateAnnotations(fieldItem->GetAnnotations(), &annotationDeduper, &valueDeduper);
420 panda_file::DeduplicateAnnotations(fieldItem->GetRuntimeTypeAnnotations(), &annotationDeduper,
421 &valueDeduper);
422 panda_file::DeduplicateAnnotations(fieldItem->GetTypeAnnotations(), &annotationDeduper, &valueDeduper);
423 return true;
424 });
425 }
426 }
427
DeduplicateItems(bool computeLayout)428 void ItemContainer::DeduplicateItems(bool computeLayout)
429 {
430 if (computeLayout) {
431 ComputeLayout();
432 }
433 DeduplicateCodeAndDebugInfo();
434 DeduplicateAnnotations();
435 }
436
ComputeLayout()437 uint32_t ItemContainer::ComputeLayout()
438 {
439 uint32_t numClasses = classMap_.size();
440 uint32_t numLiteralarrays = literalarrayMap_.size();
441 uint32_t classIdxOffset = sizeof(File::Header);
442 uint32_t curOffset = classIdxOffset + (numClasses + numLiteralarrays) * ID_SIZE;
443
444 UpdateOrderIndexes();
445 UpdateLiteralIndexes();
446
447 RebuildRegionSection();
448 RebuildLineNumberProgramIndex();
449
450 regionSectionItem_.SetOffset(curOffset);
451 regionSectionItem_.ComputeLayout();
452 curOffset += regionSectionItem_.GetSize();
453
454 for (auto &item : foreignItems_) {
455 curOffset = RoundUp(curOffset, item->Alignment());
456 item->SetOffset(curOffset);
457 item->ComputeLayout();
458 curOffset += item->GetSize();
459 }
460
461 for (auto &item : items_) {
462 if (!item->NeedsEmit()) {
463 continue;
464 }
465
466 curOffset = RoundUp(curOffset, item->Alignment());
467 item->SetOffset(curOffset);
468 item->ComputeLayout();
469 curOffset += item->GetSize();
470 }
471
472 // Line number program should be last because it's size is known only after deduplication
473 curOffset = RoundUp(curOffset, lineNumberProgramIndexItem_.Alignment());
474 lineNumberProgramIndexItem_.SetOffset(curOffset);
475 lineNumberProgramIndexItem_.ComputeLayout();
476 curOffset += lineNumberProgramIndexItem_.GetSize();
477
478 end_->SetOffset(curOffset);
479
480 return curOffset;
481 }
482
RebuildLineNumberProgramIndex()483 void ItemContainer::RebuildLineNumberProgramIndex()
484 {
485 lineNumberProgramIndexItem_.Reset();
486 lineNumberProgramIndexItem_.UpdateItems(nullptr, nullptr);
487 }
488
RebuildRegionSection()489 void ItemContainer::RebuildRegionSection()
490 {
491 regionSectionItem_.Reset();
492
493 for (auto &item : foreignItems_) {
494 ProcessIndexDependecies(item.get());
495 }
496
497 for (auto &item : items_) {
498 if (!item->NeedsEmit()) {
499 continue;
500 }
501
502 ProcessIndexDependecies(item.get());
503 }
504
505 if (!regionSectionItem_.IsEmpty()) {
506 regionSectionItem_.GetCurrentHeader()->SetEnd(end_);
507 }
508
509 regionSectionItem_.UpdateItems();
510 }
511
UpdateOrderIndexes()512 void ItemContainer::UpdateOrderIndexes()
513 {
514 size_t idx = 0;
515
516 for (auto &item : foreignItems_) {
517 item->SetOrderIndex(idx++);
518 item->Visit([&idx](BaseItem *paramItem) {
519 paramItem->SetOrderIndex(idx++);
520 return true;
521 });
522 }
523
524 for (auto &item : items_) {
525 if (!item->NeedsEmit()) {
526 continue;
527 }
528
529 item->SetOrderIndex(idx++);
530 item->Visit([&idx](BaseItem *paramItem) {
531 paramItem->SetOrderIndex(idx++);
532 return true;
533 });
534 }
535
536 end_->SetOrderIndex(idx++);
537 }
538
UpdateLiteralIndexes()539 void ItemContainer::UpdateLiteralIndexes()
540 {
541 size_t idx = 0;
542
543 for (auto &it : literalarrayMap_) {
544 it.second->SetIndex(idx++);
545 }
546 }
547
ReorderItems(panda::panda_file::pgo::ProfileOptimizer * profileOpt)548 void ItemContainer::ReorderItems(panda::panda_file::pgo::ProfileOptimizer *profileOpt)
549 {
550 profileOpt->ProfileGuidedRelayout(items_);
551 }
552
ProcessIndexDependecies(BaseItem * item)553 void ItemContainer::ProcessIndexDependecies(BaseItem *item)
554 {
555 auto deps = item->GetIndexDependencies();
556
557 item->Visit([&deps](BaseItem *paramItem) {
558 const auto &itemDeps = paramItem->GetIndexDependencies();
559 deps.insert(deps.end(), itemDeps.cbegin(), itemDeps.cend());
560 return true;
561 });
562
563 if (regionSectionItem_.IsEmpty()) {
564 regionSectionItem_.AddHeader();
565 regionSectionItem_.GetCurrentHeader()->SetStart(item);
566 }
567
568 if (regionSectionItem_.GetCurrentHeader()->Add(deps)) {
569 return;
570 }
571
572 regionSectionItem_.GetCurrentHeader()->SetEnd(item);
573 regionSectionItem_.AddHeader();
574 regionSectionItem_.GetCurrentHeader()->SetStart(item);
575
576 if (!regionSectionItem_.GetCurrentHeader()->Add(deps)) {
577 LOG(FATAL, PANDAFILE) << "Cannot add " << deps.size() << " items to index";
578 }
579 }
580
WriteHeaderIndexInfo(Writer * writer)581 bool ItemContainer::WriteHeaderIndexInfo(Writer *writer)
582 {
583 if (!writer->Write<uint32_t>(classMap_.size())) {
584 return false;
585 }
586
587 if (!writer->Write<uint32_t>(sizeof(File::Header))) {
588 return false;
589 }
590
591 if (!writer->Write<uint32_t>(lineNumberProgramIndexItem_.GetNumItems())) {
592 return false;
593 }
594
595 if (!writer->Write<uint32_t>(lineNumberProgramIndexItem_.GetOffset())) {
596 return false;
597 }
598
599 if (!writer->Write<uint32_t>(literalarrayMap_.size())) {
600 return false;
601 }
602
603 uint32_t literalarrayIdxOffset = sizeof(File::Header) + classMap_.size() * ID_SIZE;
604 if (!writer->Write<uint32_t>(literalarrayIdxOffset)) {
605 return false;
606 }
607
608 if (!writer->Write<uint32_t>(regionSectionItem_.GetNumHeaders())) {
609 return false;
610 }
611
612 size_t indexSectionOff = literalarrayIdxOffset + literalarrayMap_.size() * ID_SIZE;
613 return writer->Write<uint32_t>(indexSectionOff);
614 }
615
WriteHeader(Writer * writer,ssize_t * checksumOffset)616 bool ItemContainer::WriteHeader(Writer *writer, ssize_t *checksumOffset)
617 {
618 uint32_t fileSize = ComputeLayout();
619
620 std::vector<uint8_t> magic;
621 magic.assign(File::MAGIC.cbegin(), File::MAGIC.cend());
622 if (!writer->WriteBytes(magic)) {
623 return false;
624 }
625
626 *checksumOffset = static_cast<ssize_t>(writer->GetOffset());
627 uint32_t checksum = 0;
628 if (!writer->Write(checksum)) {
629 return false;
630 }
631 writer->CountChecksum(true);
632
633 std::vector<uint8_t> versionVec(std::begin(VERSION), std::end(VERSION));
634 if (!writer->WriteBytes(versionVec)) {
635 return false;
636 }
637
638 if (!writer->Write(fileSize)) {
639 return false;
640 }
641
642 uint32_t foreignOffset = GetForeignOffset();
643 if (!writer->Write(foreignOffset)) {
644 return false;
645 }
646
647 uint32_t foreignSize = GetForeignSize();
648 if (!writer->Write(foreignSize)) {
649 return false;
650 }
651
652 if (!writer->Write<uint32_t>(static_cast<uint32_t>(isQuickened_))) {
653 return false;
654 }
655
656 return WriteHeaderIndexInfo(writer);
657 }
658
Write(Writer * writer,bool deduplicateItems,bool computeLayout)659 bool ItemContainer::Write(Writer *writer, bool deduplicateItems, bool computeLayout)
660 {
661 if (deduplicateItems) {
662 DeduplicateItems(computeLayout);
663 }
664
665 ssize_t checksumOffset = -1;
666 if (!WriteHeader(writer, &checksumOffset)) {
667 return false;
668 }
669 ASSERT(checksumOffset != -1);
670
671 // Write class idx
672
673 for (auto &entry : classMap_) {
674 if (!writer->Write(entry.second->GetOffset())) {
675 return false;
676 }
677 }
678
679 // Write literalArray idx
680
681 for (auto &entry : literalarrayMap_) {
682 if (!writer->Write(entry.second->GetOffset())) {
683 return false;
684 }
685 }
686
687 // Write index section
688
689 if (!regionSectionItem_.Write(writer)) {
690 return false;
691 }
692
693 for (auto &item : foreignItems_) {
694 if (!writer->Align(item->Alignment())) {
695 return false;
696 }
697
698 if (!item->Write(writer)) {
699 return false;
700 }
701 }
702
703 for (auto &item : items_) {
704 if (!item->NeedsEmit()) {
705 continue;
706 }
707
708 if (!writer->Align(item->Alignment())) {
709 return false;
710 }
711
712 if (!item->Write(writer)) {
713 return false;
714 }
715 }
716
717 if (!writer->Align(lineNumberProgramIndexItem_.Alignment())) {
718 return false;
719 }
720
721 // Write line number program idx
722
723 if (!lineNumberProgramIndexItem_.Write(writer)) {
724 return false;
725 }
726
727 writer->CountChecksum(false);
728 writer->WriteChecksum(checksumOffset);
729
730 return true;
731 }
732
GetStat()733 std::map<std::string, size_t> ItemContainer::GetStat()
734 {
735 std::map<std::string, size_t> stat;
736
737 DeduplicateItems();
738 ComputeLayout();
739
740 stat["header_item"] = sizeof(File::Header);
741 stat["class_idx_item"] = classMap_.size() * ID_SIZE;
742 stat["line_number_program_idx_item"] = lineNumberProgramIndexItem_.GetNumItems() * ID_SIZE;
743 stat["literalarray_idx"] = literalarrayMap_.size() * ID_SIZE;
744
745 stat["region_section_item"] = regionSectionItem_.GetSize();
746 stat["foreign_item"] = GetForeignSize();
747
748 size_t numIns = 0;
749 size_t codesize = 0;
750 for (auto &item : items_) {
751 if (!item->NeedsEmit()) {
752 continue;
753 }
754
755 const auto &name = item->GetName();
756 size_t size = item->GetSize();
757 auto it = stat.find(name);
758 if (it != stat.cend()) {
759 stat[name] += size;
760 } else if (size != 0) {
761 stat[name] = size;
762 }
763 if (name == "code_item") {
764 numIns += static_cast<CodeItem *>(item.get())->GetNumInstructions();
765 codesize += static_cast<CodeItem *>(item.get())->GetCodeSize();
766 }
767 }
768 stat["instructions_number"] = numIns;
769 stat["codesize"] = codesize;
770
771 return stat;
772 }
773
DumpItemsStat(std::ostream & os) const774 void ItemContainer::DumpItemsStat(std::ostream &os) const
775 {
776 struct Stat {
777 size_t n;
778 size_t totalSize;
779 };
780
781 std::map<std::string, Stat> stat;
782
783 auto collectStat = [&stat](auto &items) {
784 for (auto &item : items) {
785 if (!item->NeedsEmit()) {
786 continue;
787 }
788
789 const auto &name = item->GetName();
790 size_t size = item->GetSize();
791 auto it = stat.find(name);
792 if (it != stat.cend()) {
793 stat[name].n += 1;
794 stat[name].totalSize += size;
795 } else if (size != 0) {
796 stat[name] = {1, size};
797 }
798 }
799 };
800
801 collectStat(foreignItems_);
802 collectStat(items_);
803
804 for (auto &[name, elem] : stat) {
805 os << name << ":" << std::endl;
806 os << " n = " << elem.n << std::endl;
807 os << " total size = " << elem.totalSize << std::endl;
808 }
809 }
810
GetForeignOffset() const811 size_t ItemContainer::GetForeignOffset() const
812 {
813 if (foreignItems_.empty()) {
814 return 0;
815 }
816
817 return foreignItems_.front()->GetOffset();
818 }
819
GetForeignSize() const820 size_t ItemContainer::GetForeignSize() const
821 {
822 if (foreignItems_.empty()) {
823 return 0;
824 }
825
826 size_t begin = foreignItems_.front()->GetOffset();
827 size_t end = foreignItems_.back()->GetOffset() + foreignItems_.back()->GetSize();
828
829 return end - begin;
830 }
831
Write(Writer * writer)832 bool ItemContainer::RegionHeaderItem::Write(Writer *writer)
833 {
834 ASSERT(GetOffset() == writer->GetOffset());
835 ASSERT(start_ != nullptr);
836 ASSERT(start_->GetOffset() != 0);
837 ASSERT(end_ != nullptr);
838 ASSERT(end_->GetOffset() != 0);
839
840 if (!writer->Write<uint32_t>(start_->GetOffset())) {
841 return false;
842 }
843
844 if (!writer->Write<uint32_t>(end_->GetOffset())) {
845 return false;
846 }
847
848 for (auto *indexItem : indexes_) {
849 if (!writer->Write<uint32_t>(indexItem->GetNumItems())) {
850 return false;
851 }
852
853 ASSERT(indexItem->GetOffset() != 0);
854 if (!writer->Write<uint32_t>(indexItem->GetOffset())) {
855 return false;
856 }
857 }
858
859 return true;
860 }
861
Add(const std::list<IndexedItem * > & items)862 bool ItemContainer::RegionHeaderItem::Add(const std::list<IndexedItem *> &items)
863 {
864 std::list<IndexedItem *> addedItems;
865
866 for (auto *item : items) {
867 auto type = item->GetIndexType();
868 ASSERT(type != IndexType::NONE);
869
870 auto *indexItem = GetIndexByType(type);
871
872 if (indexItem->Has(item)) {
873 continue;
874 }
875
876 if (!indexItem->Add(item)) {
877 Remove(addedItems);
878 return false;
879 }
880
881 addedItems.push_back(item);
882 }
883
884 return true;
885 }
886
Remove(const std::list<IndexedItem * > & items)887 void ItemContainer::RegionHeaderItem::Remove(const std::list<IndexedItem *> &items)
888 {
889 for (auto *item : items) {
890 auto type = item->GetIndexType();
891 ASSERT(type != IndexType::NONE);
892
893 auto *indexItem = GetIndexByType(type);
894 indexItem->Remove(item);
895 }
896 }
897
Write(Writer * writer)898 bool ItemContainer::IndexItem::Write(Writer *writer)
899 {
900 ASSERT(GetOffset() == writer->GetOffset());
901
902 for (auto *item : index_) {
903 if (!writer->Write<uint32_t>(item->GetOffset())) {
904 return false;
905 }
906 }
907
908 return true;
909 }
910
GetItemType() const911 ItemTypes ItemContainer::IndexItem::GetItemType() const
912 {
913 switch (type_) {
914 case IndexType::CLASS:
915 return ItemTypes::CLASS_INDEX_ITEM;
916 case IndexType::METHOD:
917 return ItemTypes::METHOD_INDEX_ITEM;
918 case IndexType::FIELD:
919 return ItemTypes::FIELD_INDEX_ITEM;
920 case IndexType::PROTO:
921 return ItemTypes::PROTO_INDEX_ITEM;
922 case IndexType::LINE_NUMBER_PROG:
923 return ItemTypes::LINE_NUMBER_PROGRAM_INDEX_ITEM;
924 default:
925 break;
926 }
927
928 UNREACHABLE();
929 }
930
Add(IndexedItem * item)931 bool ItemContainer::IndexItem::Add(IndexedItem *item)
932 {
933 auto size = index_.size();
934 ASSERT(size <= maxIndex_);
935
936 if (size == maxIndex_) {
937 return false;
938 }
939
940 auto res = index_.insert(item);
941 ASSERT(res.second);
942
943 return res.second;
944 }
945
AddHeader()946 void ItemContainer::RegionSectionItem::AddHeader()
947 {
948 std::vector<IndexItem *> indexItems;
949 for (size_t i = 0; i < INDEX_COUNT_16; i++) {
950 auto type = static_cast<IndexType>(i);
951 indexes_.emplace_back(type, MAX_INDEX_16);
952 indexItems.push_back(&indexes_.back());
953 }
954 headers_.emplace_back(indexItems);
955 }
956
CalculateSize() const957 size_t ItemContainer::RegionSectionItem::CalculateSize() const
958 {
959 size_t size = headers_.size() * sizeof(File::RegionHeader);
960 for (auto &indexItem : indexes_) {
961 size += indexItem.GetSize();
962 }
963 return size;
964 }
965
ComputeLayout()966 void ItemContainer::RegionSectionItem::ComputeLayout()
967 {
968 size_t offset = GetOffset();
969
970 for (auto &header : headers_) {
971 header.SetOffset(offset);
972 header.ComputeLayout();
973 offset += header.GetSize();
974 }
975
976 for (auto &index : indexes_) {
977 index.SetOffset(offset);
978 index.ComputeLayout();
979 offset += index.GetSize();
980 }
981 }
982
Write(Writer * writer)983 bool ItemContainer::RegionSectionItem::Write(Writer *writer)
984 {
985 ASSERT(GetOffset() == writer->GetOffset());
986
987 for (auto &header : headers_) {
988 if (!header.Write(writer)) {
989 return false;
990 }
991 }
992
993 for (auto &index : indexes_) {
994 if (!index.Write(writer)) {
995 return false;
996 }
997 }
998
999 return true;
1000 }
1001
ProtoKey(TypeItem * retType,const std::vector<MethodParamItem> & params)1002 ItemContainer::ProtoKey::ProtoKey(TypeItem *retType, const std::vector<MethodParamItem> ¶ms)
1003 {
1004 Add(retType);
1005 for (const auto ¶m : params) {
1006 Add(param.GetType());
1007 }
1008 size_t shortyHash = std::hash<std::string>()(shorty_);
1009 size_t retTypeHash = std::hash<TypeItem *>()(retType);
1010 // combine hashes of shorty and ref_types
1011 hash_ = panda::MergeHashes(shortyHash, retTypeHash);
1012 // combine hashes of all param types
1013 for (const auto &item : params) {
1014 size_t paramTypeHash = std::hash<TypeItem *>()(item.GetType());
1015 hash_ = panda::MergeHashes(hash_, paramTypeHash);
1016 }
1017 }
1018
Add(TypeItem * item)1019 void ItemContainer::ProtoKey::Add(TypeItem *item)
1020 {
1021 auto type = item->GetType();
1022 shorty_.append(Type::GetSignatureByTypeId(type));
1023 if (type.IsReference()) {
1024 refTypes_.push_back(item);
1025 }
1026 }
1027
1028 } // namespace panda::panda_file
1029