1 /**
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "runtime/mem/runslots.h"
17
18 #include <cstring>
19
20 #include "runtime/include/object_header.h"
21
22 namespace panda::mem {
23
24 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
25 #define LOG_RUNSLOTS(level) LOG(level, ALLOC) << "RunSlots: "
26
27 template <typename LockTypeT>
Initialize(size_t slot_size,uintptr_t pool_pointer,bool initialize_lock)28 void RunSlots<LockTypeT>::Initialize(size_t slot_size, uintptr_t pool_pointer, bool initialize_lock)
29 {
30 ASAN_UNPOISON_MEMORY_REGION(this, RUNSLOTS_SIZE);
31 LOG_RUNSLOTS(DEBUG) << "Initializing RunSlots:";
32 ASSERT_PRINT((slot_size >= SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES)), "Size of slot in RunSlots is too small");
33 ASSERT_PRINT((slot_size <= SlotToSize(SlotsSizes::SLOT_MAX_SIZE_BYTES)), "Size of slot in RunSlots is too big");
34 ASSERT(pool_pointer != 0);
35 pool_pointer_ = pool_pointer;
36 ASSERT_PRINT(!(ToUintPtr(this) & RUNSLOTS_ALIGNMENT_MASK), "RunSlots object must have alignment");
37 slot_size_ = slot_size;
38 size_t first_slot_offset = ComputeFirstSlotOffset(slot_size);
39 first_uninitialized_slot_offset_ = first_slot_offset;
40 ASSERT(first_uninitialized_slot_offset_ != 0);
41 next_free_ = nullptr;
42 used_slots_ = 0;
43 next_runslot_ = nullptr;
44 prev_runslot_ = nullptr;
45 if (initialize_lock) {
46 new (&lock_) LockTypeT();
47 }
48 memset_s(bitmap_.data(), BITMAP_ARRAY_SIZE, 0x0, BITMAP_ARRAY_SIZE);
49 LOG_RUNSLOTS(DEBUG) << "- Memory started from = 0x" << std::hex << ToUintPtr(this);
50 LOG_RUNSLOTS(DEBUG) << "- Pool size = " << RUNSLOTS_SIZE << " bytes";
51 LOG_RUNSLOTS(DEBUG) << "- Slots size = " << slot_size_ << " bytes";
52 LOG_RUNSLOTS(DEBUG) << "- First free slot = " << std::hex << static_cast<void *>(next_free_);
53 LOG_RUNSLOTS(DEBUG) << "- First uninitialized slot offset = " << std::hex
54 << static_cast<void *>(ToVoidPtr(first_uninitialized_slot_offset_));
55 LOG_RUNSLOTS(DEBUG) << "- Pool pointer = " << std::hex << static_cast<void *>(ToVoidPtr(pool_pointer_));
56 LOG_RUNSLOTS(DEBUG) << "Successful finished RunSlots init";
57 ASAN_POISON_MEMORY_REGION(this, RUNSLOTS_SIZE);
58 }
59
60 template <typename LockTypeT>
PopFreeSlot()61 FreeSlot *RunSlots<LockTypeT>::PopFreeSlot()
62 {
63 ASAN_UNPOISON_MEMORY_REGION(this, GetHeaderSize());
64 FreeSlot *free_slot = nullptr;
65 if (next_free_ == nullptr) {
66 void *uninitialized_slot = PopUninitializedSlot();
67 if (uninitialized_slot == nullptr) {
68 LOG_RUNSLOTS(DEBUG) << "Failed to get free slot - there are no free slots in RunSlots";
69 ASAN_POISON_MEMORY_REGION(this, GetHeaderSize());
70 return nullptr;
71 }
72 free_slot = static_cast<FreeSlot *>(uninitialized_slot);
73 } else {
74 free_slot = next_free_;
75 ASAN_UNPOISON_MEMORY_REGION(free_slot, sizeof(FreeSlot));
76 next_free_ = next_free_->GetNext();
77 ASAN_POISON_MEMORY_REGION(free_slot, sizeof(FreeSlot));
78 }
79 MarkAsOccupied(free_slot);
80 used_slots_++;
81 LOG_RUNSLOTS(DEBUG) << "Successfully get free slot " << std::hex << static_cast<void *>(free_slot)
82 << ". Used slots in this RunSlots = " << std::dec << used_slots_;
83 ASAN_POISON_MEMORY_REGION(this, GetHeaderSize());
84 return free_slot;
85 }
86
87 template <typename LockTypeT>
PushFreeSlot(FreeSlot * mem_slot)88 void RunSlots<LockTypeT>::PushFreeSlot(FreeSlot *mem_slot)
89 {
90 ASAN_UNPOISON_MEMORY_REGION(this, GetHeaderSize());
91 LOG_RUNSLOTS(DEBUG) << "Free slot in RunSlots at addr " << std::hex << static_cast<void *>(mem_slot);
92 // We need to poison/unpoison mem_slot here because we could allocate an object with size less than FreeSlot size
93 ASAN_UNPOISON_MEMORY_REGION(mem_slot, sizeof(FreeSlot));
94 mem_slot->SetNext(next_free_);
95 ASAN_POISON_MEMORY_REGION(mem_slot, sizeof(FreeSlot));
96 next_free_ = mem_slot;
97 MarkAsFree(mem_slot);
98 used_slots_--;
99 LOG_RUNSLOTS(DEBUG) << "Used slots in RunSlots = " << used_slots_;
100 ASAN_POISON_MEMORY_REGION(this, GetHeaderSize());
101 }
102
103 template <typename LockTypeT>
ComputeFirstSlotOffset(size_t slot_size)104 size_t RunSlots<LockTypeT>::ComputeFirstSlotOffset(size_t slot_size)
105 {
106 size_t slots_for_header = (GetHeaderSize() / slot_size);
107 if ((GetHeaderSize() % slot_size) > 0) {
108 slots_for_header++;
109 }
110 return slots_for_header * slot_size;
111 }
112
113 template <typename LockTypeT>
PopUninitializedSlot()114 void *RunSlots<LockTypeT>::PopUninitializedSlot()
115 {
116 if (first_uninitialized_slot_offset_ != 0) {
117 ASSERT(RUNSLOTS_SIZE > first_uninitialized_slot_offset_);
118 void *uninitialized_slot = ToVoidPtr(ToUintPtr(this) + first_uninitialized_slot_offset_);
119 first_uninitialized_slot_offset_ += slot_size_;
120 if (first_uninitialized_slot_offset_ >= RUNSLOTS_SIZE) {
121 ASSERT(first_uninitialized_slot_offset_ == RUNSLOTS_SIZE);
122 first_uninitialized_slot_offset_ = 0;
123 }
124 return uninitialized_slot;
125 }
126 return nullptr;
127 }
128
129 template <typename LockTypeT>
MarkAsOccupied(const FreeSlot * slot_mem)130 void RunSlots<LockTypeT>::MarkAsOccupied(const FreeSlot *slot_mem)
131 {
132 uintptr_t bit_index =
133 (ToUintPtr(slot_mem) & (RUNSLOTS_SIZE - 1U)) >> SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES_POWER_OF_TWO);
134 uintptr_t array_index = bit_index >> BITS_IN_BYTE_POWER_OF_TWO;
135 uintptr_t bit_in_array_element = bit_index & ((1U << BITS_IN_BYTE_POWER_OF_TWO) - 1U);
136 ASSERT(!(bitmap_[array_index] & (1U << bit_in_array_element)));
137 bitmap_[array_index] |= 1U << bit_in_array_element;
138 }
139
140 template <typename LockTypeT>
MarkAsFree(const FreeSlot * slot_mem)141 void RunSlots<LockTypeT>::MarkAsFree(const FreeSlot *slot_mem)
142 {
143 uintptr_t bit_index =
144 (ToUintPtr(slot_mem) & (RUNSLOTS_SIZE - 1U)) >> SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES_POWER_OF_TWO);
145 uintptr_t array_index = bit_index >> BITS_IN_BYTE_POWER_OF_TWO;
146 uintptr_t bit_in_array_element = bit_index & ((1U << BITS_IN_BYTE_POWER_OF_TWO) - 1U);
147 ASSERT(bitmap_[array_index] & (1U << bit_in_array_element));
148 bitmap_[array_index] ^= 1U << bit_in_array_element;
149 }
150
151 template <typename LockTypeT>
BitMapToSlot(size_t array_index,size_t bit)152 FreeSlot *RunSlots<LockTypeT>::BitMapToSlot(size_t array_index, size_t bit)
153 {
154 return static_cast<FreeSlot *>(
155 ToVoidPtr(ToUintPtr(this) + (((array_index << BITS_IN_BYTE_POWER_OF_TWO) + bit)
156 << SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES_POWER_OF_TWO))));
157 }
158
159 template <typename LockTypeT>
operator ()(RunSlots * run)160 size_t RunSlots<LockTypeT>::RunVerifier::operator()(RunSlots *run)
161 {
162 // 1. should verify whether run's bracket size is the same as recorded in RunSlotsAllocator, but RunSlotsAllocator
163 // does not record this
164 // 2. should verify thread local run's ownership, but thread local run not implemented yet
165
166 // check alloc'ed size
167 auto size_check_func = [this, &run](const ObjectHeader *obj) {
168 auto size_power_of_two = ConvertToPowerOfTwoUnsafe(obj->ObjectSize());
169 if ((1U << size_power_of_two) != run->GetSlotsSize()) {
170 ++(this->fail_cnt_);
171 }
172 };
173 run->IterateOverOccupiedSlots(size_check_func);
174
175 return fail_cnt_;
176 }
177
178 template <typename LockTypeT>
IsLive(const ObjectHeader * obj) const179 bool RunSlots<LockTypeT>::IsLive(const ObjectHeader *obj) const
180 {
181 ASAN_UNPOISON_MEMORY_REGION(this, GetHeaderSize());
182 uintptr_t mem_tail_by_runslots = ToUintPtr(obj) & (RUNSLOTS_SIZE - 1U);
183 if ((mem_tail_by_runslots & (static_cast<uintptr_t>(slot_size_) - 1)) != 0) {
184 ASAN_POISON_MEMORY_REGION(this, GetHeaderSize());
185 return false;
186 }
187 uintptr_t bit_index = mem_tail_by_runslots >> SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES_POWER_OF_TWO);
188 uintptr_t array_index = bit_index >> BITS_IN_BYTE_POWER_OF_TWO;
189 uintptr_t bit_in_array_element = bit_index & ((1U << BITS_IN_BYTE_POWER_OF_TWO) - 1U);
190 auto live_word = bitmap_[array_index] & (1U << bit_in_array_element);
191 ASAN_POISON_MEMORY_REGION(this, GetHeaderSize());
192 return live_word != 0;
193 }
194
195 template class RunSlots<RunSlotsLockConfig::CommonLock>;
196 template class RunSlots<RunSlotsLockConfig::DummyLock>;
197 } // namespace panda::mem
198