1 /** 2 * Copyright (c) 2024 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 #ifndef PANDA_RUNTIME_MEM_TLAB_H 16 #define PANDA_RUNTIME_MEM_TLAB_H 17 18 #include "libpandabase/utils/logger.h" 19 #include "libpandabase/mem/mem.h" 20 #include "libpandabase/mem/pool_map.h" 21 #include "libpandabase/mem/mem_range.h" 22 23 namespace ark::mem { 24 25 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) 26 #define LOG_TLAB_ALLOCATOR(level) LOG(level, ALLOC) << "TLAB: " 27 28 #ifdef NDEBUG 29 static constexpr bool PANDA_TRACK_TLAB_ALLOCATIONS = false; 30 #else 31 static constexpr bool PANDA_TRACK_TLAB_ALLOCATIONS = true; 32 #endif 33 // Current TLAB structure looks like that: 34 // 35 // |--------------------------| 36 // |........TLAB class........| 37 // |--------------------------| 38 // |.........end addr.........|------------| 39 // |.......free pointer.......|--------| | 40 // |........start addr........|----| | | 41 // |--------------------------| | | | 42 // | | | 43 // | | | 44 // |--------------------------| | | | 45 // |..Memory for allocations..| | | | 46 // |--------------------------| | | | 47 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|<---| | | 48 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 49 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 50 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 51 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 52 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|<-------| | 53 // |..........................| | 54 // |..........................| | 55 // |..........................| | 56 // |..........................| | 57 // |..........................|<-----------| 58 // |--------------------------| 59 // 60 // Each TLAB is connected with certain thread: 61 // (NOTE: In current implementation, we can reach max one TLAB from a thread metadata) 62 // NOTE(aemelenko): If we don't use links on next, prev TLABS, 63 // it is better to remove these fields. 64 // |------------------------| |---------------| 65 // | Thread Metainformation | ----------> | Current TLAB |---- 66 // |------------------------| |---------------| | 67 // | 68 // | 69 // |---------------| | 70 // | Previous TLAB |<---| 71 // |---------------| 72 73 // How to use TLAB from the compiler if we want to allocate an object for class 'cls' with size 'allocation_size': 74 // NOTE: If we have PANDA_TRACK_TLAB_ALLOCATIONS option on, JIT should always call Runtime at AllocateObject calls. 75 // Pseudocode: 76 // IF allocation_size > TLAB::GetMaxSize() || IsFinalizable(cls) 77 // call HeapManager::AllocateObject(obj_class, allocation_size) 78 // ELSE 79 // // We should use TLS for this purpose. 80 // // Read current TLAB pointer from TLS: 81 // load TLS.TLAB -> cur_TLAB 82 // // Read uintptr_t value from TLAB structure: 83 // load (AddressOf(cur_TLAB) + TLAB::TLABFreePointerOffset) -> free_pointer 84 // // Read uintptr_t value from TLAB structure: 85 // load (AddressOf(cur_TLAB) + TLAB::TLABEndAddrOffset) -> end_pointer 86 // // Align the size of an object to DEFAULT_ALIGNMENT_IN_BYTES 87 // // One can use GetAlignedObjectSize() method for that. 88 // align (allocation_size, DEFAULT_ALIGNMENT_IN_BYTES) -> allocation_size 89 // IF free_pointer + allocation_size > end_pointer 90 // // Goto slow path 91 // call HeapManager::AllocateObject(obj_class, allocation_size) 92 // // Calculate new_free_pointer: 93 // new_free_pointer = AddressOf(free_pointer) + allocation_size 94 // // Store new_free_pointer to (cur_TLAB + TLAB::TLABFreePointerOffset): 95 // store (AddressOf(cur_TLAB) + TLAB::TLABFreePointerOffset) <- new_free_pointer 96 // return free_pointer 97 // 98 // After that the Compiler should initialize class word inside new object and 99 // set correct GC bits in the mark word: 100 // ObjectHeader obj_header 101 // obj_header.SetClass(cls) 102 // GetGC()->InitGCBitsForAllocationInTLAB(&obj_header) 103 // free_pointer <- obj_header 104 // 105 // Runtime should provide these parameters: 106 // HeapManager::GetTLABMaxAllocSize() - max size that can be allocated via TLAB. (depends on the allocator used by GC) 107 // HeapManager::UseTLABForAllocations() - do we need to use TLABs for allocations. (it is a runtime option) 108 // GC::InitGCBitsForAllocationInTLAB() - method for initialize GC bits inside the object header 109 // during allocations through TLAB 110 // TLAB::TLABFreePointerOffset() - an offset of a free pointer field inside TLAB. 111 // TLAB::TLABEndAddrOffset() - an offset of a end buffer pointer field inside TLAB. 112 // 113 114 class TLAB { 115 public: 116 /** 117 * @brief Construct TLAB with the buffer at @param address with @param size 118 * @param address - a pointer into the memory where TLAB memory will be created 119 * @param size - a size of the allocated memory for the TLAB 120 */ 121 explicit TLAB(void *address = nullptr, size_t size = 0); 122 ~TLAB(); 123 124 void Destroy(); 125 126 /** 127 * @brief Fill a TLAB with the buffer at @param address with @param size 128 * @param address - a pointer into the memory where TLAB memory will be created 129 * @param size - a size of the allocated memory for the TLAB 130 */ 131 void Fill(void *address, size_t size); 132 133 /// @brief Set TLAB to be empty Reset()134 void Reset() 135 { 136 Fill(nullptr, 0U); 137 } 138 IsUninitialized()139 bool IsUninitialized() 140 { 141 return (memoryStartAddr_ == nullptr) || (curFreePosition_ == nullptr) || (memoryEndAddr_ == nullptr); 142 } 143 144 NO_MOVE_SEMANTIC(TLAB); 145 NO_COPY_SEMANTIC(TLAB); 146 147 /** 148 * @brief Allocates memory with size @param size and aligned with DEFAULT_ALIGNMENT alignment 149 * @param size - size of the allocated memory 150 * @return pointer to the allocated memory on success, or nullptr on fail 151 */ 152 void *Alloc(size_t size); 153 154 /** 155 * @brief Iterates over all objects in this TLAB 156 * @param object_visitor 157 */ 158 void IterateOverObjects(const std::function<void(ObjectHeader *objectHeader)> &objectVisitor); 159 160 /** 161 * @brief Iterates over objects in the range inclusively. 162 * @param mem_visitor - function pointer or functor 163 * @param mem_range - memory range 164 */ 165 void IterateOverObjectsInRange(const std::function<void(ObjectHeader *objectHeader)> &memVisitor, 166 const MemRange &memRange); 167 168 /** 169 * Collects dead objects and move alive with provided visitor 170 * @param death_checker - functor for check if object alive 171 * @param object_move_visitor - object visitor 172 */ 173 template <typename ObjectMoveVisitorT> CollectAndMove(const GCObjectVisitor & deathChecker,const ObjectMoveVisitorT & objectMoveVisitor)174 void CollectAndMove(const GCObjectVisitor &deathChecker, const ObjectMoveVisitorT &objectMoveVisitor) 175 { 176 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove started"; 177 IterateOverObjects([&](ObjectHeader *objectHeader) { 178 // We are interested only in moving alive objects, after that we cleanup this buffer 179 if (deathChecker(objectHeader) == ObjectStatus::ALIVE_OBJECT) { 180 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove found alive object with addr " << objectHeader; 181 objectMoveVisitor(objectHeader); 182 } 183 }); 184 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove finished"; 185 } 186 187 bool ContainObject(const ObjectHeader *obj); 188 189 bool IsLive(const ObjectHeader *obj); 190 GetNextTLAB()191 TLAB *GetNextTLAB() 192 { 193 return nextTlab_; 194 } 195 GetPrevTLAB()196 TLAB *GetPrevTLAB() 197 { 198 return prevTlab_; 199 } 200 SetNextTLAB(TLAB * tlabPointer)201 void SetNextTLAB(TLAB *tlabPointer) 202 { 203 nextTlab_ = tlabPointer; 204 } 205 SetPrevTLAB(TLAB * tlabPointer)206 void SetPrevTLAB(TLAB *tlabPointer) 207 { 208 prevTlab_ = tlabPointer; 209 } 210 GetStartAddr()211 void *GetStartAddr() const 212 { 213 return memoryStartAddr_; 214 } 215 GetEndAddr()216 void *GetEndAddr() const 217 { 218 return memoryEndAddr_; 219 } 220 GetCurPos()221 void *GetCurPos() const 222 { 223 return curFreePosition_; 224 } 225 GetOccupiedSize()226 size_t GetOccupiedSize() const 227 { 228 ASSERT(ToUintPtr(curFreePosition_) >= ToUintPtr(memoryStartAddr_)); 229 return ToUintPtr(curFreePosition_) - ToUintPtr(memoryStartAddr_); 230 } 231 GetFreeSize()232 size_t GetFreeSize() const 233 { 234 ASSERT(ToUintPtr(curFreePosition_) >= ToUintPtr(memoryStartAddr_)); 235 ASSERT(ToUintPtr(curFreePosition_) <= ToUintPtr(memoryEndAddr_)); 236 return ToUintPtr(memoryEndAddr_) - ToUintPtr(curFreePosition_); 237 } 238 GetMemRangeForOccupiedMemory()239 MemRange GetMemRangeForOccupiedMemory() const 240 { 241 return MemRange(ToUintPtr(memoryStartAddr_), ToUintPtr(curFreePosition_) - 1); 242 } 243 TLABStartAddrOffset()244 static constexpr size_t TLABStartAddrOffset() 245 { 246 return MEMBER_OFFSET(TLAB, memoryStartAddr_); 247 } 248 TLABFreePointerOffset()249 static constexpr size_t TLABFreePointerOffset() 250 { 251 return MEMBER_OFFSET(TLAB, curFreePosition_); 252 } 253 TLABEndAddrOffset()254 static constexpr size_t TLABEndAddrOffset() 255 { 256 return MEMBER_OFFSET(TLAB, memoryEndAddr_); 257 } 258 GetAllocatorType()259 static constexpr AllocatorType GetAllocatorType() 260 { 261 return AllocatorType::TLAB_ALLOCATOR; 262 } 263 264 static constexpr float MIN_DESIRED_FILL_FRACTION = 0.5; 265 GetSize()266 size_t GetSize() 267 { 268 ASSERT(ToUintPtr(memoryEndAddr_) >= ToUintPtr(memoryStartAddr_)); 269 return ToUintPtr(memoryEndAddr_) - ToUintPtr(memoryStartAddr_); 270 } 271 GetFillFraction()272 float GetFillFraction() 273 { 274 size_t size = GetSize(); 275 if (size == 0) { 276 // ZERO tlab case 277 // consider it is always full 278 return 1.0; 279 } 280 float fillFraction = static_cast<float>(GetOccupiedSize()) / static_cast<float>(size); 281 ASSERT(fillFraction <= 1.0); 282 return fillFraction; 283 } 284 285 private: 286 TLAB *nextTlab_; 287 TLAB *prevTlab_; 288 // NOTE(aemelenko): Maybe use OBJECT_POINTER_SIZE here for heap allocation. 289 void *memoryStartAddr_ {nullptr}; 290 void *memoryEndAddr_ {nullptr}; 291 void *curFreePosition_ {nullptr}; 292 }; 293 294 #undef LOG_TLAB_ALLOCATOR 295 296 } // namespace ark::mem 297 298 #endif // PANDA_RUNTIME_MEM_TLAB_H 299