1 /** 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 #ifndef PANDA_RUNTIME_MEM_TLAB_H 16 #define PANDA_RUNTIME_MEM_TLAB_H 17 18 #include "libpandabase/utils/logger.h" 19 #include "libpandabase/mem/mem.h" 20 #include "libpandabase/mem/pool_map.h" 21 #include "libpandabase/mem/mem_range.h" 22 23 namespace panda::mem { 24 25 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) 26 #define LOG_TLAB_ALLOCATOR(level) LOG(level, ALLOC) << "TLAB: " 27 28 static constexpr size_t PANDA_TLAB_SIZE = 4_KB; 29 static constexpr size_t PANDA_TLAB_MAX_ALLOC_SIZE = PANDA_TLAB_SIZE; 30 31 #ifdef NDEBUG 32 static constexpr bool PANDA_TRACK_TLAB_ALLOCATIONS = false; 33 #else 34 static constexpr bool PANDA_TRACK_TLAB_ALLOCATIONS = true; 35 #endif 36 // Current TLAB structure looks like that: 37 // 38 // |--------------------------| 39 // |........TLAB class........| 40 // |--------------------------| 41 // |.........end addr.........|------------| 42 // |.......free pointer.......|--------| | 43 // |........start addr........|----| | | 44 // |--------------------------| | | | 45 // | | | 46 // | | | 47 // |--------------------------| | | | 48 // |..Memory for allocations..| | | | 49 // |--------------------------| | | | 50 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|<---| | | 51 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 52 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 53 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 54 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 55 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|<-------| | 56 // |..........................| | 57 // |..........................| | 58 // |..........................| | 59 // |..........................| | 60 // |..........................|<-----------| 61 // |--------------------------| 62 // 63 // Each TLAB is connected with certain thread: 64 // (NOTE: In current implementation, we can reach max one TLAB from a thread metadata) 65 // NOTE(aemelenko): If we don't use links on next, prev TLABS, 66 // it is better to remove these fields. 67 // |------------------------| |---------------| 68 // | Thread Metainformation | ----------> | Current TLAB |---- 69 // |------------------------| |---------------| | 70 // | 71 // | 72 // |---------------| | 73 // | Previous TLAB |<---| 74 // |---------------| 75 76 // How to use TLAB from the compiler if we want to allocate an object for class 'cls' with size 'allocation_size': 77 // NOTE: If we have PANDA_TRACK_TLAB_ALLOCATIONS option on, JIT should always call Runtime at AllocateObject calls. 78 // Pseudocode: 79 // IF allocation_size > TLAB::GetMaxSize() || IsFinalizable(cls) 80 // call HeapManager::AllocateObject(obj_class, allocation_size) 81 // ELSE 82 // // We should use TLS for this purpose. 83 // // Read current TLAB pointer from TLS: 84 // load TLS.TLAB -> cur_TLAB 85 // // Read uintptr_t value from TLAB structure: 86 // load (AddressOf(cur_TLAB) + TLAB::TLABFreePointerOffset) -> free_pointer 87 // // Read uintptr_t value from TLAB structure: 88 // load (AddressOf(cur_TLAB) + TLAB::TLABEndAddrOffset) -> end_pointer 89 // // Align the size of an object to DEFAULT_ALIGNMENT_IN_BYTES 90 // // One can use GetAlignedObjectSize() method for that. 91 // align (allocation_size, DEFAULT_ALIGNMENT_IN_BYTES) -> allocation_size 92 // IF free_pointer + allocation_size > end_pointer 93 // // Goto slow path 94 // call HeapManager::AllocateObject(obj_class, allocation_size) 95 // // Calculate new_free_pointer: 96 // new_free_pointer = AddressOf(free_pointer) + allocation_size 97 // // Store new_free_pointer to (cur_TLAB + TLAB::TLABFreePointerOffset): 98 // store (AddressOf(cur_TLAB) + TLAB::TLABFreePointerOffset) <- new_free_pointer 99 // return free_pointer 100 // 101 // After that the Compiler should initialize class word inside new object and 102 // set correct GC bits in the mark word: 103 // ObjectHeader obj_header 104 // obj_header.SetClass(cls) 105 // GetGC()->InitGCBitsForAllocationInTLAB(&obj_header) 106 // free_pointer <- obj_header 107 // 108 // Runtime should provide these parameters: 109 // HeapManager::GetTLABMaxAllocSize() - max size that can be allocated via TLAB. (depends on the allocator used by GC) 110 // HeapManager::UseTLABForAllocations() - do we need to use TLABs for allocations. (it is a runtime option) 111 // GC::InitGCBitsForAllocationInTLAB() - method for initialize GC bits inside the object header 112 // during allocations through TLAB 113 // TLAB::TLABFreePointerOffset() - an offset of a free pointer field inside TLAB. 114 // TLAB::TLABEndAddrOffset() - an offset of a end buffer pointer field inside TLAB. 115 // 116 117 class TLAB { 118 public: 119 /** 120 * @brief Construct TLAB with the buffer at @param address with @param size 121 * @param address - a pointer into the memory where TLAB memory will be created 122 * @param size - a size of the allocated memory for the TLAB 123 */ 124 explicit TLAB(void *address = nullptr, size_t size = 0); 125 ~TLAB(); 126 127 void Destroy(); 128 129 /** 130 * @brief Fill a TLAB with the buffer at @param address with @param size 131 * @param address - a pointer into the memory where TLAB memory will be created 132 * @param size - a size of the allocated memory for the TLAB 133 */ 134 void Fill(void *address, size_t size); 135 136 /// @brief Set TLAB to be empty Reset()137 void Reset() 138 { 139 Fill(nullptr, 0U); 140 } 141 IsUninitialized()142 bool IsUninitialized() 143 { 144 return (memoryStartAddr_ == nullptr) || (curFreePosition_ == nullptr) || (memoryEndAddr_ == nullptr); 145 } 146 147 NO_MOVE_SEMANTIC(TLAB); 148 NO_COPY_SEMANTIC(TLAB); 149 150 /** 151 * @brief returns maximum size which can be allocated by TLAB allocator 152 * @return 153 */ GetMaxSize()154 static constexpr size_t GetMaxSize() 155 { 156 return PANDA_TLAB_MAX_ALLOC_SIZE; 157 } 158 159 /** 160 * @brief returns default pool size which must be added to a TLAB 161 * @return 162 */ GetDefaultPoolSize()163 static constexpr size_t GetDefaultPoolSize() 164 { 165 return PANDA_TLAB_SIZE; 166 } 167 168 /** 169 * @brief Allocates memory with size @param size and aligned with DEFAULT_ALIGNMENT alignment 170 * @param size - size of the allocated memory 171 * @return pointer to the allocated memory on success, or nullptr on fail 172 */ 173 void *Alloc(size_t size); 174 175 /** 176 * @brief Iterates over all objects in this TLAB 177 * @param object_visitor 178 */ 179 void IterateOverObjects(const std::function<void(ObjectHeader *objectHeader)> &objectVisitor); 180 181 /** 182 * @brief Iterates over objects in the range inclusively. 183 * @param mem_visitor - function pointer or functor 184 * @param mem_range - memory range 185 */ 186 void IterateOverObjectsInRange(const std::function<void(ObjectHeader *objectHeader)> &memVisitor, 187 const MemRange &memRange); 188 189 /** 190 * Collects dead objects and move alive with provided visitor 191 * @param death_checker - functor for check if object alive 192 * @param object_move_visitor - object visitor 193 */ 194 template <typename ObjectMoveVisitorT> CollectAndMove(const GCObjectVisitor & deathChecker,const ObjectMoveVisitorT & objectMoveVisitor)195 void CollectAndMove(const GCObjectVisitor &deathChecker, const ObjectMoveVisitorT &objectMoveVisitor) 196 { 197 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove started"; 198 IterateOverObjects([&](ObjectHeader *objectHeader) { 199 // We are interested only in moving alive objects, after that we cleanup this buffer 200 if (deathChecker(objectHeader) == ObjectStatus::ALIVE_OBJECT) { 201 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove found alive object with addr " << objectHeader; 202 objectMoveVisitor(objectHeader); 203 } 204 }); 205 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove finished"; 206 } 207 208 bool ContainObject(const ObjectHeader *obj); 209 210 bool IsLive(const ObjectHeader *obj); 211 GetNextTLAB()212 TLAB *GetNextTLAB() 213 { 214 return nextTlab_; 215 } 216 GetPrevTLAB()217 TLAB *GetPrevTLAB() 218 { 219 return prevTlab_; 220 } 221 SetNextTLAB(TLAB * tlabPointer)222 void SetNextTLAB(TLAB *tlabPointer) 223 { 224 nextTlab_ = tlabPointer; 225 } 226 SetPrevTLAB(TLAB * tlabPointer)227 void SetPrevTLAB(TLAB *tlabPointer) 228 { 229 prevTlab_ = tlabPointer; 230 } 231 GetStartAddr()232 void *GetStartAddr() const 233 { 234 return memoryStartAddr_; 235 } 236 GetEndAddr()237 void *GetEndAddr() const 238 { 239 return memoryEndAddr_; 240 } 241 GetCurPos()242 void *GetCurPos() const 243 { 244 return curFreePosition_; 245 } 246 GetOccupiedSize()247 size_t GetOccupiedSize() const 248 { 249 ASSERT(ToUintPtr(curFreePosition_) >= ToUintPtr(memoryStartAddr_)); 250 return ToUintPtr(curFreePosition_) - ToUintPtr(memoryStartAddr_); 251 } 252 GetMemRangeForOccupiedMemory()253 MemRange GetMemRangeForOccupiedMemory() const 254 { 255 return MemRange(ToUintPtr(memoryStartAddr_), ToUintPtr(curFreePosition_) - 1); 256 } 257 TLABStartAddrOffset()258 static constexpr size_t TLABStartAddrOffset() 259 { 260 return MEMBER_OFFSET(TLAB, memoryStartAddr_); 261 } 262 TLABFreePointerOffset()263 static constexpr size_t TLABFreePointerOffset() 264 { 265 return MEMBER_OFFSET(TLAB, curFreePosition_); 266 } 267 TLABEndAddrOffset()268 static constexpr size_t TLABEndAddrOffset() 269 { 270 return MEMBER_OFFSET(TLAB, memoryEndAddr_); 271 } 272 GetAllocatorType()273 static constexpr AllocatorType GetAllocatorType() 274 { 275 return AllocatorType::TLAB_ALLOCATOR; 276 } 277 GetSize()278 size_t GetSize() 279 { 280 ASSERT(ToUintPtr(memoryEndAddr_) >= ToUintPtr(memoryStartAddr_)); 281 return ToUintPtr(memoryEndAddr_) - ToUintPtr(memoryStartAddr_); 282 } 283 284 private: GetFreeSize()285 size_t GetFreeSize() 286 { 287 ASSERT(ToUintPtr(curFreePosition_) >= ToUintPtr(memoryStartAddr_)); 288 ASSERT(ToUintPtr(curFreePosition_) <= ToUintPtr(memoryEndAddr_)); 289 return ToUintPtr(memoryEndAddr_) - ToUintPtr(curFreePosition_); 290 } 291 292 TLAB *nextTlab_; 293 TLAB *prevTlab_; 294 // NOTE(aemelenko): Maybe use OBJECT_POINTER_SIZE here for heap allocation. 295 void *memoryStartAddr_ {nullptr}; 296 void *memoryEndAddr_ {nullptr}; 297 void *curFreePosition_ {nullptr}; 298 }; 299 300 #undef LOG_TLAB_ALLOCATOR 301 302 } // namespace panda::mem 303 304 #endif // PANDA_RUNTIME_MEM_TLAB_H 305