1 /** 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 #ifndef PANDA_RUNTIME_MEM_TLAB_H 16 #define PANDA_RUNTIME_MEM_TLAB_H 17 18 #include "libpandabase/utils/logger.h" 19 #include "libpandabase/mem/mem.h" 20 #include "libpandabase/mem/pool_map.h" 21 #include "libpandabase/mem/mem_range.h" 22 23 namespace panda::mem { 24 25 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) 26 #define LOG_TLAB_ALLOCATOR(level) LOG(level, ALLOC) << "TLAB: " 27 28 static constexpr size_t PANDA_TLAB_SIZE = 4_KB; 29 static constexpr size_t PANDA_TLAB_MAX_ALLOC_SIZE = PANDA_TLAB_SIZE; 30 31 #ifdef NDEBUG 32 static constexpr bool PANDA_TRACK_TLAB_ALLOCATIONS = false; 33 #else 34 static constexpr bool PANDA_TRACK_TLAB_ALLOCATIONS = true; 35 #endif 36 // Current TLAB structure looks like that: 37 // 38 // |--------------------------| 39 // |........TLAB class........| 40 // |--------------------------| 41 // |.........end addr.........|------------| 42 // |.......free pointer.......|--------| | 43 // |........start addr........|----| | | 44 // |--------------------------| | | | 45 // | | | 46 // | | | 47 // |--------------------------| | | | 48 // |..Memory for allocations..| | | | 49 // |--------------------------| | | | 50 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|<---| | | 51 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 52 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 53 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 54 // |xxxxxxxxxxxxxxxxxxxxxxxxxx| | | 55 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|<-------| | 56 // |..........................| | 57 // |..........................| | 58 // |..........................| | 59 // |..........................| | 60 // |..........................|<-----------| 61 // |--------------------------| 62 // 63 // Each TLAB is connected with certain thread: 64 // (NOTE: In current implementation, we can reach max one TLAB from a thread metadata) 65 // TODO(aemelenko): If we don't use links on next, prev TLABS, 66 // it is better to remove these fields. 67 // |------------------------| |---------------| 68 // | Thread Metainformation | ----------> | Current TLAB |---- 69 // |------------------------| |---------------| | 70 // | 71 // | 72 // |---------------| | 73 // | Previous TLAB |<---| 74 // |---------------| 75 76 // How to use TLAB from the compiler if we want to allocate an object for class 'cls' with size 'allocation_size': 77 // NOTE: If we have PANDA_TRACK_TLAB_ALLOCATIONS option on, JIT should always call Runtime at AllocateObject calls. 78 // Pseudocode: 79 // IF allocation_size > TLAB::GetMaxSize() || IsFinalizable(cls) 80 // call HeapManager::AllocateObject(obj_class, allocation_size) 81 // ELSE 82 // // We should use TLS for this purpose. 83 // // Read current TLAB pointer from TLS: 84 // load TLS.TLAB -> cur_TLAB 85 // // Read uintptr_t value from TLAB structure: 86 // load (AddressOf(cur_TLAB) + TLAB::TLABFreePointerOffset) -> free_pointer 87 // // Read uintptr_t value from TLAB structure: 88 // load (AddressOf(cur_TLAB) + TLAB::TLABEndAddrOffset) -> end_pointer 89 // // Align the size of an object to DEFAULT_ALIGNMENT_IN_BYTES 90 // // One can use GetAlignedObjectSize() method for that. 91 // align (allocation_size, DEFAULT_ALIGNMENT_IN_BYTES) -> allocation_size 92 // IF free_pointer + allocation_size > end_pointer 93 // // Goto slow path 94 // call HeapManager::AllocateObject(obj_class, allocation_size) 95 // // Calculate new_free_pointer: 96 // new_free_pointer = AddressOf(free_pointer) + allocation_size 97 // // Store new_free_pointer to (cur_TLAB + TLAB::TLABFreePointerOffset): 98 // store (AddressOf(cur_TLAB) + TLAB::TLABFreePointerOffset) <- new_free_pointer 99 // return free_pointer 100 // 101 // After that the Compiler should initialize class word inside new object and 102 // set correct GC bits in the mark word: 103 // ObjectHeader obj_header 104 // obj_header.SetClass(cls) 105 // GetGC()->InitGCBitsForAllocationInTLAB(&obj_header) 106 // free_pointer <- obj_header 107 // 108 // Runtime should provide these parameters: 109 // HeapManager::GetTLABMaxAllocSize() - max size that can be allocated via TLAB. (depends on the allocator used by GC) 110 // HeapManager::UseTLABForAllocations() - do we need to use TLABs for allocations. (it is a runtime option) 111 // GC::InitGCBitsForAllocationInTLAB() - method for initialize GC bits inside the object header 112 // during allocations through TLAB 113 // TLAB::TLABFreePointerOffset() - an offset of a free pointer field inside TLAB. 114 // TLAB::TLABEndAddrOffset() - an offset of a end buffer pointer field inside TLAB. 115 // 116 117 class TLAB { 118 public: 119 /** 120 * \brief Construct TLAB with the buffer at \param address with \param size 121 * @param address - a pointer into the memory where TLAB memory will be created 122 * @param size - a size of the allocated memory for the TLAB 123 */ 124 explicit TLAB(void *address = nullptr, size_t size = 0); 125 ~TLAB(); 126 127 void Destroy(); 128 129 /** 130 * \brief Fill a TLAB with the buffer at \param address with \param size 131 * @param address - a pointer into the memory where TLAB memory will be created 132 * @param size - a size of the allocated memory for the TLAB 133 */ 134 void Fill(void *address, size_t size); 135 136 /** 137 * \brief Set TLAB to be empty 138 */ Reset()139 void Reset() 140 { 141 Fill(nullptr, 0U); 142 } 143 IsUninitialized()144 bool IsUninitialized() 145 { 146 return (memory_start_addr_ == nullptr) || (cur_free_position_ == nullptr) || (memory_end_addr_ == nullptr); 147 } 148 149 NO_MOVE_SEMANTIC(TLAB); 150 NO_COPY_SEMANTIC(TLAB); 151 152 /** 153 * \brief returns maximum size which can be allocated by TLAB allocator 154 * @return 155 */ GetMaxSize()156 static constexpr size_t GetMaxSize() 157 { 158 return PANDA_TLAB_MAX_ALLOC_SIZE; 159 } 160 161 /** 162 * \brief returns default pool size which must be added to a TLAB 163 * @return 164 */ GetDefaultPoolSize()165 static constexpr size_t GetDefaultPoolSize() 166 { 167 return PANDA_TLAB_SIZE; 168 } 169 170 /** 171 * \brief Allocates memory with size \param size and aligned with DEFAULT_ALIGNMENT alignment 172 * @param size - size of the allocated memory 173 * @return pointer to the allocated memory on success, or nullptr on fail 174 */ 175 void *Alloc(size_t size); 176 177 /** 178 * \brief Iterates over all objects in this TLAB 179 * @param object_visitor 180 */ 181 void IterateOverObjects(const std::function<void(ObjectHeader *object_header)> &object_visitor); 182 183 /** 184 * \brief Iterates over objects in the range inclusively. 185 * @param mem_visitor - function pointer or functor 186 * @param mem_range - memory range 187 */ 188 void IterateOverObjectsInRange(const std::function<void(ObjectHeader *object_header)> &mem_visitor, 189 const MemRange &mem_range); 190 191 /** 192 * Collects dead objects and move alive with provided visitor 193 * @param death_checker - functor for check if object alive 194 * @param object_move_visitor - object visitor 195 */ 196 template <typename ObjectMoveVisitorT> CollectAndMove(const GCObjectVisitor & death_checker,const ObjectMoveVisitorT & object_move_visitor)197 void CollectAndMove(const GCObjectVisitor &death_checker, const ObjectMoveVisitorT &object_move_visitor) 198 { 199 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove started"; 200 IterateOverObjects([&](ObjectHeader *object_header) { 201 // We are interested only in moving alive objects, after that we cleanup this buffer 202 if (death_checker(object_header) == ObjectStatus::ALIVE_OBJECT) { 203 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove found alive object with addr " << object_header; 204 object_move_visitor(object_header); 205 } 206 }); 207 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove finished"; 208 } 209 210 bool ContainObject(const ObjectHeader *obj); 211 212 bool IsLive(const ObjectHeader *obj); 213 GetNextTLAB()214 TLAB *GetNextTLAB() 215 { 216 return next_tlab_; 217 } 218 GetPrevTLAB()219 TLAB *GetPrevTLAB() 220 { 221 return prev_tlab_; 222 } 223 SetNextTLAB(TLAB * tlab_pointer)224 void SetNextTLAB(TLAB *tlab_pointer) 225 { 226 next_tlab_ = tlab_pointer; 227 } 228 SetPrevTLAB(TLAB * tlab_pointer)229 void SetPrevTLAB(TLAB *tlab_pointer) 230 { 231 prev_tlab_ = tlab_pointer; 232 } 233 GetStartAddr()234 void *GetStartAddr() const 235 { 236 return memory_start_addr_; 237 } 238 GetEndAddr()239 void *GetEndAddr() const 240 { 241 return memory_end_addr_; 242 } 243 GetCurPos()244 void *GetCurPos() const 245 { 246 return cur_free_position_; 247 } 248 GetOccupiedSize()249 size_t GetOccupiedSize() const 250 { 251 ASSERT(ToUintPtr(cur_free_position_) >= ToUintPtr(memory_start_addr_)); 252 return ToUintPtr(cur_free_position_) - ToUintPtr(memory_start_addr_); 253 } 254 GetMemRangeForOccupiedMemory()255 MemRange GetMemRangeForOccupiedMemory() const 256 { 257 return MemRange(ToUintPtr(memory_start_addr_), ToUintPtr(cur_free_position_) - 1); 258 } 259 TLABStartAddrOffset()260 static constexpr size_t TLABStartAddrOffset() 261 { 262 return MEMBER_OFFSET(TLAB, memory_start_addr_); 263 } 264 TLABFreePointerOffset()265 static constexpr size_t TLABFreePointerOffset() 266 { 267 return MEMBER_OFFSET(TLAB, cur_free_position_); 268 } 269 TLABEndAddrOffset()270 static constexpr size_t TLABEndAddrOffset() 271 { 272 return MEMBER_OFFSET(TLAB, memory_end_addr_); 273 } 274 GetAllocatorType()275 static constexpr AllocatorType GetAllocatorType() 276 { 277 return AllocatorType::TLAB_ALLOCATOR; 278 } 279 GetSize()280 size_t GetSize() 281 { 282 ASSERT(ToUintPtr(memory_end_addr_) >= ToUintPtr(memory_start_addr_)); 283 return ToUintPtr(memory_end_addr_) - ToUintPtr(memory_start_addr_); 284 } 285 286 private: GetFreeSize()287 size_t GetFreeSize() 288 { 289 ASSERT(ToUintPtr(cur_free_position_) >= ToUintPtr(memory_start_addr_)); 290 ASSERT(ToUintPtr(cur_free_position_) <= ToUintPtr(memory_end_addr_)); 291 return ToUintPtr(memory_end_addr_) - ToUintPtr(cur_free_position_); 292 } 293 294 TLAB *next_tlab_; 295 TLAB *prev_tlab_; 296 // TODO(aemelenko): Maybe use OBJECT_POINTER_SIZE here for heap allocation. 297 void *memory_start_addr_ {nullptr}; 298 void *memory_end_addr_ {nullptr}; 299 void *cur_free_position_ {nullptr}; 300 }; 301 302 #undef LOG_TLAB_ALLOCATOR 303 304 } // namespace panda::mem 305 306 #endif // PANDA_RUNTIME_MEM_TLAB_H 307