19 …gned_at_fallback(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t of…  in mi_heap_malloc_zero_aligned_at_fallback()  argument
22   mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment));  in mi_heap_malloc_zero_aligned_at_fallback()
24   const uintptr_t align_mask = alignment - 1;  // for any x, `(x & align_mask) == (x % alignment)`  in mi_heap_malloc_zero_aligned_at_fallback()
27   // use regular allocation if it is guaranteed to fit the alignment constraints  in mi_heap_malloc_zero_aligned_at_fallback()
28 …if (offset==0 && alignment<=padsize && padsize<=MI_MAX_ALIGN_GUARANTEE && (padsize&align_mask)==0)…  in mi_heap_malloc_zero_aligned_at_fallback()
30     mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0);  in mi_heap_malloc_zero_aligned_at_fallback()
36   if mi_unlikely(alignment > MI_ALIGNMENT_MAX) {  in mi_heap_malloc_zero_aligned_at_fallback()
37 …// use OS allocation for very large alignment and allocate inside a huge page (dedicated segment w…  in mi_heap_malloc_zero_aligned_at_fallback()
41       // todo: cannot support offset alignment for very large alignments yet  in mi_heap_malloc_zero_aligned_at_fallback()
43 …tion with a very large alignment cannot be used with an alignment offset (size %zu, alignment %zu,…  in mi_heap_malloc_zero_aligned_at_fallback()
48 …p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment); // the page block size should be la…  in mi_heap_malloc_zero_aligned_at_fallback()
54     oversize = size + alignment - 1;  in mi_heap_malloc_zero_aligned_at_fallback()
61   const uintptr_t adjust  = (poffset == 0 ? 0 : alignment - poffset);  in mi_heap_malloc_zero_aligned_at_fallback()
62   mi_assert_internal(adjust < alignment);  in mi_heap_malloc_zero_aligned_at_fallback()
73   mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0);  in mi_heap_malloc_zero_aligned_at_fallback()
78   if (alignment > MI_ALIGNMENT_MAX) {  in mi_heap_malloc_zero_aligned_at_fallback()
93 …_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t of…  in mi_heap_malloc_zero_aligned_at()  argument
96 …if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see <…  in mi_heap_malloc_zero_aligned_at()
98 …LOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu)\n",…  in mi_heap_malloc_zero_aligned_at()
105 …EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment);  in mi_heap_malloc_zero_aligned_at()
109 …const uintptr_t align_mask = alignment-1;       // for any x, `(x & align_mask) == (x % alignment)`  in mi_heap_malloc_zero_aligned_at()
112   // try first if there happens to be a small block available with just the right alignment  in mi_heap_malloc_zero_aligned_at()
113   if mi_likely(padsize <= MI_SMALL_SIZE_MAX && alignment <= padsize) {  in mi_heap_malloc_zero_aligned_at()
123       mi_assert_internal(((uintptr_t)p + offset) % alignment == 0);  in mi_heap_malloc_zero_aligned_at()
129   return mi_heap_malloc_zero_aligned_at_fallback(heap, size, alignment, offset, zero);  in mi_heap_malloc_zero_aligned_at()
137 …ct void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) …  in mi_heap_malloc_aligned_at()  argument
138   return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false);  in mi_heap_malloc_aligned_at()
141 …trict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcep…  in mi_heap_malloc_aligned()  argument
142   if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) return NULL;  in mi_heap_malloc_aligned()
145   if mi_likely(_mi_is_power_of_two(size) && size >= alignment && size <= MI_SMALL_SIZE_MAX)  in mi_heap_malloc_aligned()
148 …if mi_likely((alignment == sizeof(void*) || (alignment == MI_MAX_ALIGN_SIZE && size > (MI_MAX_ALIG…  in mi_heap_malloc_aligned()
152     // fast path for common alignment and size  in mi_heap_malloc_aligned()
156     return mi_heap_malloc_aligned_at(heap, size, alignment, 0);  in mi_heap_malloc_aligned()
169 …ct void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) …  in mi_heap_zalloc_aligned_at()  argument
170   return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true);  in mi_heap_zalloc_aligned_at()
173 …trict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcep…  in mi_heap_zalloc_aligned()  argument
174   return mi_heap_zalloc_aligned_at(heap, size, alignment, 0);  in mi_heap_zalloc_aligned()
177 …ap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) …  in mi_heap_calloc_aligned_at()  argument
180   return mi_heap_zalloc_aligned_at(heap, total, alignment, offset);  in mi_heap_calloc_aligned_at()
183 …_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcep…  in mi_heap_calloc_aligned()  argument
184   return mi_heap_calloc_aligned_at(heap,count,size,alignment,0);  in mi_heap_calloc_aligned()
187 mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t…  in mi_malloc_aligned_at()  argument
188   return mi_heap_malloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset);  in mi_malloc_aligned_at()
191 mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_n…  in mi_malloc_aligned()  argument
192   return mi_heap_malloc_aligned(mi_prim_get_default_heap(), size, alignment);  in mi_malloc_aligned()
195 mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t…  in mi_zalloc_aligned_at()  argument
196   return mi_heap_zalloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset);  in mi_zalloc_aligned_at()
199 mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_n…  in mi_zalloc_aligned()  argument
200   return mi_heap_zalloc_aligned(mi_prim_get_default_heap(), size, alignment);  in mi_zalloc_aligned()
203 …l_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) …  in mi_calloc_aligned_at()  argument
204   return mi_heap_calloc_aligned_at(mi_prim_get_default_heap(), count, size, alignment, offset);  in mi_calloc_aligned_at()
207 …decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcep…  in mi_calloc_aligned()  argument
208   return mi_heap_calloc_aligned(mi_prim_get_default_heap(), count, size, alignment);  in mi_calloc_aligned()
216 …ealloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, …  in mi_heap_realloc_zero_aligned_at()  argument
217   mi_assert(alignment > 0);  in mi_heap_realloc_zero_aligned_at()
218   if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero);  in mi_heap_realloc_zero_aligned_at()
219   if (p == NULL) return mi_heap_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero);  in mi_heap_realloc_zero_aligned_at()
222       && (((uintptr_t)p + offset) % alignment) == 0) {  in mi_heap_realloc_zero_aligned_at()
227     void* newp = mi_heap_malloc_aligned_at(heap,newsize,alignment,offset);  in mi_heap_realloc_zero_aligned_at()
241 …p_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero) mi_a…  in mi_heap_realloc_zero_aligned()  argument
242   mi_assert(alignment > 0);  in mi_heap_realloc_zero_aligned()
243   if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero);  in mi_heap_realloc_zero_aligned()
244   size_t offset = ((uintptr_t)p % alignment); // use offset of previous allocation (p can be NULL)  in mi_heap_realloc_zero_aligned()
245   return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero);  in mi_heap_realloc_zero_aligned()
248 …eap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) …  in mi_heap_realloc_aligned_at()  argument
249   return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false);  in mi_heap_realloc_aligned_at()
252 …i_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcep…  in mi_heap_realloc_aligned()  argument
253   return mi_heap_realloc_zero_aligned(heap,p,newsize,alignment,false);  in mi_heap_realloc_aligned()
256 …ap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) …  in mi_heap_rezalloc_aligned_at()  argument
257   return mi_heap_realloc_zero_aligned_at(heap, p, newsize, alignment, offset, true);  in mi_heap_rezalloc_aligned_at()
260 …_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcep…  in mi_heap_rezalloc_aligned()  argument
261   return mi_heap_realloc_zero_aligned(heap, p, newsize, alignment, true);  in mi_heap_rezalloc_aligned()
264 …igned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) …  in mi_heap_recalloc_aligned_at()  argument
267   return mi_heap_rezalloc_aligned_at(heap, p, total, alignment, offset);  in mi_heap_recalloc_aligned_at()
270 …_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcep…  in mi_heap_recalloc_aligned()  argument
273   return mi_heap_rezalloc_aligned(heap, p, total, alignment);  in mi_heap_recalloc_aligned()
276 mi_decl_nodiscard void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t off…  in mi_realloc_aligned_at()  argument
277   return mi_heap_realloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset);  in mi_realloc_aligned_at()
280 mi_decl_nodiscard void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexc…  in mi_realloc_aligned()  argument
281   return mi_heap_realloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment);  in mi_realloc_aligned()
284 mi_decl_nodiscard void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t of…  in mi_rezalloc_aligned_at()  argument
285   return mi_heap_rezalloc_aligned_at(mi_prim_get_default_heap(), p, newsize, alignment, offset);  in mi_rezalloc_aligned_at()
288 mi_decl_nodiscard void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noex…  in mi_rezalloc_aligned()  argument
289   return mi_heap_rezalloc_aligned(mi_prim_get_default_heap(), p, newsize, alignment);  in mi_rezalloc_aligned()
292 …d* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) …  in mi_recalloc_aligned_at()  argument
293 …return mi_heap_recalloc_aligned_at(mi_prim_get_default_heap(), p, newcount, size, alignment, offse…  in mi_recalloc_aligned_at()
296 mi_decl_nodiscard void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment…  in mi_recalloc_aligned()  argument
297   return mi_heap_recalloc_aligned(mi_prim_get_default_heap(), p, newcount, size, alignment);  in mi_recalloc_aligned()