39 #define RTREE_LEAFKEY_INVALID ((uintptr_t)1)
129     rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
131 JEMALLOC_ALWAYS_INLINE uintptr_t
132 rtree_leafkey(uintptr_t key) {  in rtree_leafkey()
137 	uintptr_t mask = ~((ZU(1) << maskbits) - 1);  in rtree_leafkey()
142 rtree_cache_direct_map(uintptr_t key) {  in rtree_cache_direct_map()
150 JEMALLOC_ALWAYS_INLINE uintptr_t
151 rtree_subkey(uintptr_t key, unsigned level) {  in rtree_subkey()
156 	uintptr_t mask = (ZU(1) << maskbits) - 1;  in rtree_subkey()
172 JEMALLOC_ALWAYS_INLINE uintptr_t
175 	return (uintptr_t)atomic_load_p(&elm->le_bits, dependent  in rtree_leaf_elm_bits_read()
180 rtree_leaf_elm_bits_extent_get(uintptr_t bits) {  in rtree_leaf_elm_bits_extent_get()
186 	uintptr_t high_bit_mask = ((uintptr_t)1 << LG_VADDR) - 1;  in rtree_leaf_elm_bits_extent_get()
188 	uintptr_t low_bit_mask = ~(uintptr_t)1;  in rtree_leaf_elm_bits_extent_get()
189 	uintptr_t mask = high_bit_mask & low_bit_mask;  in rtree_leaf_elm_bits_extent_get()
193 	return (extent_t *)((uintptr_t)((intptr_t)(bits << RTREE_NHIB) >>  in rtree_leaf_elm_bits_extent_get()
194 	    RTREE_NHIB) & ~((uintptr_t)0x1));  in rtree_leaf_elm_bits_extent_get()
199 rtree_leaf_elm_bits_szind_get(uintptr_t bits) {  in rtree_leaf_elm_bits_szind_get()
204 rtree_leaf_elm_bits_slab_get(uintptr_t bits) {  in rtree_leaf_elm_bits_slab_get()
205 	return (bool)(bits & (uintptr_t)0x1);  in rtree_leaf_elm_bits_slab_get()
214 	uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);  in rtree_leaf_elm_extent_read()
227 	uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);  in rtree_leaf_elm_szind_read()
239 	uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);  in rtree_leaf_elm_slab_read()
251 	uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, true);  in rtree_leaf_elm_extent_write()
252 	uintptr_t bits = ((uintptr_t)rtree_leaf_elm_bits_szind_get(old_bits) <<  in rtree_leaf_elm_extent_write()
253 	    LG_VADDR) | ((uintptr_t)extent & (((uintptr_t)0x1 << LG_VADDR) - 1))  in rtree_leaf_elm_extent_write()
254 	    | ((uintptr_t)rtree_leaf_elm_bits_slab_get(old_bits));  in rtree_leaf_elm_extent_write()
267 	uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm,  in rtree_leaf_elm_szind_write()
269 	uintptr_t bits = ((uintptr_t)szind << LG_VADDR) |  in rtree_leaf_elm_szind_write()
270 	    ((uintptr_t)rtree_leaf_elm_bits_extent_get(old_bits) &  in rtree_leaf_elm_szind_write()
271 	    (((uintptr_t)0x1 << LG_VADDR) - 1)) |  in rtree_leaf_elm_szind_write()
272 	    ((uintptr_t)rtree_leaf_elm_bits_slab_get(old_bits));  in rtree_leaf_elm_szind_write()
283 	uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm,  in rtree_leaf_elm_slab_write()
285 	uintptr_t bits = ((uintptr_t)rtree_leaf_elm_bits_szind_get(old_bits) <<  in rtree_leaf_elm_slab_write()
286 	    LG_VADDR) | ((uintptr_t)rtree_leaf_elm_bits_extent_get(old_bits) &  in rtree_leaf_elm_slab_write()
287 	    (((uintptr_t)0x1 << LG_VADDR) - 1)) | ((uintptr_t)slab);  in rtree_leaf_elm_slab_write()
298 	uintptr_t bits = ((uintptr_t)szind << LG_VADDR) |  in rtree_leaf_elm_write()
299 	    ((uintptr_t)extent & (((uintptr_t)0x1 << LG_VADDR) - 1)) |  in rtree_leaf_elm_write()
300 	    ((uintptr_t)slab);  in rtree_leaf_elm_write()
328     uintptr_t key, bool dependent, bool init_missing) {  in rtree_leaf_elm_lookup()
333 	uintptr_t leafkey = rtree_leafkey(key);  in rtree_leaf_elm_lookup()
345 		uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1);  in rtree_leaf_elm_lookup()
379 		uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1);	\  in rtree_leaf_elm_lookup()
396 rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,  in rtree_write()
414 rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,  in rtree_read()
430     uintptr_t key, bool dependent) {  in rtree_extent_read()
444     uintptr_t key, bool dependent) {  in rtree_szind_read()
462     uintptr_t key, bool dependent, extent_t **r_extent, szind_t *r_szind) {  in rtree_extent_szind_read()
478     uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab) {  in rtree_szind_slab_read()
488 	uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);  in rtree_szind_slab_read()
500     uintptr_t key, szind_t szind, bool slab) {  in rtree_szind_slab_update()
509     uintptr_t key) {  in rtree_clear()