Lines Matching refs:entry
183 void *entry; in xas_start() local
190 entry = xa_head(xas->xa); in xas_start()
191 if (!xa_is_node(entry)) { in xas_start()
195 if ((xas->xa_index >> xa_to_node(entry)->shift) > XA_CHUNK_MASK) in xas_start()
200 return entry; in xas_start()
207 void *entry = xa_entry(xas->xa, node, offset); in xas_descend() local
210 while (xa_is_sibling(entry)) { in xas_descend()
211 offset = xa_to_sibling(entry); in xas_descend()
212 entry = xa_entry(xas->xa, node, offset); in xas_descend()
213 if (node->shift && xa_is_node(entry)) in xas_descend()
214 entry = XA_RETRY_ENTRY; in xas_descend()
218 return entry; in xas_descend()
238 void *entry = xas_start(xas); in xas_load() local
240 while (xa_is_node(entry)) { in xas_load()
241 struct xa_node *node = xa_to_node(entry); in xas_load()
245 entry = xas_descend(xas, node); in xas_load()
249 return entry; in xas_load()
431 static unsigned long max_index(void *entry) in max_index() argument
433 if (!xa_is_node(entry)) in max_index()
435 return (XA_CHUNK_SIZE << xa_to_node(entry)->shift) - 1; in max_index()
444 void *entry; in xas_shrink() local
449 entry = xa_entry_locked(xa, node, 0); in xas_shrink()
450 if (!entry) in xas_shrink()
452 if (!xa_is_node(entry) && node->shift) in xas_shrink()
454 if (xa_is_zero(entry) && xa_zero_busy(xa)) in xas_shrink()
455 entry = NULL; in xas_shrink()
458 RCU_INIT_POINTER(xa->xa_head, entry); in xas_shrink()
464 if (!xa_is_node(entry)) in xas_shrink()
468 if (!xa_is_node(entry)) in xas_shrink()
470 node = xa_to_node(entry); in xas_shrink()
530 void *entry = xa_entry_locked(xas->xa, node, offset); in xas_free_nodes() local
532 if (node->shift && xa_is_node(entry)) { in xas_free_nodes()
533 node = xa_to_node(entry); in xas_free_nodes()
537 if (entry) in xas_free_nodes()
643 void *entry; in xas_create() local
650 entry = xa_head_locked(xa); in xas_create()
652 if (!entry && xa_zero_busy(xa)) in xas_create()
653 entry = XA_ZERO_ENTRY; in xas_create()
654 shift = xas_expand(xas, entry); in xas_create()
659 entry = xa_head_locked(xa); in xas_create()
667 entry = xa_entry_locked(xa, node, offset); in xas_create()
671 entry = xa_head_locked(xa); in xas_create()
677 if (!entry) { in xas_create()
684 } else if (xa_is_node(entry)) { in xas_create()
685 node = xa_to_node(entry); in xas_create()
689 entry = xas_descend(xas, node); in xas_create()
693 return entry; in xas_create()
776 void *xas_store(struct xa_state *xas, void *entry) in xas_store() argument
784 bool value = xa_is_value(entry); in xas_store()
786 if (entry) { in xas_store()
787 bool allow_root = !xa_is_node(entry) && !xa_is_zero(entry); in xas_store()
798 if ((first == entry) && !xas->xa_sibs) in xas_store()
809 if (!entry) in xas_store()
820 rcu_assign_pointer(*slot, entry); in xas_store()
825 count += !next - !entry; in xas_store()
827 if (entry) { in xas_store()
830 if (!xa_is_sibling(entry)) in xas_store()
831 entry = xa_mk_sibling(xas->xa_offset); in xas_store()
838 if (!entry && (offset > max)) in xas_store()
1018 void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order, in xas_split_alloc() argument
1041 RCU_INIT_POINTER(node->slots[i], entry); in xas_split_alloc()
1069 void xas_split(struct xa_state *xas, void *entry, unsigned int order) in xas_split() argument
1092 child->nr_values = xa_is_value(entry) ? in xas_split()
1106 rcu_assign_pointer(node->slots[canon], entry); in xas_split()
1110 values += (xa_is_value(entry) - xa_is_value(curr)) * in xas_split()
1168 void *entry; in __xas_prev() local
1188 entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset); in __xas_prev()
1189 if (!xa_is_node(entry)) in __xas_prev()
1190 return entry; in __xas_prev()
1192 xas->xa_node = xa_to_node(entry); in __xas_prev()
1207 void *entry; in __xas_next() local
1227 entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset); in __xas_next()
1228 if (!xa_is_node(entry)) in __xas_next()
1229 return entry; in __xas_next()
1231 xas->xa_node = xa_to_node(entry); in __xas_next()
1255 void *entry; in xas_find() local
1266 entry = xas_load(xas); in xas_find()
1267 if (entry || xas_not_node(xas->xa_node)) in xas_find()
1268 return entry; in xas_find()
1283 entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset); in xas_find()
1284 if (xa_is_node(entry)) { in xas_find()
1285 xas->xa_node = xa_to_node(entry); in xas_find()
1289 if (entry && !xa_is_sibling(entry)) in xas_find()
1290 return entry; in xas_find()
1326 void *entry; in xas_find_marked() local
1338 entry = xa_head(xas->xa); in xas_find_marked()
1340 if (xas->xa_index > max_index(entry)) in xas_find_marked()
1342 if (!xa_is_node(entry)) { in xas_find_marked()
1344 return entry; in xas_find_marked()
1348 xas->xa_node = xa_to_node(entry); in xas_find_marked()
1363 entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset); in xas_find_marked()
1364 if (xa_is_sibling(entry)) { in xas_find_marked()
1365 xas->xa_offset = xa_to_sibling(entry); in xas_find_marked()
1382 entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset); in xas_find_marked()
1383 if (!entry && !(xa_track_free(xas->xa) && mark == XA_FREE_MARK)) in xas_find_marked()
1385 if (!xa_is_node(entry)) in xas_find_marked()
1386 return entry; in xas_find_marked()
1387 xas->xa_node = xa_to_node(entry); in xas_find_marked()
1473 void *entry; in xa_load() local
1477 entry = xas_load(&xas); in xa_load()
1478 if (xa_is_zero(entry)) in xa_load()
1479 entry = NULL; in xa_load()
1480 } while (xas_retry(&xas, entry)); in xa_load()
1483 return entry; in xa_load()
1529 void *entry; in xa_erase() local
1532 entry = __xa_erase(xa, index); in xa_erase()
1535 return entry; in xa_erase()
1554 void *__xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) in __xa_store() argument
1559 if (WARN_ON_ONCE(xa_is_advanced(entry))) in __xa_store()
1561 if (xa_track_free(xa) && !entry) in __xa_store()
1562 entry = XA_ZERO_ENTRY; in __xa_store()
1565 curr = xas_store(&xas, entry); in __xa_store()
1591 void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) in xa_store() argument
1596 curr = __xa_store(xa, index, entry, gfp); in xa_store()
1620 void *old, void *entry, gfp_t gfp) in __xa_cmpxchg() argument
1625 if (WARN_ON_ONCE(xa_is_advanced(entry))) in __xa_cmpxchg()
1631 xas_store(&xas, entry); in __xa_cmpxchg()
1632 if (xa_track_free(xa) && entry && !curr) in __xa_cmpxchg()
1657 int __xa_insert(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) in __xa_insert() argument
1662 if (WARN_ON_ONCE(xa_is_advanced(entry))) in __xa_insert()
1664 if (!entry) in __xa_insert()
1665 entry = XA_ZERO_ENTRY; in __xa_insert()
1670 xas_store(&xas, entry); in __xa_insert()
1733 unsigned long last, void *entry, gfp_t gfp) in xa_store_range() argument
1737 if (WARN_ON_ONCE(xa_is_internal(entry))) in xa_store_range()
1744 if (entry) { in xa_store_range()
1755 xas_store(&xas, entry); in xa_store_range()
1809 void *entry; in xa_get_order() local
1812 entry = xas_load(&xas); in xa_get_order()
1813 if (entry) in xa_get_order()
1842 int __xa_alloc(struct xarray *xa, u32 *id, void *entry, in __xa_alloc() argument
1847 if (WARN_ON_ONCE(xa_is_advanced(entry))) in __xa_alloc()
1852 if (!entry) in __xa_alloc()
1853 entry = XA_ZERO_ENTRY; in __xa_alloc()
1862 xas_store(&xas, entry); in __xa_alloc()
1894 int __xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry, in __xa_alloc_cyclic() argument
1901 ret = __xa_alloc(xa, id, entry, limit, gfp); in __xa_alloc_cyclic()
1909 ret = __xa_alloc(xa, id, entry, limit, gfp); in __xa_alloc_cyclic()
1936 void *entry = xas_load(&xas); in __xa_set_mark() local
1938 if (entry) in __xa_set_mark()
1954 void *entry = xas_load(&xas); in __xa_clear_mark() local
1956 if (entry) in __xa_clear_mark()
1976 void *entry; in xa_get_mark() local
1979 entry = xas_start(&xas); in xa_get_mark()
1981 if (!xa_is_node(entry)) in xa_get_mark()
1983 entry = xas_descend(&xas, xa_to_node(entry)); in xa_get_mark()
2050 void *entry; in xa_find() local
2055 entry = xas_find_marked(&xas, max, filter); in xa_find()
2057 entry = xas_find(&xas, max); in xa_find()
2058 } while (xas_retry(&xas, entry)); in xa_find()
2061 if (entry) in xa_find()
2063 return entry; in xa_find()
2100 void *entry; in xa_find_after() local
2108 entry = xas_find_marked(&xas, max, filter); in xa_find_after()
2110 entry = xas_find(&xas, max); in xa_find_after()
2116 if (!xas_retry(&xas, entry)) in xa_find_after()
2121 if (entry) in xa_find_after()
2123 return entry; in xa_find_after()
2130 void *entry; in xas_extract_present() local
2134 xas_for_each(xas, entry, max) { in xas_extract_present()
2135 if (xas_retry(xas, entry)) in xas_extract_present()
2137 dst[i++] = entry; in xas_extract_present()
2149 void *entry; in xas_extract_marked() local
2153 xas_for_each_marked(xas, entry, max, mark) { in xas_extract_marked()
2154 if (xas_retry(xas, entry)) in xas_extract_marked()
2156 dst[i++] = entry; in xas_extract_marked()
2244 void *entry; in xa_destroy() local
2248 entry = xa_head_locked(xa); in xa_destroy()
2254 if (xa_is_node(entry)) in xa_destroy()
2255 xas_free_nodes(&xas, xa_to_node(entry)); in xa_destroy()
2293 void xa_dump_entry(const void *entry, unsigned long index, unsigned long shift) in xa_dump_entry() argument
2295 if (!entry) in xa_dump_entry()
2300 if (xa_is_node(entry)) { in xa_dump_entry()
2302 pr_cont("%px\n", entry); in xa_dump_entry()
2305 struct xa_node *node = xa_to_node(entry); in xa_dump_entry()
2311 } else if (xa_is_value(entry)) in xa_dump_entry()
2312 pr_cont("value %ld (0x%lx) [%px]\n", xa_to_value(entry), in xa_dump_entry()
2313 xa_to_value(entry), entry); in xa_dump_entry()
2314 else if (!xa_is_internal(entry)) in xa_dump_entry()
2315 pr_cont("%px\n", entry); in xa_dump_entry()
2316 else if (xa_is_retry(entry)) in xa_dump_entry()
2317 pr_cont("retry (%ld)\n", xa_to_internal(entry)); in xa_dump_entry()
2318 else if (xa_is_sibling(entry)) in xa_dump_entry()
2319 pr_cont("sibling (slot %ld)\n", xa_to_sibling(entry)); in xa_dump_entry()
2320 else if (xa_is_zero(entry)) in xa_dump_entry()
2321 pr_cont("zero (%ld)\n", xa_to_internal(entry)); in xa_dump_entry()
2323 pr_cont("UNKNOWN ENTRY (%px)\n", entry); in xa_dump_entry()
2328 void *entry = xa->xa_head; in xa_dump() local
2331 pr_info("xarray: %px head %px flags %x marks %d %d %d\n", xa, entry, in xa_dump()
2334 if (xa_is_node(entry)) in xa_dump()
2335 shift = xa_to_node(entry)->shift + XA_CHUNK_SHIFT; in xa_dump()
2336 xa_dump_entry(entry, 0, shift); in xa_dump()