66 static inline void xa_mark_set(struct xarray *xa, xa_mark_t mark)  in xa_mark_set()  argument
68 	if (!(xa->xa_flags & XA_FLAGS_MARK(mark)))  in xa_mark_set()
69 		xa->xa_flags |= XA_FLAGS_MARK(mark);  in xa_mark_set()
72 static inline void xa_mark_clear(struct xarray *xa, xa_mark_t mark)  in xa_mark_clear()  argument
74 	if (xa->xa_flags & XA_FLAGS_MARK(mark))  in xa_mark_clear()
75 		xa->xa_flags &= ~(XA_FLAGS_MARK(mark));  in xa_mark_clear()
78 static inline unsigned long *node_marks(struct xa_node *node, xa_mark_t mark)  in node_marks()  argument
80 	return node->marks[(__force unsigned)mark];  in node_marks()
84 		unsigned int offset, xa_mark_t mark)  in node_get_mark()  argument
86 	return test_bit(offset, node_marks(node, mark));  in node_get_mark()
91 				xa_mark_t mark)  in node_set_mark()  argument
93 	return __test_and_set_bit(offset, node_marks(node, mark));  in node_set_mark()
98 				xa_mark_t mark)  in node_clear_mark()  argument
100 	return __test_and_clear_bit(offset, node_marks(node, mark));  in node_clear_mark()
103 static inline bool node_any_mark(struct xa_node *node, xa_mark_t mark)  in node_any_mark()  argument
105 	return !bitmap_empty(node_marks(node, mark), XA_CHUNK_SIZE);  in node_any_mark()
108 static inline void node_mark_all(struct xa_node *node, xa_mark_t mark)  in node_mark_all()  argument
110 	bitmap_fill(node_marks(node, mark), XA_CHUNK_SIZE);  in node_mark_all()
113 #define mark_inc(mark) do { \  argument
114 	mark = (__force xa_mark_t)((__force unsigned)(mark) + 1); \
126 	unsigned int mark = 0;  in xas_squash_marks()  local
133 		unsigned long *marks = xas->xa_node->marks[mark];  in xas_squash_marks()
138 	} while (mark++ != (__force unsigned)XA_MARK_MAX);  in xas_squash_marks()
575 		xa_mark_t mark = 0;  in xas_expand()  local
589 			if (xa_track_free(xa) && mark == XA_FREE_MARK) {  in xas_expand()
595 			} else if (xa_marked(xa, mark)) {  in xas_expand()
596 				node_set_mark(node, 0, mark);  in xas_expand()
598 			if (mark == XA_MARK_MAX)  in xas_expand()
600 			mark_inc(mark);  in xas_expand()
851 bool xas_get_mark(const struct xa_state *xas, xa_mark_t mark)  in xas_get_mark()  argument
856 		return xa_marked(xas->xa, mark);  in xas_get_mark()
857 	return node_get_mark(xas->xa_node, xas->xa_offset, mark);  in xas_get_mark()
870 void xas_set_mark(const struct xa_state *xas, xa_mark_t mark)  in xas_set_mark()  argument
879 		if (node_set_mark(node, offset, mark))  in xas_set_mark()
885 	if (!xa_marked(xas->xa, mark))  in xas_set_mark()
886 		xa_mark_set(xas->xa, mark);  in xas_set_mark()
899 void xas_clear_mark(const struct xa_state *xas, xa_mark_t mark)  in xas_clear_mark()  argument
908 		if (!node_clear_mark(node, offset, mark))  in xas_clear_mark()
910 		if (node_any_mark(node, mark))  in xas_clear_mark()
917 	if (xa_marked(xas->xa, mark))  in xas_clear_mark()
918 		xa_mark_clear(xas->xa, mark);  in xas_clear_mark()
935 	xa_mark_t mark = 0;  in xas_init_marks()  local
938 		if (xa_track_free(xas->xa) && mark == XA_FREE_MARK)  in xas_init_marks()
939 			xas_set_mark(xas, mark);  in xas_init_marks()
941 			xas_clear_mark(xas, mark);  in xas_init_marks()
942 		if (mark == XA_MARK_MAX)  in xas_init_marks()
944 		mark_inc(mark);  in xas_init_marks()
1148 void *xas_find_marked(struct xa_state *xas, unsigned long max, xa_mark_t mark)  in xas_find_marked()  argument
1169 			if (xa_marked(xas->xa, mark))  in xas_find_marked()
1196 		offset = xas_find_chunk(xas, advance, mark);  in xas_find_marked()
1698 void __xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in __xa_set_mark()  argument
1704 		xas_set_mark(&xas, mark);  in __xa_set_mark()
1716 void __xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in __xa_clear_mark()  argument
1722 		xas_clear_mark(&xas, mark);  in __xa_clear_mark()
1738 bool xa_get_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in xa_get_mark()  argument
1745 	while (xas_get_mark(&xas, mark)) {  in xa_get_mark()
1768 void xa_set_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in xa_set_mark()  argument
1771 	__xa_set_mark(xa, index, mark);  in xa_set_mark()
1786 void xa_clear_mark(struct xarray *xa, unsigned long index, xa_mark_t mark)  in xa_clear_mark()  argument
1789 	__xa_clear_mark(xa, index, mark);  in xa_clear_mark()
1911 			unsigned long max, unsigned int n, xa_mark_t mark)  in xas_extract_marked()  argument
1917 	xas_for_each_marked(xas, entry, max, mark) {  in xas_extract_marked()