1 // SPDX-License-Identifier: GPL-2.0
23 	if (!c->parent)  in propagate_protected_usage()
26 	min = READ_ONCE(c->min);  in propagate_protected_usage()
27 	if (min || atomic_long_read(&c->min_usage)) {  in propagate_protected_usage()
29 		old_protected = atomic_long_xchg(&c->min_usage, protected);  in propagate_protected_usage()
30 		delta = protected - old_protected;  in propagate_protected_usage()
32 			atomic_long_add(delta, &c->parent->children_min_usage);  in propagate_protected_usage()
35 	low = READ_ONCE(c->low);  in propagate_protected_usage()
36 	if (low || atomic_long_read(&c->low_usage)) {  in propagate_protected_usage()
38 		old_protected = atomic_long_xchg(&c->low_usage, protected);  in propagate_protected_usage()
39 		delta = protected - old_protected;  in propagate_protected_usage()
41 			atomic_long_add(delta, &c->parent->children_low_usage);  in propagate_protected_usage()
46  * page_counter_cancel - take pages out of the local counter
48  * @nr_pages: number of pages to cancel
54 	new = atomic_long_sub_return(nr_pages, &counter->usage);  in page_counter_cancel()
61  * page_counter_charge - hierarchically charge pages
63  * @nr_pages: number of pages to charge
71 	for (c = counter; c; c = c->parent) {  in page_counter_charge()
74 		new = atomic_long_add_return(nr_pages, &c->usage);  in page_counter_charge()
80 		if (new > READ_ONCE(c->watermark))  in page_counter_charge()
81 			WRITE_ONCE(c->watermark, new);  in page_counter_charge()
86  * page_counter_try_charge - try to hierarchically charge pages
88  * @nr_pages: number of pages to charge
92  * of its ancestors has hit its configured limit.
100 	for (c = counter; c; c = c->parent) {  in page_counter_try_charge()
103 		 * Charge speculatively to avoid an expensive CAS.  If  in page_counter_try_charge()
104 		 * a bigger charge fails, it might falsely lock out a  in page_counter_try_charge()
105 		 * racing smaller charge and send it into reclaim  in page_counter_try_charge()
108 		 * case of a THP locking out a regular page charge.  in page_counter_try_charge()
116 		new = atomic_long_add_return(nr_pages, &c->usage);  in page_counter_try_charge()
117 		if (new > c->max) {  in page_counter_try_charge()
118 			atomic_long_sub(nr_pages, &c->usage);  in page_counter_try_charge()
125 			data_race(c->failcnt++);  in page_counter_try_charge()
134 		if (new > READ_ONCE(c->watermark))  in page_counter_try_charge()
135 			WRITE_ONCE(c->watermark, new);  in page_counter_try_charge()
140 	for (c = counter; c != *fail; c = c->parent)  in page_counter_try_charge()
147  * page_counter_uncharge - hierarchically uncharge pages
149  * @nr_pages: number of pages to uncharge
155 	for (c = counter; c; c = c->parent)  in page_counter_uncharge()
160  * page_counter_set_max - set the maximum number of pages allowed
164  * Returns 0 on success, -EBUSY if the current number of pages on the
177 		 * below the concurrently-changing counter value.  in page_counter_set_max()
180 		 * and after, so the read-swap-read is ordered and  in page_counter_set_max()
186 		usage = atomic_long_read(&counter->usage);  in page_counter_set_max()
189 			return -EBUSY;  in page_counter_set_max()
191 		old = xchg(&counter->max, nr_pages);  in page_counter_set_max()
193 		if (atomic_long_read(&counter->usage) <= usage)  in page_counter_set_max()
196 		counter->max = old;  in page_counter_set_max()
202  * page_counter_set_min - set the amount of protected memory
212 	WRITE_ONCE(counter->min, nr_pages);  in page_counter_set_min()
214 	for (c = counter; c; c = c->parent)  in page_counter_set_min()
215 		propagate_protected_usage(c, atomic_long_read(&c->usage));  in page_counter_set_min()
219  * page_counter_set_low - set the amount of protected memory
229 	WRITE_ONCE(counter->low, nr_pages);  in page_counter_set_low()
231 	for (c = counter; c; c = c->parent)  in page_counter_set_low()
232 		propagate_protected_usage(c, atomic_long_read(&c->usage));  in page_counter_set_low()
236  * page_counter_memparse - memparse() for page counter limits
239  * @nr_pages: returns the result in number of pages
241  * Returns -EINVAL, or 0 and @nr_pages on success.  @nr_pages will be
247 	char *end;  in page_counter_memparse()  local
255 	bytes = memparse(buf, &end);  in page_counter_memparse()
256 	if (*end != '\0')  in page_counter_memparse()
257 		return -EINVAL;  in page_counter_memparse()