147 	struct rvt_wss *wss = rdi->wss;  in rvt_wss_exit()  local
149 	if (!wss)  in rvt_wss_exit()
153 	kfree(wss->entries);  in rvt_wss_exit()
154 	wss->entries = NULL;  in rvt_wss_exit()
155 	kfree(rdi->wss);  in rvt_wss_exit()
156 	rdi->wss = NULL;  in rvt_wss_exit()
173 	struct rvt_wss *wss;  in rvt_wss_init()  local
177 		rdi->wss = NULL;  in rvt_wss_init()
181 	rdi->wss = kzalloc_node(sizeof(*rdi->wss), GFP_KERNEL, node);  in rvt_wss_init()
182 	if (!rdi->wss)  in rvt_wss_init()
184 	wss = rdi->wss;  in rvt_wss_init()
208 	wss->pages_mask = table_bits - 1;  in rvt_wss_init()
209 	wss->num_entries = table_bits / BITS_PER_LONG;  in rvt_wss_init()
211 	wss->threshold = (llc_bits * wss_threshold) / 100;  in rvt_wss_init()
212 	if (wss->threshold == 0)  in rvt_wss_init()
213 		wss->threshold = 1;  in rvt_wss_init()
215 	wss->clean_period = wss_clean_period;  in rvt_wss_init()
216 	atomic_set(&wss->clean_counter, wss_clean_period);  in rvt_wss_init()
218 	wss->entries = kcalloc_node(wss->num_entries, sizeof(*wss->entries),  in rvt_wss_init()
220 	if (!wss->entries) {  in rvt_wss_init()
243 static void wss_advance_clean_counter(struct rvt_wss *wss)  in wss_advance_clean_counter()  argument
250 	if (atomic_dec_and_test(&wss->clean_counter)) {  in wss_advance_clean_counter()
264 		atomic_set(&wss->clean_counter, wss->clean_period);  in wss_advance_clean_counter()
272 		entry = (atomic_inc_return(&wss->clean_entry) - 1)  in wss_advance_clean_counter()
273 			& (wss->num_entries - 1);  in wss_advance_clean_counter()
276 		bits = xchg(&wss->entries[entry], 0);  in wss_advance_clean_counter()
280 			atomic_sub(weight, &wss->total_count);  in wss_advance_clean_counter()
287 static void wss_insert(struct rvt_wss *wss, void *address)  in wss_insert()  argument
289 	u32 page = ((unsigned long)address >> PAGE_SHIFT) & wss->pages_mask;  in wss_insert()
293 	if (!test_and_set_bit(nr, &wss->entries[entry]))  in wss_insert()
294 		atomic_inc(&wss->total_count);  in wss_insert()
296 	wss_advance_clean_counter(wss);  in wss_insert()
302 static inline bool wss_exceeds_threshold(struct rvt_wss *wss)  in wss_exceeds_threshold()  argument
304 	return atomic_read(&wss->total_count) >= wss->threshold;  in wss_exceeds_threshold()
2874 	struct rvt_wss *wss = rdi->wss;  in rvt_copy_sge()  local
2886 			wss_insert(wss, sge->vaddr);  in rvt_copy_sge()
2888 				wss_insert(wss, (sge->vaddr + PAGE_SIZE));  in rvt_copy_sge()
2890 			cacheless_copy = wss_exceeds_threshold(wss);  in rvt_copy_sge()
2892 			wss_advance_clean_counter(wss);  in rvt_copy_sge()