• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Open file cache.
3  *
4  * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com>
5  */
6 
7 #include <linux/hash.h>
8 #include <linux/slab.h>
9 #include <linux/file.h>
10 #include <linux/pagemap.h>
11 #include <linux/sched.h>
12 #include <linux/list_lru.h>
13 #include <linux/fsnotify_backend.h>
14 #include <linux/fsnotify.h>
15 #include <linux/seq_file.h>
16 #include <linux/rhashtable.h>
17 
18 #include "vfs.h"
19 #include "nfsd.h"
20 #include "nfsfh.h"
21 #include "netns.h"
22 #include "filecache.h"
23 #include "trace.h"
24 
25 #define NFSD_LAUNDRETTE_DELAY		     (2 * HZ)
26 
27 #define NFSD_FILE_CACHE_UP		     (0)
28 
29 /* We only care about NFSD_MAY_READ/WRITE for this cache */
30 #define NFSD_FILE_MAY_MASK	(NFSD_MAY_READ|NFSD_MAY_WRITE)
31 
32 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
33 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
34 static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
35 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
36 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
37 
38 struct nfsd_fcache_disposal {
39 	struct work_struct work;
40 	spinlock_t lock;
41 	struct list_head freeme;
42 };
43 
44 static struct workqueue_struct *nfsd_filecache_wq __read_mostly;
45 
46 static struct kmem_cache		*nfsd_file_slab;
47 static struct kmem_cache		*nfsd_file_mark_slab;
48 static struct list_lru			nfsd_file_lru;
49 static unsigned long			nfsd_file_flags;
50 static struct fsnotify_group		*nfsd_file_fsnotify_group;
51 static struct delayed_work		nfsd_filecache_laundrette;
52 static struct rhashtable		nfsd_file_rhash_tbl
53 						____cacheline_aligned_in_smp;
54 
55 enum nfsd_file_lookup_type {
56 	NFSD_FILE_KEY_INODE,
57 	NFSD_FILE_KEY_FULL,
58 };
59 
60 struct nfsd_file_lookup_key {
61 	struct inode			*inode;
62 	struct net			*net;
63 	const struct cred		*cred;
64 	unsigned char			need;
65 	bool				gc;
66 	enum nfsd_file_lookup_type	type;
67 };
68 
69 /*
70  * The returned hash value is based solely on the address of an in-code
71  * inode, a pointer to a slab-allocated object. The entropy in such a
72  * pointer is concentrated in its middle bits.
73  */
nfsd_file_inode_hash(const struct inode * inode,u32 seed)74 static u32 nfsd_file_inode_hash(const struct inode *inode, u32 seed)
75 {
76 	unsigned long ptr = (unsigned long)inode;
77 	u32 k;
78 
79 	k = ptr >> L1_CACHE_SHIFT;
80 	k &= 0x00ffffff;
81 	return jhash2(&k, 1, seed);
82 }
83 
84 /**
85  * nfsd_file_key_hashfn - Compute the hash value of a lookup key
86  * @data: key on which to compute the hash value
87  * @len: rhash table's key_len parameter (unused)
88  * @seed: rhash table's random seed of the day
89  *
90  * Return value:
91  *   Computed 32-bit hash value
92  */
nfsd_file_key_hashfn(const void * data,u32 len,u32 seed)93 static u32 nfsd_file_key_hashfn(const void *data, u32 len, u32 seed)
94 {
95 	const struct nfsd_file_lookup_key *key = data;
96 
97 	return nfsd_file_inode_hash(key->inode, seed);
98 }
99 
100 /**
101  * nfsd_file_obj_hashfn - Compute the hash value of an nfsd_file
102  * @data: object on which to compute the hash value
103  * @len: rhash table's key_len parameter (unused)
104  * @seed: rhash table's random seed of the day
105  *
106  * Return value:
107  *   Computed 32-bit hash value
108  */
nfsd_file_obj_hashfn(const void * data,u32 len,u32 seed)109 static u32 nfsd_file_obj_hashfn(const void *data, u32 len, u32 seed)
110 {
111 	const struct nfsd_file *nf = data;
112 
113 	return nfsd_file_inode_hash(nf->nf_inode, seed);
114 }
115 
116 static bool
nfsd_match_cred(const struct cred * c1,const struct cred * c2)117 nfsd_match_cred(const struct cred *c1, const struct cred *c2)
118 {
119 	int i;
120 
121 	if (!uid_eq(c1->fsuid, c2->fsuid))
122 		return false;
123 	if (!gid_eq(c1->fsgid, c2->fsgid))
124 		return false;
125 	if (c1->group_info == NULL || c2->group_info == NULL)
126 		return c1->group_info == c2->group_info;
127 	if (c1->group_info->ngroups != c2->group_info->ngroups)
128 		return false;
129 	for (i = 0; i < c1->group_info->ngroups; i++) {
130 		if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
131 			return false;
132 	}
133 	return true;
134 }
135 
136 /**
137  * nfsd_file_obj_cmpfn - Match a cache item against search criteria
138  * @arg: search criteria
139  * @ptr: cache item to check
140  *
141  * Return values:
142  *   %0 - Item matches search criteria
143  *   %1 - Item does not match search criteria
144  */
nfsd_file_obj_cmpfn(struct rhashtable_compare_arg * arg,const void * ptr)145 static int nfsd_file_obj_cmpfn(struct rhashtable_compare_arg *arg,
146 			       const void *ptr)
147 {
148 	const struct nfsd_file_lookup_key *key = arg->key;
149 	const struct nfsd_file *nf = ptr;
150 
151 	switch (key->type) {
152 	case NFSD_FILE_KEY_INODE:
153 		if (nf->nf_inode != key->inode)
154 			return 1;
155 		break;
156 	case NFSD_FILE_KEY_FULL:
157 		if (nf->nf_inode != key->inode)
158 			return 1;
159 		if (nf->nf_may != key->need)
160 			return 1;
161 		if (nf->nf_net != key->net)
162 			return 1;
163 		if (!nfsd_match_cred(nf->nf_cred, key->cred))
164 			return 1;
165 		if (!!test_bit(NFSD_FILE_GC, &nf->nf_flags) != key->gc)
166 			return 1;
167 		if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
168 			return 1;
169 		break;
170 	}
171 	return 0;
172 }
173 
174 static const struct rhashtable_params nfsd_file_rhash_params = {
175 	.key_len		= sizeof_field(struct nfsd_file, nf_inode),
176 	.key_offset		= offsetof(struct nfsd_file, nf_inode),
177 	.head_offset		= offsetof(struct nfsd_file, nf_rhash),
178 	.hashfn			= nfsd_file_key_hashfn,
179 	.obj_hashfn		= nfsd_file_obj_hashfn,
180 	.obj_cmpfn		= nfsd_file_obj_cmpfn,
181 	/* Reduce resizing churn on light workloads */
182 	.min_size		= 512,		/* buckets */
183 	.automatic_shrinking	= true,
184 };
185 
186 static void
nfsd_file_schedule_laundrette(void)187 nfsd_file_schedule_laundrette(void)
188 {
189 	if ((atomic_read(&nfsd_file_rhash_tbl.nelems) == 0) ||
190 	    test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
191 		return;
192 
193 	queue_delayed_work(system_wq, &nfsd_filecache_laundrette,
194 			NFSD_LAUNDRETTE_DELAY);
195 }
196 
197 static void
nfsd_file_slab_free(struct rcu_head * rcu)198 nfsd_file_slab_free(struct rcu_head *rcu)
199 {
200 	struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
201 
202 	put_cred(nf->nf_cred);
203 	kmem_cache_free(nfsd_file_slab, nf);
204 }
205 
206 static void
nfsd_file_mark_free(struct fsnotify_mark * mark)207 nfsd_file_mark_free(struct fsnotify_mark *mark)
208 {
209 	struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
210 						  nfm_mark);
211 
212 	kmem_cache_free(nfsd_file_mark_slab, nfm);
213 }
214 
215 static struct nfsd_file_mark *
nfsd_file_mark_get(struct nfsd_file_mark * nfm)216 nfsd_file_mark_get(struct nfsd_file_mark *nfm)
217 {
218 	if (!refcount_inc_not_zero(&nfm->nfm_ref))
219 		return NULL;
220 	return nfm;
221 }
222 
223 static void
nfsd_file_mark_put(struct nfsd_file_mark * nfm)224 nfsd_file_mark_put(struct nfsd_file_mark *nfm)
225 {
226 	if (refcount_dec_and_test(&nfm->nfm_ref)) {
227 		fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
228 		fsnotify_put_mark(&nfm->nfm_mark);
229 	}
230 }
231 
232 static struct nfsd_file_mark *
nfsd_file_mark_find_or_create(struct nfsd_file * nf,struct inode * inode)233 nfsd_file_mark_find_or_create(struct nfsd_file *nf, struct inode *inode)
234 {
235 	int			err;
236 	struct fsnotify_mark	*mark;
237 	struct nfsd_file_mark	*nfm = NULL, *new;
238 
239 	do {
240 		fsnotify_group_lock(nfsd_file_fsnotify_group);
241 		mark = fsnotify_find_mark(&inode->i_fsnotify_marks,
242 					  nfsd_file_fsnotify_group);
243 		if (mark) {
244 			nfm = nfsd_file_mark_get(container_of(mark,
245 						 struct nfsd_file_mark,
246 						 nfm_mark));
247 			fsnotify_group_unlock(nfsd_file_fsnotify_group);
248 			if (nfm) {
249 				fsnotify_put_mark(mark);
250 				break;
251 			}
252 			/* Avoid soft lockup race with nfsd_file_mark_put() */
253 			fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
254 			fsnotify_put_mark(mark);
255 		} else {
256 			fsnotify_group_unlock(nfsd_file_fsnotify_group);
257 		}
258 
259 		/* allocate a new nfm */
260 		new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
261 		if (!new)
262 			return NULL;
263 		fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
264 		new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
265 		refcount_set(&new->nfm_ref, 1);
266 
267 		err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
268 
269 		/*
270 		 * If the add was successful, then return the object.
271 		 * Otherwise, we need to put the reference we hold on the
272 		 * nfm_mark. The fsnotify code will take a reference and put
273 		 * it on failure, so we can't just free it directly. It's also
274 		 * not safe to call fsnotify_destroy_mark on it as the
275 		 * mark->group will be NULL. Thus, we can't let the nfm_ref
276 		 * counter drive the destruction at this point.
277 		 */
278 		if (likely(!err))
279 			nfm = new;
280 		else
281 			fsnotify_put_mark(&new->nfm_mark);
282 	} while (unlikely(err == -EEXIST));
283 
284 	return nfm;
285 }
286 
287 static struct nfsd_file *
nfsd_file_alloc(struct nfsd_file_lookup_key * key,unsigned int may)288 nfsd_file_alloc(struct nfsd_file_lookup_key *key, unsigned int may)
289 {
290 	struct nfsd_file *nf;
291 
292 	nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
293 	if (nf) {
294 		INIT_LIST_HEAD(&nf->nf_lru);
295 		nf->nf_birthtime = ktime_get();
296 		nf->nf_file = NULL;
297 		nf->nf_cred = get_current_cred();
298 		nf->nf_net = key->net;
299 		nf->nf_flags = 0;
300 		__set_bit(NFSD_FILE_HASHED, &nf->nf_flags);
301 		__set_bit(NFSD_FILE_PENDING, &nf->nf_flags);
302 		if (key->gc)
303 			__set_bit(NFSD_FILE_GC, &nf->nf_flags);
304 		nf->nf_inode = key->inode;
305 		refcount_set(&nf->nf_ref, 1);
306 		nf->nf_may = key->need;
307 		nf->nf_mark = NULL;
308 	}
309 	return nf;
310 }
311 
312 /**
313  * nfsd_file_check_write_error - check for writeback errors on a file
314  * @nf: nfsd_file to check for writeback errors
315  *
316  * Check whether a nfsd_file has an unseen error. Reset the write
317  * verifier if so.
318  */
319 static void
nfsd_file_check_write_error(struct nfsd_file * nf)320 nfsd_file_check_write_error(struct nfsd_file *nf)
321 {
322 	struct file *file = nf->nf_file;
323 
324 	if ((file->f_mode & FMODE_WRITE) &&
325 	    filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err)))
326 		nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
327 }
328 
329 static void
nfsd_file_hash_remove(struct nfsd_file * nf)330 nfsd_file_hash_remove(struct nfsd_file *nf)
331 {
332 	trace_nfsd_file_unhash(nf);
333 	rhashtable_remove_fast(&nfsd_file_rhash_tbl, &nf->nf_rhash,
334 			       nfsd_file_rhash_params);
335 }
336 
337 static bool
nfsd_file_unhash(struct nfsd_file * nf)338 nfsd_file_unhash(struct nfsd_file *nf)
339 {
340 	if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
341 		nfsd_file_hash_remove(nf);
342 		return true;
343 	}
344 	return false;
345 }
346 
347 static void
nfsd_file_free(struct nfsd_file * nf)348 nfsd_file_free(struct nfsd_file *nf)
349 {
350 	s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
351 
352 	trace_nfsd_file_free(nf);
353 
354 	this_cpu_inc(nfsd_file_releases);
355 	this_cpu_add(nfsd_file_total_age, age);
356 
357 	nfsd_file_unhash(nf);
358 	if (nf->nf_mark)
359 		nfsd_file_mark_put(nf->nf_mark);
360 	if (nf->nf_file) {
361 		get_file(nf->nf_file);
362 		filp_close(nf->nf_file, NULL);
363 		nfsd_file_check_write_error(nf);
364 		fput(nf->nf_file);
365 	}
366 
367 	/*
368 	 * If this item is still linked via nf_lru, that's a bug.
369 	 * WARN and leak it to preserve system stability.
370 	 */
371 	if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
372 		return;
373 
374 	call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
375 }
376 
377 static bool
nfsd_file_check_writeback(struct nfsd_file * nf)378 nfsd_file_check_writeback(struct nfsd_file *nf)
379 {
380 	struct file *file = nf->nf_file;
381 	struct address_space *mapping;
382 
383 	if (!file || !(file->f_mode & FMODE_WRITE))
384 		return false;
385 	mapping = file->f_mapping;
386 	return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
387 		mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
388 }
389 
nfsd_file_lru_add(struct nfsd_file * nf)390 static bool nfsd_file_lru_add(struct nfsd_file *nf)
391 {
392 	set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
393 	if (list_lru_add(&nfsd_file_lru, &nf->nf_lru)) {
394 		trace_nfsd_file_lru_add(nf);
395 		return true;
396 	}
397 	return false;
398 }
399 
nfsd_file_lru_remove(struct nfsd_file * nf)400 static bool nfsd_file_lru_remove(struct nfsd_file *nf)
401 {
402 	if (list_lru_del(&nfsd_file_lru, &nf->nf_lru)) {
403 		trace_nfsd_file_lru_del(nf);
404 		return true;
405 	}
406 	return false;
407 }
408 
409 struct nfsd_file *
nfsd_file_get(struct nfsd_file * nf)410 nfsd_file_get(struct nfsd_file *nf)
411 {
412 	if (likely(refcount_inc_not_zero(&nf->nf_ref)))
413 		return nf;
414 	return NULL;
415 }
416 
417 /**
418  * nfsd_file_put - put the reference to a nfsd_file
419  * @nf: nfsd_file of which to put the reference
420  *
421  * Put a reference to a nfsd_file. In the non-GC case, we just put the
422  * reference immediately. In the GC case, if the reference would be
423  * the last one, the put it on the LRU instead to be cleaned up later.
424  */
425 void
nfsd_file_put(struct nfsd_file * nf)426 nfsd_file_put(struct nfsd_file *nf)
427 {
428 	might_sleep();
429 	trace_nfsd_file_put(nf);
430 
431 	if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
432 	    test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
433 		/*
434 		 * If this is the last reference (nf_ref == 1), then try to
435 		 * transfer it to the LRU.
436 		 */
437 		if (refcount_dec_not_one(&nf->nf_ref))
438 			return;
439 
440 		/* Try to add it to the LRU.  If that fails, decrement. */
441 		if (nfsd_file_lru_add(nf)) {
442 			/* If it's still hashed, we're done */
443 			if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
444 				nfsd_file_schedule_laundrette();
445 				return;
446 			}
447 
448 			/*
449 			 * We're racing with unhashing, so try to remove it from
450 			 * the LRU. If removal fails, then someone else already
451 			 * has our reference.
452 			 */
453 			if (!nfsd_file_lru_remove(nf))
454 				return;
455 		}
456 	}
457 	if (refcount_dec_and_test(&nf->nf_ref))
458 		nfsd_file_free(nf);
459 }
460 
461 static void
nfsd_file_dispose_list(struct list_head * dispose)462 nfsd_file_dispose_list(struct list_head *dispose)
463 {
464 	struct nfsd_file *nf;
465 
466 	while (!list_empty(dispose)) {
467 		nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
468 		list_del_init(&nf->nf_lru);
469 		nfsd_file_free(nf);
470 	}
471 }
472 
473 static void
nfsd_file_list_remove_disposal(struct list_head * dst,struct nfsd_fcache_disposal * l)474 nfsd_file_list_remove_disposal(struct list_head *dst,
475 		struct nfsd_fcache_disposal *l)
476 {
477 	spin_lock(&l->lock);
478 	list_splice_init(&l->freeme, dst);
479 	spin_unlock(&l->lock);
480 }
481 
482 static void
nfsd_file_list_add_disposal(struct list_head * files,struct net * net)483 nfsd_file_list_add_disposal(struct list_head *files, struct net *net)
484 {
485 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
486 	struct nfsd_fcache_disposal *l = nn->fcache_disposal;
487 
488 	spin_lock(&l->lock);
489 	list_splice_tail_init(files, &l->freeme);
490 	spin_unlock(&l->lock);
491 	queue_work(nfsd_filecache_wq, &l->work);
492 }
493 
494 static void
nfsd_file_list_add_pernet(struct list_head * dst,struct list_head * src,struct net * net)495 nfsd_file_list_add_pernet(struct list_head *dst, struct list_head *src,
496 		struct net *net)
497 {
498 	struct nfsd_file *nf, *tmp;
499 
500 	list_for_each_entry_safe(nf, tmp, src, nf_lru) {
501 		if (nf->nf_net == net)
502 			list_move_tail(&nf->nf_lru, dst);
503 	}
504 }
505 
506 static void
nfsd_file_dispose_list_delayed(struct list_head * dispose)507 nfsd_file_dispose_list_delayed(struct list_head *dispose)
508 {
509 	LIST_HEAD(list);
510 	struct nfsd_file *nf;
511 
512 	while(!list_empty(dispose)) {
513 		nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
514 		nfsd_file_list_add_pernet(&list, dispose, nf->nf_net);
515 		nfsd_file_list_add_disposal(&list, nf->nf_net);
516 	}
517 }
518 
519 /**
520  * nfsd_file_lru_cb - Examine an entry on the LRU list
521  * @item: LRU entry to examine
522  * @lru: controlling LRU
523  * @lock: LRU list lock (unused)
524  * @arg: dispose list
525  *
526  * Return values:
527  *   %LRU_REMOVED: @item was removed from the LRU
528  *   %LRU_ROTATE: @item is to be moved to the LRU tail
529  *   %LRU_SKIP: @item cannot be evicted
530  */
531 static enum lru_status
nfsd_file_lru_cb(struct list_head * item,struct list_lru_one * lru,spinlock_t * lock,void * arg)532 nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
533 		 spinlock_t *lock, void *arg)
534 	__releases(lock)
535 	__acquires(lock)
536 {
537 	struct list_head *head = arg;
538 	struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
539 
540 	/* We should only be dealing with GC entries here */
541 	WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));
542 
543 	/*
544 	 * Don't throw out files that are still undergoing I/O or
545 	 * that have uncleared errors pending.
546 	 */
547 	if (nfsd_file_check_writeback(nf)) {
548 		trace_nfsd_file_gc_writeback(nf);
549 		return LRU_SKIP;
550 	}
551 
552 	/* If it was recently added to the list, skip it */
553 	if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
554 		trace_nfsd_file_gc_referenced(nf);
555 		return LRU_ROTATE;
556 	}
557 
558 	/*
559 	 * Put the reference held on behalf of the LRU. If it wasn't the last
560 	 * one, then just remove it from the LRU and ignore it.
561 	 */
562 	if (!refcount_dec_and_test(&nf->nf_ref)) {
563 		trace_nfsd_file_gc_in_use(nf);
564 		list_lru_isolate(lru, &nf->nf_lru);
565 		return LRU_REMOVED;
566 	}
567 
568 	/* Refcount went to zero. Unhash it and queue it to the dispose list */
569 	nfsd_file_unhash(nf);
570 	list_lru_isolate_move(lru, &nf->nf_lru, head);
571 	this_cpu_inc(nfsd_file_evictions);
572 	trace_nfsd_file_gc_disposed(nf);
573 	return LRU_REMOVED;
574 }
575 
576 static void
nfsd_file_gc(void)577 nfsd_file_gc(void)
578 {
579 	LIST_HEAD(dispose);
580 	unsigned long ret;
581 
582 	ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb,
583 			    &dispose, list_lru_count(&nfsd_file_lru));
584 	trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
585 	nfsd_file_dispose_list_delayed(&dispose);
586 }
587 
588 static void
nfsd_file_gc_worker(struct work_struct * work)589 nfsd_file_gc_worker(struct work_struct *work)
590 {
591 	nfsd_file_gc();
592 	nfsd_file_schedule_laundrette();
593 }
594 
595 static unsigned long
nfsd_file_lru_count(struct shrinker * s,struct shrink_control * sc)596 nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
597 {
598 	return list_lru_count(&nfsd_file_lru);
599 }
600 
601 static unsigned long
nfsd_file_lru_scan(struct shrinker * s,struct shrink_control * sc)602 nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
603 {
604 	LIST_HEAD(dispose);
605 	unsigned long ret;
606 
607 	ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
608 				   nfsd_file_lru_cb, &dispose);
609 	trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
610 	nfsd_file_dispose_list_delayed(&dispose);
611 	return ret;
612 }
613 
614 static struct shrinker	nfsd_file_shrinker = {
615 	.scan_objects = nfsd_file_lru_scan,
616 	.count_objects = nfsd_file_lru_count,
617 	.seeks = 1,
618 };
619 
620 /**
621  * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file
622  * @nf: nfsd_file to attempt to queue
623  * @dispose: private list to queue successfully-put objects
624  *
625  * Unhash an nfsd_file, try to get a reference to it, and then put that
626  * reference. If it's the last reference, queue it to the dispose list.
627  */
628 static void
nfsd_file_cond_queue(struct nfsd_file * nf,struct list_head * dispose)629 nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose)
630 	__must_hold(RCU)
631 {
632 	int decrement = 1;
633 
634 	/* If we raced with someone else unhashing, ignore it */
635 	if (!nfsd_file_unhash(nf))
636 		return;
637 
638 	/* If we can't get a reference, ignore it */
639 	if (!nfsd_file_get(nf))
640 		return;
641 
642 	/* Extra decrement if we remove from the LRU */
643 	if (nfsd_file_lru_remove(nf))
644 		++decrement;
645 
646 	/* If refcount goes to 0, then put on the dispose list */
647 	if (refcount_sub_and_test(decrement, &nf->nf_ref)) {
648 		list_add(&nf->nf_lru, dispose);
649 		trace_nfsd_file_closing(nf);
650 	}
651 }
652 
653 /**
654  * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode
655  * @inode:   inode on which to close out nfsd_files
656  * @dispose: list on which to gather nfsd_files to close out
657  *
658  * An nfsd_file represents a struct file being held open on behalf of nfsd. An
659  * open file however can block other activity (such as leases), or cause
660  * undesirable behavior (e.g. spurious silly-renames when reexporting NFS).
661  *
662  * This function is intended to find open nfsd_files when this sort of
663  * conflicting access occurs and then attempt to close those files out.
664  *
665  * Populates the dispose list with entries that have already had their
666  * refcounts go to zero. The actual free of an nfsd_file can be expensive,
667  * so we leave it up to the caller whether it wants to wait or not.
668  */
669 static void
nfsd_file_queue_for_close(struct inode * inode,struct list_head * dispose)670 nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose)
671 {
672 	struct nfsd_file_lookup_key key = {
673 		.type	= NFSD_FILE_KEY_INODE,
674 		.inode	= inode,
675 	};
676 	struct nfsd_file *nf;
677 
678 	rcu_read_lock();
679 	do {
680 		nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
681 				       nfsd_file_rhash_params);
682 		if (!nf)
683 			break;
684 		nfsd_file_cond_queue(nf, dispose);
685 	} while (1);
686 	rcu_read_unlock();
687 }
688 
689 /**
690  * nfsd_file_close_inode - attempt a delayed close of a nfsd_file
691  * @inode: inode of the file to attempt to remove
692  *
693  * Close out any open nfsd_files that can be reaped for @inode. The
694  * actual freeing is deferred to the dispose_list_delayed infrastructure.
695  *
696  * This is used by the fsnotify callbacks and setlease notifier.
697  */
698 static void
nfsd_file_close_inode(struct inode * inode)699 nfsd_file_close_inode(struct inode *inode)
700 {
701 	LIST_HEAD(dispose);
702 
703 	nfsd_file_queue_for_close(inode, &dispose);
704 	nfsd_file_dispose_list_delayed(&dispose);
705 }
706 
707 /**
708  * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
709  * @inode: inode of the file to attempt to remove
710  *
711  * Close out any open nfsd_files that can be reaped for @inode. The
712  * nfsd_files are closed out synchronously.
713  *
714  * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames
715  * when reexporting NFS.
716  */
717 void
nfsd_file_close_inode_sync(struct inode * inode)718 nfsd_file_close_inode_sync(struct inode *inode)
719 {
720 	struct nfsd_file *nf;
721 	LIST_HEAD(dispose);
722 
723 	trace_nfsd_file_close(inode);
724 
725 	nfsd_file_queue_for_close(inode, &dispose);
726 	while (!list_empty(&dispose)) {
727 		nf = list_first_entry(&dispose, struct nfsd_file, nf_lru);
728 		list_del_init(&nf->nf_lru);
729 		nfsd_file_free(nf);
730 	}
731 	flush_delayed_fput();
732 }
733 
734 /**
735  * nfsd_file_delayed_close - close unused nfsd_files
736  * @work: dummy
737  *
738  * Walk the LRU list and destroy any entries that have not been used since
739  * the last scan.
740  */
741 static void
nfsd_file_delayed_close(struct work_struct * work)742 nfsd_file_delayed_close(struct work_struct *work)
743 {
744 	LIST_HEAD(head);
745 	struct nfsd_fcache_disposal *l = container_of(work,
746 			struct nfsd_fcache_disposal, work);
747 
748 	nfsd_file_list_remove_disposal(&head, l);
749 	nfsd_file_dispose_list(&head);
750 }
751 
752 static int
nfsd_file_lease_notifier_call(struct notifier_block * nb,unsigned long arg,void * data)753 nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
754 			    void *data)
755 {
756 	struct file_lock *fl = data;
757 
758 	/* Only close files for F_SETLEASE leases */
759 	if (fl->fl_flags & FL_LEASE)
760 		nfsd_file_close_inode(file_inode(fl->fl_file));
761 	return 0;
762 }
763 
764 static struct notifier_block nfsd_file_lease_notifier = {
765 	.notifier_call = nfsd_file_lease_notifier_call,
766 };
767 
768 static int
nfsd_file_fsnotify_handle_event(struct fsnotify_mark * mark,u32 mask,struct inode * inode,struct inode * dir,const struct qstr * name,u32 cookie)769 nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
770 				struct inode *inode, struct inode *dir,
771 				const struct qstr *name, u32 cookie)
772 {
773 	if (WARN_ON_ONCE(!inode))
774 		return 0;
775 
776 	trace_nfsd_file_fsnotify_handle_event(inode, mask);
777 
778 	/* Should be no marks on non-regular files */
779 	if (!S_ISREG(inode->i_mode)) {
780 		WARN_ON_ONCE(1);
781 		return 0;
782 	}
783 
784 	/* don't close files if this was not the last link */
785 	if (mask & FS_ATTRIB) {
786 		if (inode->i_nlink)
787 			return 0;
788 	}
789 
790 	nfsd_file_close_inode(inode);
791 	return 0;
792 }
793 
794 
795 static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
796 	.handle_inode_event = nfsd_file_fsnotify_handle_event,
797 	.free_mark = nfsd_file_mark_free,
798 };
799 
800 int
nfsd_file_cache_init(void)801 nfsd_file_cache_init(void)
802 {
803 	int ret;
804 
805 	lockdep_assert_held(&nfsd_mutex);
806 	if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
807 		return 0;
808 
809 	ret = rhashtable_init(&nfsd_file_rhash_tbl, &nfsd_file_rhash_params);
810 	if (ret)
811 		return ret;
812 
813 	ret = -ENOMEM;
814 	nfsd_filecache_wq = alloc_workqueue("nfsd_filecache", 0, 0);
815 	if (!nfsd_filecache_wq)
816 		goto out;
817 
818 	nfsd_file_slab = kmem_cache_create("nfsd_file",
819 				sizeof(struct nfsd_file), 0, 0, NULL);
820 	if (!nfsd_file_slab) {
821 		pr_err("nfsd: unable to create nfsd_file_slab\n");
822 		goto out_err;
823 	}
824 
825 	nfsd_file_mark_slab = kmem_cache_create("nfsd_file_mark",
826 					sizeof(struct nfsd_file_mark), 0, 0, NULL);
827 	if (!nfsd_file_mark_slab) {
828 		pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
829 		goto out_err;
830 	}
831 
832 
833 	ret = list_lru_init(&nfsd_file_lru);
834 	if (ret) {
835 		pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
836 		goto out_err;
837 	}
838 
839 	ret = register_shrinker(&nfsd_file_shrinker, "nfsd-filecache");
840 	if (ret) {
841 		pr_err("nfsd: failed to register nfsd_file_shrinker: %d\n", ret);
842 		goto out_lru;
843 	}
844 
845 	ret = lease_register_notifier(&nfsd_file_lease_notifier);
846 	if (ret) {
847 		pr_err("nfsd: unable to register lease notifier: %d\n", ret);
848 		goto out_shrinker;
849 	}
850 
851 	nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
852 							FSNOTIFY_GROUP_NOFS);
853 	if (IS_ERR(nfsd_file_fsnotify_group)) {
854 		pr_err("nfsd: unable to create fsnotify group: %ld\n",
855 			PTR_ERR(nfsd_file_fsnotify_group));
856 		ret = PTR_ERR(nfsd_file_fsnotify_group);
857 		nfsd_file_fsnotify_group = NULL;
858 		goto out_notifier;
859 	}
860 
861 	INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
862 out:
863 	return ret;
864 out_notifier:
865 	lease_unregister_notifier(&nfsd_file_lease_notifier);
866 out_shrinker:
867 	unregister_shrinker(&nfsd_file_shrinker);
868 out_lru:
869 	list_lru_destroy(&nfsd_file_lru);
870 out_err:
871 	kmem_cache_destroy(nfsd_file_slab);
872 	nfsd_file_slab = NULL;
873 	kmem_cache_destroy(nfsd_file_mark_slab);
874 	nfsd_file_mark_slab = NULL;
875 	destroy_workqueue(nfsd_filecache_wq);
876 	nfsd_filecache_wq = NULL;
877 	rhashtable_destroy(&nfsd_file_rhash_tbl);
878 	goto out;
879 }
880 
881 /**
882  * __nfsd_file_cache_purge: clean out the cache for shutdown
883  * @net: net-namespace to shut down the cache (may be NULL)
884  *
885  * Walk the nfsd_file cache and close out any that match @net. If @net is NULL,
886  * then close out everything. Called when an nfsd instance is being shut down.
887  */
888 static void
__nfsd_file_cache_purge(struct net * net)889 __nfsd_file_cache_purge(struct net *net)
890 {
891 	struct rhashtable_iter iter;
892 	struct nfsd_file *nf;
893 	LIST_HEAD(dispose);
894 
895 	rhashtable_walk_enter(&nfsd_file_rhash_tbl, &iter);
896 	do {
897 		rhashtable_walk_start(&iter);
898 
899 		nf = rhashtable_walk_next(&iter);
900 		while (!IS_ERR_OR_NULL(nf)) {
901 			if (!net || nf->nf_net == net)
902 				nfsd_file_cond_queue(nf, &dispose);
903 			nf = rhashtable_walk_next(&iter);
904 		}
905 
906 		rhashtable_walk_stop(&iter);
907 	} while (nf == ERR_PTR(-EAGAIN));
908 	rhashtable_walk_exit(&iter);
909 
910 	nfsd_file_dispose_list(&dispose);
911 }
912 
913 static struct nfsd_fcache_disposal *
nfsd_alloc_fcache_disposal(void)914 nfsd_alloc_fcache_disposal(void)
915 {
916 	struct nfsd_fcache_disposal *l;
917 
918 	l = kmalloc(sizeof(*l), GFP_KERNEL);
919 	if (!l)
920 		return NULL;
921 	INIT_WORK(&l->work, nfsd_file_delayed_close);
922 	spin_lock_init(&l->lock);
923 	INIT_LIST_HEAD(&l->freeme);
924 	return l;
925 }
926 
927 static void
nfsd_free_fcache_disposal(struct nfsd_fcache_disposal * l)928 nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
929 {
930 	cancel_work_sync(&l->work);
931 	nfsd_file_dispose_list(&l->freeme);
932 	kfree(l);
933 }
934 
935 static void
nfsd_free_fcache_disposal_net(struct net * net)936 nfsd_free_fcache_disposal_net(struct net *net)
937 {
938 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
939 	struct nfsd_fcache_disposal *l = nn->fcache_disposal;
940 
941 	nfsd_free_fcache_disposal(l);
942 }
943 
944 int
nfsd_file_cache_start_net(struct net * net)945 nfsd_file_cache_start_net(struct net *net)
946 {
947 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
948 
949 	nn->fcache_disposal = nfsd_alloc_fcache_disposal();
950 	return nn->fcache_disposal ? 0 : -ENOMEM;
951 }
952 
953 /**
954  * nfsd_file_cache_purge - Remove all cache items associated with @net
955  * @net: target net namespace
956  *
957  */
958 void
nfsd_file_cache_purge(struct net * net)959 nfsd_file_cache_purge(struct net *net)
960 {
961 	lockdep_assert_held(&nfsd_mutex);
962 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
963 		__nfsd_file_cache_purge(net);
964 }
965 
966 void
nfsd_file_cache_shutdown_net(struct net * net)967 nfsd_file_cache_shutdown_net(struct net *net)
968 {
969 	nfsd_file_cache_purge(net);
970 	nfsd_free_fcache_disposal_net(net);
971 }
972 
973 void
nfsd_file_cache_shutdown(void)974 nfsd_file_cache_shutdown(void)
975 {
976 	int i;
977 
978 	lockdep_assert_held(&nfsd_mutex);
979 	if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
980 		return;
981 
982 	lease_unregister_notifier(&nfsd_file_lease_notifier);
983 	unregister_shrinker(&nfsd_file_shrinker);
984 	/*
985 	 * make sure all callers of nfsd_file_lru_cb are done before
986 	 * calling nfsd_file_cache_purge
987 	 */
988 	cancel_delayed_work_sync(&nfsd_filecache_laundrette);
989 	__nfsd_file_cache_purge(NULL);
990 	list_lru_destroy(&nfsd_file_lru);
991 	rcu_barrier();
992 	fsnotify_put_group(nfsd_file_fsnotify_group);
993 	nfsd_file_fsnotify_group = NULL;
994 	kmem_cache_destroy(nfsd_file_slab);
995 	nfsd_file_slab = NULL;
996 	fsnotify_wait_marks_destroyed();
997 	kmem_cache_destroy(nfsd_file_mark_slab);
998 	nfsd_file_mark_slab = NULL;
999 	destroy_workqueue(nfsd_filecache_wq);
1000 	nfsd_filecache_wq = NULL;
1001 	rhashtable_destroy(&nfsd_file_rhash_tbl);
1002 
1003 	for_each_possible_cpu(i) {
1004 		per_cpu(nfsd_file_cache_hits, i) = 0;
1005 		per_cpu(nfsd_file_acquisitions, i) = 0;
1006 		per_cpu(nfsd_file_releases, i) = 0;
1007 		per_cpu(nfsd_file_total_age, i) = 0;
1008 		per_cpu(nfsd_file_evictions, i) = 0;
1009 	}
1010 }
1011 
1012 /**
1013  * nfsd_file_is_cached - are there any cached open files for this inode?
1014  * @inode: inode to check
1015  *
1016  * The lookup matches inodes in all net namespaces and is atomic wrt
1017  * nfsd_file_acquire().
1018  *
1019  * Return values:
1020  *   %true: filecache contains at least one file matching this inode
1021  *   %false: filecache contains no files matching this inode
1022  */
1023 bool
nfsd_file_is_cached(struct inode * inode)1024 nfsd_file_is_cached(struct inode *inode)
1025 {
1026 	struct nfsd_file_lookup_key key = {
1027 		.type	= NFSD_FILE_KEY_INODE,
1028 		.inode	= inode,
1029 	};
1030 	bool ret = false;
1031 
1032 	if (rhashtable_lookup_fast(&nfsd_file_rhash_tbl, &key,
1033 				   nfsd_file_rhash_params) != NULL)
1034 		ret = true;
1035 	trace_nfsd_file_is_cached(inode, (int)ret);
1036 	return ret;
1037 }
1038 
1039 static __be32
nfsd_file_do_acquire(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf,bool want_gc)1040 nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1041 		     unsigned int may_flags, struct file *file,
1042 		     struct nfsd_file **pnf, bool want_gc)
1043 {
1044 	struct nfsd_file_lookup_key key = {
1045 		.type	= NFSD_FILE_KEY_FULL,
1046 		.need	= may_flags & NFSD_FILE_MAY_MASK,
1047 		.net	= SVC_NET(rqstp),
1048 		.gc	= want_gc,
1049 	};
1050 	bool open_retry = true;
1051 	struct nfsd_file *nf;
1052 	__be32 status;
1053 	int ret;
1054 
1055 	status = fh_verify(rqstp, fhp, S_IFREG,
1056 				may_flags|NFSD_MAY_OWNER_OVERRIDE);
1057 	if (status != nfs_ok)
1058 		return status;
1059 	key.inode = d_inode(fhp->fh_dentry);
1060 	key.cred = get_current_cred();
1061 
1062 retry:
1063 	rcu_read_lock();
1064 	nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
1065 			       nfsd_file_rhash_params);
1066 	if (nf)
1067 		nf = nfsd_file_get(nf);
1068 	rcu_read_unlock();
1069 
1070 	if (nf) {
1071 		if (nfsd_file_lru_remove(nf))
1072 			WARN_ON_ONCE(refcount_dec_and_test(&nf->nf_ref));
1073 		goto wait_for_construction;
1074 	}
1075 
1076 	nf = nfsd_file_alloc(&key, may_flags);
1077 	if (!nf) {
1078 		status = nfserr_jukebox;
1079 		goto out_status;
1080 	}
1081 
1082 	ret = rhashtable_lookup_insert_key(&nfsd_file_rhash_tbl,
1083 					   &key, &nf->nf_rhash,
1084 					   nfsd_file_rhash_params);
1085 	if (likely(ret == 0))
1086 		goto open_file;
1087 
1088 	nfsd_file_slab_free(&nf->nf_rcu);
1089 	nf = NULL;
1090 	if (ret == -EEXIST)
1091 		goto retry;
1092 	trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, ret);
1093 	status = nfserr_jukebox;
1094 	goto out_status;
1095 
1096 wait_for_construction:
1097 	wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
1098 
1099 	/* Did construction of this file fail? */
1100 	if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
1101 		trace_nfsd_file_cons_err(rqstp, key.inode, may_flags, nf);
1102 		if (!open_retry) {
1103 			status = nfserr_jukebox;
1104 			goto out;
1105 		}
1106 		open_retry = false;
1107 		if (refcount_dec_and_test(&nf->nf_ref))
1108 			nfsd_file_free(nf);
1109 		goto retry;
1110 	}
1111 
1112 	this_cpu_inc(nfsd_file_cache_hits);
1113 
1114 	status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
1115 out:
1116 	if (status == nfs_ok) {
1117 		this_cpu_inc(nfsd_file_acquisitions);
1118 		nfsd_file_check_write_error(nf);
1119 		*pnf = nf;
1120 	} else {
1121 		if (refcount_dec_and_test(&nf->nf_ref))
1122 			nfsd_file_free(nf);
1123 		nf = NULL;
1124 	}
1125 
1126 out_status:
1127 	put_cred(key.cred);
1128 	trace_nfsd_file_acquire(rqstp, key.inode, may_flags, nf, status);
1129 	return status;
1130 
1131 open_file:
1132 	trace_nfsd_file_alloc(nf);
1133 	nf->nf_mark = nfsd_file_mark_find_or_create(nf, key.inode);
1134 	if (nf->nf_mark) {
1135 		if (file) {
1136 			get_file(file);
1137 			nf->nf_file = file;
1138 			status = nfs_ok;
1139 			trace_nfsd_file_opened(nf, status);
1140 		} else {
1141 			status = nfsd_open_verified(rqstp, fhp, may_flags,
1142 						    &nf->nf_file);
1143 			trace_nfsd_file_open(nf, status);
1144 		}
1145 	} else
1146 		status = nfserr_jukebox;
1147 	/*
1148 	 * If construction failed, or we raced with a call to unlink()
1149 	 * then unhash.
1150 	 */
1151 	if (status == nfs_ok && key.inode->i_nlink == 0)
1152 		status = nfserr_jukebox;
1153 	if (status != nfs_ok)
1154 		nfsd_file_unhash(nf);
1155 	clear_bit_unlock(NFSD_FILE_PENDING, &nf->nf_flags);
1156 	smp_mb__after_atomic();
1157 	wake_up_bit(&nf->nf_flags, NFSD_FILE_PENDING);
1158 	goto out;
1159 }
1160 
1161 /**
1162  * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file
1163  * @rqstp: the RPC transaction being executed
1164  * @fhp: the NFS filehandle of the file to be opened
1165  * @may_flags: NFSD_MAY_ settings for the file
1166  * @pnf: OUT: new or found "struct nfsd_file" object
1167  *
1168  * The nfsd_file object returned by this API is reference-counted
1169  * and garbage-collected. The object is retained for a few
1170  * seconds after the final nfsd_file_put() in case the caller
1171  * wants to re-use it.
1172  *
1173  * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1174  * network byte order is returned.
1175  */
1176 __be32
nfsd_file_acquire_gc(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1177 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp,
1178 		     unsigned int may_flags, struct nfsd_file **pnf)
1179 {
1180 	return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, true);
1181 }
1182 
1183 /**
1184  * nfsd_file_acquire - Get a struct nfsd_file with an open file
1185  * @rqstp: the RPC transaction being executed
1186  * @fhp: the NFS filehandle of the file to be opened
1187  * @may_flags: NFSD_MAY_ settings for the file
1188  * @pnf: OUT: new or found "struct nfsd_file" object
1189  *
1190  * The nfsd_file_object returned by this API is reference-counted
1191  * but not garbage-collected. The object is unhashed after the
1192  * final nfsd_file_put().
1193  *
1194  * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1195  * network byte order is returned.
1196  */
1197 __be32
nfsd_file_acquire(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1198 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1199 		  unsigned int may_flags, struct nfsd_file **pnf)
1200 {
1201 	return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, false);
1202 }
1203 
1204 /**
1205  * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file
1206  * @rqstp: the RPC transaction being executed
1207  * @fhp: the NFS filehandle of the file just created
1208  * @may_flags: NFSD_MAY_ settings for the file
1209  * @file: cached, already-open file (may be NULL)
1210  * @pnf: OUT: new or found "struct nfsd_file" object
1211  *
1212  * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist,
1213  * and @file is non-NULL, use it to instantiate a new nfsd_file instead of
1214  * opening a new one.
1215  *
1216  * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1217  * network byte order is returned.
1218  */
1219 __be32
nfsd_file_acquire_opened(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf)1220 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp,
1221 			 unsigned int may_flags, struct file *file,
1222 			 struct nfsd_file **pnf)
1223 {
1224 	return nfsd_file_do_acquire(rqstp, fhp, may_flags, file, pnf, false);
1225 }
1226 
1227 /*
1228  * Note that fields may be added, removed or reordered in the future. Programs
1229  * scraping this file for info should test the labels to ensure they're
1230  * getting the correct field.
1231  */
nfsd_file_cache_stats_show(struct seq_file * m,void * v)1232 int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
1233 {
1234 	unsigned long releases = 0, evictions = 0;
1235 	unsigned long hits = 0, acquisitions = 0;
1236 	unsigned int i, count = 0, buckets = 0;
1237 	unsigned long lru = 0, total_age = 0;
1238 
1239 	/* Serialize with server shutdown */
1240 	mutex_lock(&nfsd_mutex);
1241 	if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
1242 		struct bucket_table *tbl;
1243 		struct rhashtable *ht;
1244 
1245 		lru = list_lru_count(&nfsd_file_lru);
1246 
1247 		rcu_read_lock();
1248 		ht = &nfsd_file_rhash_tbl;
1249 		count = atomic_read(&ht->nelems);
1250 		tbl = rht_dereference_rcu(ht->tbl, ht);
1251 		buckets = tbl->size;
1252 		rcu_read_unlock();
1253 	}
1254 	mutex_unlock(&nfsd_mutex);
1255 
1256 	for_each_possible_cpu(i) {
1257 		hits += per_cpu(nfsd_file_cache_hits, i);
1258 		acquisitions += per_cpu(nfsd_file_acquisitions, i);
1259 		releases += per_cpu(nfsd_file_releases, i);
1260 		total_age += per_cpu(nfsd_file_total_age, i);
1261 		evictions += per_cpu(nfsd_file_evictions, i);
1262 	}
1263 
1264 	seq_printf(m, "total entries: %u\n", count);
1265 	seq_printf(m, "hash buckets:  %u\n", buckets);
1266 	seq_printf(m, "lru entries:   %lu\n", lru);
1267 	seq_printf(m, "cache hits:    %lu\n", hits);
1268 	seq_printf(m, "acquisitions:  %lu\n", acquisitions);
1269 	seq_printf(m, "releases:      %lu\n", releases);
1270 	seq_printf(m, "evictions:     %lu\n", evictions);
1271 	if (releases)
1272 		seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
1273 	else
1274 		seq_printf(m, "mean age (ms): -\n");
1275 	return 0;
1276 }
1277