1 /*
2 * Open file cache.
3 *
4 * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com>
5 */
6
7 #include <linux/hash.h>
8 #include <linux/slab.h>
9 #include <linux/file.h>
10 #include <linux/pagemap.h>
11 #include <linux/sched.h>
12 #include <linux/list_lru.h>
13 #include <linux/fsnotify_backend.h>
14 #include <linux/fsnotify.h>
15 #include <linux/seq_file.h>
16 #include <linux/rhashtable.h>
17
18 #include "vfs.h"
19 #include "nfsd.h"
20 #include "nfsfh.h"
21 #include "netns.h"
22 #include "filecache.h"
23 #include "trace.h"
24
25 #define NFSD_LAUNDRETTE_DELAY (2 * HZ)
26
27 #define NFSD_FILE_CACHE_UP (0)
28
29 /* We only care about NFSD_MAY_READ/WRITE for this cache */
30 #define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE)
31
32 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
33 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
34 static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
35 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
36 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
37
38 struct nfsd_fcache_disposal {
39 struct work_struct work;
40 spinlock_t lock;
41 struct list_head freeme;
42 };
43
44 static struct workqueue_struct *nfsd_filecache_wq __read_mostly;
45
46 static struct kmem_cache *nfsd_file_slab;
47 static struct kmem_cache *nfsd_file_mark_slab;
48 static struct list_lru nfsd_file_lru;
49 static unsigned long nfsd_file_flags;
50 static struct fsnotify_group *nfsd_file_fsnotify_group;
51 static struct delayed_work nfsd_filecache_laundrette;
52 static struct rhashtable nfsd_file_rhash_tbl
53 ____cacheline_aligned_in_smp;
54
55 enum nfsd_file_lookup_type {
56 NFSD_FILE_KEY_INODE,
57 NFSD_FILE_KEY_FULL,
58 };
59
60 struct nfsd_file_lookup_key {
61 struct inode *inode;
62 struct net *net;
63 const struct cred *cred;
64 unsigned char need;
65 bool gc;
66 enum nfsd_file_lookup_type type;
67 };
68
69 /*
70 * The returned hash value is based solely on the address of an in-code
71 * inode, a pointer to a slab-allocated object. The entropy in such a
72 * pointer is concentrated in its middle bits.
73 */
nfsd_file_inode_hash(const struct inode * inode,u32 seed)74 static u32 nfsd_file_inode_hash(const struct inode *inode, u32 seed)
75 {
76 unsigned long ptr = (unsigned long)inode;
77 u32 k;
78
79 k = ptr >> L1_CACHE_SHIFT;
80 k &= 0x00ffffff;
81 return jhash2(&k, 1, seed);
82 }
83
84 /**
85 * nfsd_file_key_hashfn - Compute the hash value of a lookup key
86 * @data: key on which to compute the hash value
87 * @len: rhash table's key_len parameter (unused)
88 * @seed: rhash table's random seed of the day
89 *
90 * Return value:
91 * Computed 32-bit hash value
92 */
nfsd_file_key_hashfn(const void * data,u32 len,u32 seed)93 static u32 nfsd_file_key_hashfn(const void *data, u32 len, u32 seed)
94 {
95 const struct nfsd_file_lookup_key *key = data;
96
97 return nfsd_file_inode_hash(key->inode, seed);
98 }
99
100 /**
101 * nfsd_file_obj_hashfn - Compute the hash value of an nfsd_file
102 * @data: object on which to compute the hash value
103 * @len: rhash table's key_len parameter (unused)
104 * @seed: rhash table's random seed of the day
105 *
106 * Return value:
107 * Computed 32-bit hash value
108 */
nfsd_file_obj_hashfn(const void * data,u32 len,u32 seed)109 static u32 nfsd_file_obj_hashfn(const void *data, u32 len, u32 seed)
110 {
111 const struct nfsd_file *nf = data;
112
113 return nfsd_file_inode_hash(nf->nf_inode, seed);
114 }
115
116 static bool
nfsd_match_cred(const struct cred * c1,const struct cred * c2)117 nfsd_match_cred(const struct cred *c1, const struct cred *c2)
118 {
119 int i;
120
121 if (!uid_eq(c1->fsuid, c2->fsuid))
122 return false;
123 if (!gid_eq(c1->fsgid, c2->fsgid))
124 return false;
125 if (c1->group_info == NULL || c2->group_info == NULL)
126 return c1->group_info == c2->group_info;
127 if (c1->group_info->ngroups != c2->group_info->ngroups)
128 return false;
129 for (i = 0; i < c1->group_info->ngroups; i++) {
130 if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
131 return false;
132 }
133 return true;
134 }
135
136 /**
137 * nfsd_file_obj_cmpfn - Match a cache item against search criteria
138 * @arg: search criteria
139 * @ptr: cache item to check
140 *
141 * Return values:
142 * %0 - Item matches search criteria
143 * %1 - Item does not match search criteria
144 */
nfsd_file_obj_cmpfn(struct rhashtable_compare_arg * arg,const void * ptr)145 static int nfsd_file_obj_cmpfn(struct rhashtable_compare_arg *arg,
146 const void *ptr)
147 {
148 const struct nfsd_file_lookup_key *key = arg->key;
149 const struct nfsd_file *nf = ptr;
150
151 switch (key->type) {
152 case NFSD_FILE_KEY_INODE:
153 if (nf->nf_inode != key->inode)
154 return 1;
155 break;
156 case NFSD_FILE_KEY_FULL:
157 if (nf->nf_inode != key->inode)
158 return 1;
159 if (nf->nf_may != key->need)
160 return 1;
161 if (nf->nf_net != key->net)
162 return 1;
163 if (!nfsd_match_cred(nf->nf_cred, key->cred))
164 return 1;
165 if (!!test_bit(NFSD_FILE_GC, &nf->nf_flags) != key->gc)
166 return 1;
167 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
168 return 1;
169 break;
170 }
171 return 0;
172 }
173
174 static const struct rhashtable_params nfsd_file_rhash_params = {
175 .key_len = sizeof_field(struct nfsd_file, nf_inode),
176 .key_offset = offsetof(struct nfsd_file, nf_inode),
177 .head_offset = offsetof(struct nfsd_file, nf_rhash),
178 .hashfn = nfsd_file_key_hashfn,
179 .obj_hashfn = nfsd_file_obj_hashfn,
180 .obj_cmpfn = nfsd_file_obj_cmpfn,
181 /* Reduce resizing churn on light workloads */
182 .min_size = 512, /* buckets */
183 .automatic_shrinking = true,
184 };
185
186 static void
nfsd_file_schedule_laundrette(void)187 nfsd_file_schedule_laundrette(void)
188 {
189 if ((atomic_read(&nfsd_file_rhash_tbl.nelems) == 0) ||
190 test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
191 return;
192
193 queue_delayed_work(system_wq, &nfsd_filecache_laundrette,
194 NFSD_LAUNDRETTE_DELAY);
195 }
196
197 static void
nfsd_file_slab_free(struct rcu_head * rcu)198 nfsd_file_slab_free(struct rcu_head *rcu)
199 {
200 struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
201
202 put_cred(nf->nf_cred);
203 kmem_cache_free(nfsd_file_slab, nf);
204 }
205
206 static void
nfsd_file_mark_free(struct fsnotify_mark * mark)207 nfsd_file_mark_free(struct fsnotify_mark *mark)
208 {
209 struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
210 nfm_mark);
211
212 kmem_cache_free(nfsd_file_mark_slab, nfm);
213 }
214
215 static struct nfsd_file_mark *
nfsd_file_mark_get(struct nfsd_file_mark * nfm)216 nfsd_file_mark_get(struct nfsd_file_mark *nfm)
217 {
218 if (!refcount_inc_not_zero(&nfm->nfm_ref))
219 return NULL;
220 return nfm;
221 }
222
223 static void
nfsd_file_mark_put(struct nfsd_file_mark * nfm)224 nfsd_file_mark_put(struct nfsd_file_mark *nfm)
225 {
226 if (refcount_dec_and_test(&nfm->nfm_ref)) {
227 fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
228 fsnotify_put_mark(&nfm->nfm_mark);
229 }
230 }
231
232 static struct nfsd_file_mark *
nfsd_file_mark_find_or_create(struct nfsd_file * nf,struct inode * inode)233 nfsd_file_mark_find_or_create(struct nfsd_file *nf, struct inode *inode)
234 {
235 int err;
236 struct fsnotify_mark *mark;
237 struct nfsd_file_mark *nfm = NULL, *new;
238
239 do {
240 fsnotify_group_lock(nfsd_file_fsnotify_group);
241 mark = fsnotify_find_mark(&inode->i_fsnotify_marks,
242 nfsd_file_fsnotify_group);
243 if (mark) {
244 nfm = nfsd_file_mark_get(container_of(mark,
245 struct nfsd_file_mark,
246 nfm_mark));
247 fsnotify_group_unlock(nfsd_file_fsnotify_group);
248 if (nfm) {
249 fsnotify_put_mark(mark);
250 break;
251 }
252 /* Avoid soft lockup race with nfsd_file_mark_put() */
253 fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
254 fsnotify_put_mark(mark);
255 } else {
256 fsnotify_group_unlock(nfsd_file_fsnotify_group);
257 }
258
259 /* allocate a new nfm */
260 new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
261 if (!new)
262 return NULL;
263 fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
264 new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
265 refcount_set(&new->nfm_ref, 1);
266
267 err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
268
269 /*
270 * If the add was successful, then return the object.
271 * Otherwise, we need to put the reference we hold on the
272 * nfm_mark. The fsnotify code will take a reference and put
273 * it on failure, so we can't just free it directly. It's also
274 * not safe to call fsnotify_destroy_mark on it as the
275 * mark->group will be NULL. Thus, we can't let the nfm_ref
276 * counter drive the destruction at this point.
277 */
278 if (likely(!err))
279 nfm = new;
280 else
281 fsnotify_put_mark(&new->nfm_mark);
282 } while (unlikely(err == -EEXIST));
283
284 return nfm;
285 }
286
287 static struct nfsd_file *
nfsd_file_alloc(struct nfsd_file_lookup_key * key,unsigned int may)288 nfsd_file_alloc(struct nfsd_file_lookup_key *key, unsigned int may)
289 {
290 struct nfsd_file *nf;
291
292 nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
293 if (nf) {
294 INIT_LIST_HEAD(&nf->nf_lru);
295 nf->nf_birthtime = ktime_get();
296 nf->nf_file = NULL;
297 nf->nf_cred = get_current_cred();
298 nf->nf_net = key->net;
299 nf->nf_flags = 0;
300 __set_bit(NFSD_FILE_HASHED, &nf->nf_flags);
301 __set_bit(NFSD_FILE_PENDING, &nf->nf_flags);
302 if (key->gc)
303 __set_bit(NFSD_FILE_GC, &nf->nf_flags);
304 nf->nf_inode = key->inode;
305 refcount_set(&nf->nf_ref, 1);
306 nf->nf_may = key->need;
307 nf->nf_mark = NULL;
308 }
309 return nf;
310 }
311
312 /**
313 * nfsd_file_check_write_error - check for writeback errors on a file
314 * @nf: nfsd_file to check for writeback errors
315 *
316 * Check whether a nfsd_file has an unseen error. Reset the write
317 * verifier if so.
318 */
319 static void
nfsd_file_check_write_error(struct nfsd_file * nf)320 nfsd_file_check_write_error(struct nfsd_file *nf)
321 {
322 struct file *file = nf->nf_file;
323
324 if ((file->f_mode & FMODE_WRITE) &&
325 filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err)))
326 nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
327 }
328
329 static void
nfsd_file_hash_remove(struct nfsd_file * nf)330 nfsd_file_hash_remove(struct nfsd_file *nf)
331 {
332 trace_nfsd_file_unhash(nf);
333 rhashtable_remove_fast(&nfsd_file_rhash_tbl, &nf->nf_rhash,
334 nfsd_file_rhash_params);
335 }
336
337 static bool
nfsd_file_unhash(struct nfsd_file * nf)338 nfsd_file_unhash(struct nfsd_file *nf)
339 {
340 if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
341 nfsd_file_hash_remove(nf);
342 return true;
343 }
344 return false;
345 }
346
347 static void
nfsd_file_free(struct nfsd_file * nf)348 nfsd_file_free(struct nfsd_file *nf)
349 {
350 s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
351
352 trace_nfsd_file_free(nf);
353
354 this_cpu_inc(nfsd_file_releases);
355 this_cpu_add(nfsd_file_total_age, age);
356
357 nfsd_file_unhash(nf);
358 if (nf->nf_mark)
359 nfsd_file_mark_put(nf->nf_mark);
360 if (nf->nf_file) {
361 get_file(nf->nf_file);
362 filp_close(nf->nf_file, NULL);
363 nfsd_file_check_write_error(nf);
364 fput(nf->nf_file);
365 }
366
367 /*
368 * If this item is still linked via nf_lru, that's a bug.
369 * WARN and leak it to preserve system stability.
370 */
371 if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
372 return;
373
374 call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
375 }
376
377 static bool
nfsd_file_check_writeback(struct nfsd_file * nf)378 nfsd_file_check_writeback(struct nfsd_file *nf)
379 {
380 struct file *file = nf->nf_file;
381 struct address_space *mapping;
382
383 if (!file || !(file->f_mode & FMODE_WRITE))
384 return false;
385 mapping = file->f_mapping;
386 return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
387 mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
388 }
389
nfsd_file_lru_add(struct nfsd_file * nf)390 static bool nfsd_file_lru_add(struct nfsd_file *nf)
391 {
392 set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
393 if (list_lru_add(&nfsd_file_lru, &nf->nf_lru)) {
394 trace_nfsd_file_lru_add(nf);
395 return true;
396 }
397 return false;
398 }
399
nfsd_file_lru_remove(struct nfsd_file * nf)400 static bool nfsd_file_lru_remove(struct nfsd_file *nf)
401 {
402 if (list_lru_del(&nfsd_file_lru, &nf->nf_lru)) {
403 trace_nfsd_file_lru_del(nf);
404 return true;
405 }
406 return false;
407 }
408
409 struct nfsd_file *
nfsd_file_get(struct nfsd_file * nf)410 nfsd_file_get(struct nfsd_file *nf)
411 {
412 if (likely(refcount_inc_not_zero(&nf->nf_ref)))
413 return nf;
414 return NULL;
415 }
416
417 /**
418 * nfsd_file_put - put the reference to a nfsd_file
419 * @nf: nfsd_file of which to put the reference
420 *
421 * Put a reference to a nfsd_file. In the non-GC case, we just put the
422 * reference immediately. In the GC case, if the reference would be
423 * the last one, the put it on the LRU instead to be cleaned up later.
424 */
425 void
nfsd_file_put(struct nfsd_file * nf)426 nfsd_file_put(struct nfsd_file *nf)
427 {
428 might_sleep();
429 trace_nfsd_file_put(nf);
430
431 if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
432 test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
433 /*
434 * If this is the last reference (nf_ref == 1), then try to
435 * transfer it to the LRU.
436 */
437 if (refcount_dec_not_one(&nf->nf_ref))
438 return;
439
440 /* Try to add it to the LRU. If that fails, decrement. */
441 if (nfsd_file_lru_add(nf)) {
442 /* If it's still hashed, we're done */
443 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
444 nfsd_file_schedule_laundrette();
445 return;
446 }
447
448 /*
449 * We're racing with unhashing, so try to remove it from
450 * the LRU. If removal fails, then someone else already
451 * has our reference.
452 */
453 if (!nfsd_file_lru_remove(nf))
454 return;
455 }
456 }
457 if (refcount_dec_and_test(&nf->nf_ref))
458 nfsd_file_free(nf);
459 }
460
461 static void
nfsd_file_dispose_list(struct list_head * dispose)462 nfsd_file_dispose_list(struct list_head *dispose)
463 {
464 struct nfsd_file *nf;
465
466 while (!list_empty(dispose)) {
467 nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
468 list_del_init(&nf->nf_lru);
469 nfsd_file_free(nf);
470 }
471 }
472
473 static void
nfsd_file_list_remove_disposal(struct list_head * dst,struct nfsd_fcache_disposal * l)474 nfsd_file_list_remove_disposal(struct list_head *dst,
475 struct nfsd_fcache_disposal *l)
476 {
477 spin_lock(&l->lock);
478 list_splice_init(&l->freeme, dst);
479 spin_unlock(&l->lock);
480 }
481
482 static void
nfsd_file_list_add_disposal(struct list_head * files,struct net * net)483 nfsd_file_list_add_disposal(struct list_head *files, struct net *net)
484 {
485 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
486 struct nfsd_fcache_disposal *l = nn->fcache_disposal;
487
488 spin_lock(&l->lock);
489 list_splice_tail_init(files, &l->freeme);
490 spin_unlock(&l->lock);
491 queue_work(nfsd_filecache_wq, &l->work);
492 }
493
494 static void
nfsd_file_list_add_pernet(struct list_head * dst,struct list_head * src,struct net * net)495 nfsd_file_list_add_pernet(struct list_head *dst, struct list_head *src,
496 struct net *net)
497 {
498 struct nfsd_file *nf, *tmp;
499
500 list_for_each_entry_safe(nf, tmp, src, nf_lru) {
501 if (nf->nf_net == net)
502 list_move_tail(&nf->nf_lru, dst);
503 }
504 }
505
506 static void
nfsd_file_dispose_list_delayed(struct list_head * dispose)507 nfsd_file_dispose_list_delayed(struct list_head *dispose)
508 {
509 LIST_HEAD(list);
510 struct nfsd_file *nf;
511
512 while(!list_empty(dispose)) {
513 nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
514 nfsd_file_list_add_pernet(&list, dispose, nf->nf_net);
515 nfsd_file_list_add_disposal(&list, nf->nf_net);
516 }
517 }
518
519 /**
520 * nfsd_file_lru_cb - Examine an entry on the LRU list
521 * @item: LRU entry to examine
522 * @lru: controlling LRU
523 * @lock: LRU list lock (unused)
524 * @arg: dispose list
525 *
526 * Return values:
527 * %LRU_REMOVED: @item was removed from the LRU
528 * %LRU_ROTATE: @item is to be moved to the LRU tail
529 * %LRU_SKIP: @item cannot be evicted
530 */
531 static enum lru_status
nfsd_file_lru_cb(struct list_head * item,struct list_lru_one * lru,spinlock_t * lock,void * arg)532 nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
533 spinlock_t *lock, void *arg)
534 __releases(lock)
535 __acquires(lock)
536 {
537 struct list_head *head = arg;
538 struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
539
540 /* We should only be dealing with GC entries here */
541 WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));
542
543 /*
544 * Don't throw out files that are still undergoing I/O or
545 * that have uncleared errors pending.
546 */
547 if (nfsd_file_check_writeback(nf)) {
548 trace_nfsd_file_gc_writeback(nf);
549 return LRU_SKIP;
550 }
551
552 /* If it was recently added to the list, skip it */
553 if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
554 trace_nfsd_file_gc_referenced(nf);
555 return LRU_ROTATE;
556 }
557
558 /*
559 * Put the reference held on behalf of the LRU. If it wasn't the last
560 * one, then just remove it from the LRU and ignore it.
561 */
562 if (!refcount_dec_and_test(&nf->nf_ref)) {
563 trace_nfsd_file_gc_in_use(nf);
564 list_lru_isolate(lru, &nf->nf_lru);
565 return LRU_REMOVED;
566 }
567
568 /* Refcount went to zero. Unhash it and queue it to the dispose list */
569 nfsd_file_unhash(nf);
570 list_lru_isolate_move(lru, &nf->nf_lru, head);
571 this_cpu_inc(nfsd_file_evictions);
572 trace_nfsd_file_gc_disposed(nf);
573 return LRU_REMOVED;
574 }
575
576 static void
nfsd_file_gc(void)577 nfsd_file_gc(void)
578 {
579 LIST_HEAD(dispose);
580 unsigned long ret;
581
582 ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb,
583 &dispose, list_lru_count(&nfsd_file_lru));
584 trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
585 nfsd_file_dispose_list_delayed(&dispose);
586 }
587
588 static void
nfsd_file_gc_worker(struct work_struct * work)589 nfsd_file_gc_worker(struct work_struct *work)
590 {
591 nfsd_file_gc();
592 nfsd_file_schedule_laundrette();
593 }
594
595 static unsigned long
nfsd_file_lru_count(struct shrinker * s,struct shrink_control * sc)596 nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
597 {
598 return list_lru_count(&nfsd_file_lru);
599 }
600
601 static unsigned long
nfsd_file_lru_scan(struct shrinker * s,struct shrink_control * sc)602 nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
603 {
604 LIST_HEAD(dispose);
605 unsigned long ret;
606
607 ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
608 nfsd_file_lru_cb, &dispose);
609 trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
610 nfsd_file_dispose_list_delayed(&dispose);
611 return ret;
612 }
613
614 static struct shrinker nfsd_file_shrinker = {
615 .scan_objects = nfsd_file_lru_scan,
616 .count_objects = nfsd_file_lru_count,
617 .seeks = 1,
618 };
619
620 /**
621 * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file
622 * @nf: nfsd_file to attempt to queue
623 * @dispose: private list to queue successfully-put objects
624 *
625 * Unhash an nfsd_file, try to get a reference to it, and then put that
626 * reference. If it's the last reference, queue it to the dispose list.
627 */
628 static void
nfsd_file_cond_queue(struct nfsd_file * nf,struct list_head * dispose)629 nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose)
630 __must_hold(RCU)
631 {
632 int decrement = 1;
633
634 /* If we raced with someone else unhashing, ignore it */
635 if (!nfsd_file_unhash(nf))
636 return;
637
638 /* If we can't get a reference, ignore it */
639 if (!nfsd_file_get(nf))
640 return;
641
642 /* Extra decrement if we remove from the LRU */
643 if (nfsd_file_lru_remove(nf))
644 ++decrement;
645
646 /* If refcount goes to 0, then put on the dispose list */
647 if (refcount_sub_and_test(decrement, &nf->nf_ref)) {
648 list_add(&nf->nf_lru, dispose);
649 trace_nfsd_file_closing(nf);
650 }
651 }
652
653 /**
654 * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode
655 * @inode: inode on which to close out nfsd_files
656 * @dispose: list on which to gather nfsd_files to close out
657 *
658 * An nfsd_file represents a struct file being held open on behalf of nfsd. An
659 * open file however can block other activity (such as leases), or cause
660 * undesirable behavior (e.g. spurious silly-renames when reexporting NFS).
661 *
662 * This function is intended to find open nfsd_files when this sort of
663 * conflicting access occurs and then attempt to close those files out.
664 *
665 * Populates the dispose list with entries that have already had their
666 * refcounts go to zero. The actual free of an nfsd_file can be expensive,
667 * so we leave it up to the caller whether it wants to wait or not.
668 */
669 static void
nfsd_file_queue_for_close(struct inode * inode,struct list_head * dispose)670 nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose)
671 {
672 struct nfsd_file_lookup_key key = {
673 .type = NFSD_FILE_KEY_INODE,
674 .inode = inode,
675 };
676 struct nfsd_file *nf;
677
678 rcu_read_lock();
679 do {
680 nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
681 nfsd_file_rhash_params);
682 if (!nf)
683 break;
684 nfsd_file_cond_queue(nf, dispose);
685 } while (1);
686 rcu_read_unlock();
687 }
688
689 /**
690 * nfsd_file_close_inode - attempt a delayed close of a nfsd_file
691 * @inode: inode of the file to attempt to remove
692 *
693 * Close out any open nfsd_files that can be reaped for @inode. The
694 * actual freeing is deferred to the dispose_list_delayed infrastructure.
695 *
696 * This is used by the fsnotify callbacks and setlease notifier.
697 */
698 static void
nfsd_file_close_inode(struct inode * inode)699 nfsd_file_close_inode(struct inode *inode)
700 {
701 LIST_HEAD(dispose);
702
703 nfsd_file_queue_for_close(inode, &dispose);
704 nfsd_file_dispose_list_delayed(&dispose);
705 }
706
707 /**
708 * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
709 * @inode: inode of the file to attempt to remove
710 *
711 * Close out any open nfsd_files that can be reaped for @inode. The
712 * nfsd_files are closed out synchronously.
713 *
714 * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames
715 * when reexporting NFS.
716 */
717 void
nfsd_file_close_inode_sync(struct inode * inode)718 nfsd_file_close_inode_sync(struct inode *inode)
719 {
720 struct nfsd_file *nf;
721 LIST_HEAD(dispose);
722
723 trace_nfsd_file_close(inode);
724
725 nfsd_file_queue_for_close(inode, &dispose);
726 while (!list_empty(&dispose)) {
727 nf = list_first_entry(&dispose, struct nfsd_file, nf_lru);
728 list_del_init(&nf->nf_lru);
729 nfsd_file_free(nf);
730 }
731 flush_delayed_fput();
732 }
733
734 /**
735 * nfsd_file_delayed_close - close unused nfsd_files
736 * @work: dummy
737 *
738 * Walk the LRU list and destroy any entries that have not been used since
739 * the last scan.
740 */
741 static void
nfsd_file_delayed_close(struct work_struct * work)742 nfsd_file_delayed_close(struct work_struct *work)
743 {
744 LIST_HEAD(head);
745 struct nfsd_fcache_disposal *l = container_of(work,
746 struct nfsd_fcache_disposal, work);
747
748 nfsd_file_list_remove_disposal(&head, l);
749 nfsd_file_dispose_list(&head);
750 }
751
752 static int
nfsd_file_lease_notifier_call(struct notifier_block * nb,unsigned long arg,void * data)753 nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
754 void *data)
755 {
756 struct file_lock *fl = data;
757
758 /* Only close files for F_SETLEASE leases */
759 if (fl->fl_flags & FL_LEASE)
760 nfsd_file_close_inode(file_inode(fl->fl_file));
761 return 0;
762 }
763
764 static struct notifier_block nfsd_file_lease_notifier = {
765 .notifier_call = nfsd_file_lease_notifier_call,
766 };
767
768 static int
nfsd_file_fsnotify_handle_event(struct fsnotify_mark * mark,u32 mask,struct inode * inode,struct inode * dir,const struct qstr * name,u32 cookie)769 nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
770 struct inode *inode, struct inode *dir,
771 const struct qstr *name, u32 cookie)
772 {
773 if (WARN_ON_ONCE(!inode))
774 return 0;
775
776 trace_nfsd_file_fsnotify_handle_event(inode, mask);
777
778 /* Should be no marks on non-regular files */
779 if (!S_ISREG(inode->i_mode)) {
780 WARN_ON_ONCE(1);
781 return 0;
782 }
783
784 /* don't close files if this was not the last link */
785 if (mask & FS_ATTRIB) {
786 if (inode->i_nlink)
787 return 0;
788 }
789
790 nfsd_file_close_inode(inode);
791 return 0;
792 }
793
794
795 static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
796 .handle_inode_event = nfsd_file_fsnotify_handle_event,
797 .free_mark = nfsd_file_mark_free,
798 };
799
800 int
nfsd_file_cache_init(void)801 nfsd_file_cache_init(void)
802 {
803 int ret;
804
805 lockdep_assert_held(&nfsd_mutex);
806 if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
807 return 0;
808
809 ret = rhashtable_init(&nfsd_file_rhash_tbl, &nfsd_file_rhash_params);
810 if (ret)
811 return ret;
812
813 ret = -ENOMEM;
814 nfsd_filecache_wq = alloc_workqueue("nfsd_filecache", 0, 0);
815 if (!nfsd_filecache_wq)
816 goto out;
817
818 nfsd_file_slab = kmem_cache_create("nfsd_file",
819 sizeof(struct nfsd_file), 0, 0, NULL);
820 if (!nfsd_file_slab) {
821 pr_err("nfsd: unable to create nfsd_file_slab\n");
822 goto out_err;
823 }
824
825 nfsd_file_mark_slab = kmem_cache_create("nfsd_file_mark",
826 sizeof(struct nfsd_file_mark), 0, 0, NULL);
827 if (!nfsd_file_mark_slab) {
828 pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
829 goto out_err;
830 }
831
832
833 ret = list_lru_init(&nfsd_file_lru);
834 if (ret) {
835 pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
836 goto out_err;
837 }
838
839 ret = register_shrinker(&nfsd_file_shrinker, "nfsd-filecache");
840 if (ret) {
841 pr_err("nfsd: failed to register nfsd_file_shrinker: %d\n", ret);
842 goto out_lru;
843 }
844
845 ret = lease_register_notifier(&nfsd_file_lease_notifier);
846 if (ret) {
847 pr_err("nfsd: unable to register lease notifier: %d\n", ret);
848 goto out_shrinker;
849 }
850
851 nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
852 FSNOTIFY_GROUP_NOFS);
853 if (IS_ERR(nfsd_file_fsnotify_group)) {
854 pr_err("nfsd: unable to create fsnotify group: %ld\n",
855 PTR_ERR(nfsd_file_fsnotify_group));
856 ret = PTR_ERR(nfsd_file_fsnotify_group);
857 nfsd_file_fsnotify_group = NULL;
858 goto out_notifier;
859 }
860
861 INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
862 out:
863 return ret;
864 out_notifier:
865 lease_unregister_notifier(&nfsd_file_lease_notifier);
866 out_shrinker:
867 unregister_shrinker(&nfsd_file_shrinker);
868 out_lru:
869 list_lru_destroy(&nfsd_file_lru);
870 out_err:
871 kmem_cache_destroy(nfsd_file_slab);
872 nfsd_file_slab = NULL;
873 kmem_cache_destroy(nfsd_file_mark_slab);
874 nfsd_file_mark_slab = NULL;
875 destroy_workqueue(nfsd_filecache_wq);
876 nfsd_filecache_wq = NULL;
877 rhashtable_destroy(&nfsd_file_rhash_tbl);
878 goto out;
879 }
880
881 /**
882 * __nfsd_file_cache_purge: clean out the cache for shutdown
883 * @net: net-namespace to shut down the cache (may be NULL)
884 *
885 * Walk the nfsd_file cache and close out any that match @net. If @net is NULL,
886 * then close out everything. Called when an nfsd instance is being shut down.
887 */
888 static void
__nfsd_file_cache_purge(struct net * net)889 __nfsd_file_cache_purge(struct net *net)
890 {
891 struct rhashtable_iter iter;
892 struct nfsd_file *nf;
893 LIST_HEAD(dispose);
894
895 rhashtable_walk_enter(&nfsd_file_rhash_tbl, &iter);
896 do {
897 rhashtable_walk_start(&iter);
898
899 nf = rhashtable_walk_next(&iter);
900 while (!IS_ERR_OR_NULL(nf)) {
901 if (!net || nf->nf_net == net)
902 nfsd_file_cond_queue(nf, &dispose);
903 nf = rhashtable_walk_next(&iter);
904 }
905
906 rhashtable_walk_stop(&iter);
907 } while (nf == ERR_PTR(-EAGAIN));
908 rhashtable_walk_exit(&iter);
909
910 nfsd_file_dispose_list(&dispose);
911 }
912
913 static struct nfsd_fcache_disposal *
nfsd_alloc_fcache_disposal(void)914 nfsd_alloc_fcache_disposal(void)
915 {
916 struct nfsd_fcache_disposal *l;
917
918 l = kmalloc(sizeof(*l), GFP_KERNEL);
919 if (!l)
920 return NULL;
921 INIT_WORK(&l->work, nfsd_file_delayed_close);
922 spin_lock_init(&l->lock);
923 INIT_LIST_HEAD(&l->freeme);
924 return l;
925 }
926
927 static void
nfsd_free_fcache_disposal(struct nfsd_fcache_disposal * l)928 nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
929 {
930 cancel_work_sync(&l->work);
931 nfsd_file_dispose_list(&l->freeme);
932 kfree(l);
933 }
934
935 static void
nfsd_free_fcache_disposal_net(struct net * net)936 nfsd_free_fcache_disposal_net(struct net *net)
937 {
938 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
939 struct nfsd_fcache_disposal *l = nn->fcache_disposal;
940
941 nfsd_free_fcache_disposal(l);
942 }
943
944 int
nfsd_file_cache_start_net(struct net * net)945 nfsd_file_cache_start_net(struct net *net)
946 {
947 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
948
949 nn->fcache_disposal = nfsd_alloc_fcache_disposal();
950 return nn->fcache_disposal ? 0 : -ENOMEM;
951 }
952
953 /**
954 * nfsd_file_cache_purge - Remove all cache items associated with @net
955 * @net: target net namespace
956 *
957 */
958 void
nfsd_file_cache_purge(struct net * net)959 nfsd_file_cache_purge(struct net *net)
960 {
961 lockdep_assert_held(&nfsd_mutex);
962 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
963 __nfsd_file_cache_purge(net);
964 }
965
966 void
nfsd_file_cache_shutdown_net(struct net * net)967 nfsd_file_cache_shutdown_net(struct net *net)
968 {
969 nfsd_file_cache_purge(net);
970 nfsd_free_fcache_disposal_net(net);
971 }
972
973 void
nfsd_file_cache_shutdown(void)974 nfsd_file_cache_shutdown(void)
975 {
976 int i;
977
978 lockdep_assert_held(&nfsd_mutex);
979 if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
980 return;
981
982 lease_unregister_notifier(&nfsd_file_lease_notifier);
983 unregister_shrinker(&nfsd_file_shrinker);
984 /*
985 * make sure all callers of nfsd_file_lru_cb are done before
986 * calling nfsd_file_cache_purge
987 */
988 cancel_delayed_work_sync(&nfsd_filecache_laundrette);
989 __nfsd_file_cache_purge(NULL);
990 list_lru_destroy(&nfsd_file_lru);
991 rcu_barrier();
992 fsnotify_put_group(nfsd_file_fsnotify_group);
993 nfsd_file_fsnotify_group = NULL;
994 kmem_cache_destroy(nfsd_file_slab);
995 nfsd_file_slab = NULL;
996 fsnotify_wait_marks_destroyed();
997 kmem_cache_destroy(nfsd_file_mark_slab);
998 nfsd_file_mark_slab = NULL;
999 destroy_workqueue(nfsd_filecache_wq);
1000 nfsd_filecache_wq = NULL;
1001 rhashtable_destroy(&nfsd_file_rhash_tbl);
1002
1003 for_each_possible_cpu(i) {
1004 per_cpu(nfsd_file_cache_hits, i) = 0;
1005 per_cpu(nfsd_file_acquisitions, i) = 0;
1006 per_cpu(nfsd_file_releases, i) = 0;
1007 per_cpu(nfsd_file_total_age, i) = 0;
1008 per_cpu(nfsd_file_evictions, i) = 0;
1009 }
1010 }
1011
1012 /**
1013 * nfsd_file_is_cached - are there any cached open files for this inode?
1014 * @inode: inode to check
1015 *
1016 * The lookup matches inodes in all net namespaces and is atomic wrt
1017 * nfsd_file_acquire().
1018 *
1019 * Return values:
1020 * %true: filecache contains at least one file matching this inode
1021 * %false: filecache contains no files matching this inode
1022 */
1023 bool
nfsd_file_is_cached(struct inode * inode)1024 nfsd_file_is_cached(struct inode *inode)
1025 {
1026 struct nfsd_file_lookup_key key = {
1027 .type = NFSD_FILE_KEY_INODE,
1028 .inode = inode,
1029 };
1030 bool ret = false;
1031
1032 if (rhashtable_lookup_fast(&nfsd_file_rhash_tbl, &key,
1033 nfsd_file_rhash_params) != NULL)
1034 ret = true;
1035 trace_nfsd_file_is_cached(inode, (int)ret);
1036 return ret;
1037 }
1038
1039 static __be32
nfsd_file_do_acquire(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf,bool want_gc)1040 nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1041 unsigned int may_flags, struct file *file,
1042 struct nfsd_file **pnf, bool want_gc)
1043 {
1044 struct nfsd_file_lookup_key key = {
1045 .type = NFSD_FILE_KEY_FULL,
1046 .need = may_flags & NFSD_FILE_MAY_MASK,
1047 .net = SVC_NET(rqstp),
1048 .gc = want_gc,
1049 };
1050 bool open_retry = true;
1051 struct nfsd_file *nf;
1052 __be32 status;
1053 int ret;
1054
1055 status = fh_verify(rqstp, fhp, S_IFREG,
1056 may_flags|NFSD_MAY_OWNER_OVERRIDE);
1057 if (status != nfs_ok)
1058 return status;
1059 key.inode = d_inode(fhp->fh_dentry);
1060 key.cred = get_current_cred();
1061
1062 retry:
1063 rcu_read_lock();
1064 nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
1065 nfsd_file_rhash_params);
1066 if (nf)
1067 nf = nfsd_file_get(nf);
1068 rcu_read_unlock();
1069
1070 if (nf) {
1071 if (nfsd_file_lru_remove(nf))
1072 WARN_ON_ONCE(refcount_dec_and_test(&nf->nf_ref));
1073 goto wait_for_construction;
1074 }
1075
1076 nf = nfsd_file_alloc(&key, may_flags);
1077 if (!nf) {
1078 status = nfserr_jukebox;
1079 goto out_status;
1080 }
1081
1082 ret = rhashtable_lookup_insert_key(&nfsd_file_rhash_tbl,
1083 &key, &nf->nf_rhash,
1084 nfsd_file_rhash_params);
1085 if (likely(ret == 0))
1086 goto open_file;
1087
1088 nfsd_file_slab_free(&nf->nf_rcu);
1089 nf = NULL;
1090 if (ret == -EEXIST)
1091 goto retry;
1092 trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, ret);
1093 status = nfserr_jukebox;
1094 goto out_status;
1095
1096 wait_for_construction:
1097 wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
1098
1099 /* Did construction of this file fail? */
1100 if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
1101 trace_nfsd_file_cons_err(rqstp, key.inode, may_flags, nf);
1102 if (!open_retry) {
1103 status = nfserr_jukebox;
1104 goto out;
1105 }
1106 open_retry = false;
1107 if (refcount_dec_and_test(&nf->nf_ref))
1108 nfsd_file_free(nf);
1109 goto retry;
1110 }
1111
1112 this_cpu_inc(nfsd_file_cache_hits);
1113
1114 status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
1115 out:
1116 if (status == nfs_ok) {
1117 this_cpu_inc(nfsd_file_acquisitions);
1118 nfsd_file_check_write_error(nf);
1119 *pnf = nf;
1120 } else {
1121 if (refcount_dec_and_test(&nf->nf_ref))
1122 nfsd_file_free(nf);
1123 nf = NULL;
1124 }
1125
1126 out_status:
1127 put_cred(key.cred);
1128 trace_nfsd_file_acquire(rqstp, key.inode, may_flags, nf, status);
1129 return status;
1130
1131 open_file:
1132 trace_nfsd_file_alloc(nf);
1133 nf->nf_mark = nfsd_file_mark_find_or_create(nf, key.inode);
1134 if (nf->nf_mark) {
1135 if (file) {
1136 get_file(file);
1137 nf->nf_file = file;
1138 status = nfs_ok;
1139 trace_nfsd_file_opened(nf, status);
1140 } else {
1141 status = nfsd_open_verified(rqstp, fhp, may_flags,
1142 &nf->nf_file);
1143 trace_nfsd_file_open(nf, status);
1144 }
1145 } else
1146 status = nfserr_jukebox;
1147 /*
1148 * If construction failed, or we raced with a call to unlink()
1149 * then unhash.
1150 */
1151 if (status == nfs_ok && key.inode->i_nlink == 0)
1152 status = nfserr_jukebox;
1153 if (status != nfs_ok)
1154 nfsd_file_unhash(nf);
1155 clear_bit_unlock(NFSD_FILE_PENDING, &nf->nf_flags);
1156 smp_mb__after_atomic();
1157 wake_up_bit(&nf->nf_flags, NFSD_FILE_PENDING);
1158 goto out;
1159 }
1160
1161 /**
1162 * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file
1163 * @rqstp: the RPC transaction being executed
1164 * @fhp: the NFS filehandle of the file to be opened
1165 * @may_flags: NFSD_MAY_ settings for the file
1166 * @pnf: OUT: new or found "struct nfsd_file" object
1167 *
1168 * The nfsd_file object returned by this API is reference-counted
1169 * and garbage-collected. The object is retained for a few
1170 * seconds after the final nfsd_file_put() in case the caller
1171 * wants to re-use it.
1172 *
1173 * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1174 * network byte order is returned.
1175 */
1176 __be32
nfsd_file_acquire_gc(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1177 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp,
1178 unsigned int may_flags, struct nfsd_file **pnf)
1179 {
1180 return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, true);
1181 }
1182
1183 /**
1184 * nfsd_file_acquire - Get a struct nfsd_file with an open file
1185 * @rqstp: the RPC transaction being executed
1186 * @fhp: the NFS filehandle of the file to be opened
1187 * @may_flags: NFSD_MAY_ settings for the file
1188 * @pnf: OUT: new or found "struct nfsd_file" object
1189 *
1190 * The nfsd_file_object returned by this API is reference-counted
1191 * but not garbage-collected. The object is unhashed after the
1192 * final nfsd_file_put().
1193 *
1194 * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1195 * network byte order is returned.
1196 */
1197 __be32
nfsd_file_acquire(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct nfsd_file ** pnf)1198 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1199 unsigned int may_flags, struct nfsd_file **pnf)
1200 {
1201 return nfsd_file_do_acquire(rqstp, fhp, may_flags, NULL, pnf, false);
1202 }
1203
1204 /**
1205 * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file
1206 * @rqstp: the RPC transaction being executed
1207 * @fhp: the NFS filehandle of the file just created
1208 * @may_flags: NFSD_MAY_ settings for the file
1209 * @file: cached, already-open file (may be NULL)
1210 * @pnf: OUT: new or found "struct nfsd_file" object
1211 *
1212 * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist,
1213 * and @file is non-NULL, use it to instantiate a new nfsd_file instead of
1214 * opening a new one.
1215 *
1216 * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1217 * network byte order is returned.
1218 */
1219 __be32
nfsd_file_acquire_opened(struct svc_rqst * rqstp,struct svc_fh * fhp,unsigned int may_flags,struct file * file,struct nfsd_file ** pnf)1220 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp,
1221 unsigned int may_flags, struct file *file,
1222 struct nfsd_file **pnf)
1223 {
1224 return nfsd_file_do_acquire(rqstp, fhp, may_flags, file, pnf, false);
1225 }
1226
1227 /*
1228 * Note that fields may be added, removed or reordered in the future. Programs
1229 * scraping this file for info should test the labels to ensure they're
1230 * getting the correct field.
1231 */
nfsd_file_cache_stats_show(struct seq_file * m,void * v)1232 int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
1233 {
1234 unsigned long releases = 0, evictions = 0;
1235 unsigned long hits = 0, acquisitions = 0;
1236 unsigned int i, count = 0, buckets = 0;
1237 unsigned long lru = 0, total_age = 0;
1238
1239 /* Serialize with server shutdown */
1240 mutex_lock(&nfsd_mutex);
1241 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
1242 struct bucket_table *tbl;
1243 struct rhashtable *ht;
1244
1245 lru = list_lru_count(&nfsd_file_lru);
1246
1247 rcu_read_lock();
1248 ht = &nfsd_file_rhash_tbl;
1249 count = atomic_read(&ht->nelems);
1250 tbl = rht_dereference_rcu(ht->tbl, ht);
1251 buckets = tbl->size;
1252 rcu_read_unlock();
1253 }
1254 mutex_unlock(&nfsd_mutex);
1255
1256 for_each_possible_cpu(i) {
1257 hits += per_cpu(nfsd_file_cache_hits, i);
1258 acquisitions += per_cpu(nfsd_file_acquisitions, i);
1259 releases += per_cpu(nfsd_file_releases, i);
1260 total_age += per_cpu(nfsd_file_total_age, i);
1261 evictions += per_cpu(nfsd_file_evictions, i);
1262 }
1263
1264 seq_printf(m, "total entries: %u\n", count);
1265 seq_printf(m, "hash buckets: %u\n", buckets);
1266 seq_printf(m, "lru entries: %lu\n", lru);
1267 seq_printf(m, "cache hits: %lu\n", hits);
1268 seq_printf(m, "acquisitions: %lu\n", acquisitions);
1269 seq_printf(m, "releases: %lu\n", releases);
1270 seq_printf(m, "evictions: %lu\n", evictions);
1271 if (releases)
1272 seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
1273 else
1274 seq_printf(m, "mean age (ms): -\n");
1275 return 0;
1276 }
1277