1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * fs/dcache.c
4 *
5 * Complete reimplementation
6 * (C) 1997 Thomas Schoebel-Theuer,
7 * with heavy changes by Linus Torvalds
8 */
9
10 /*
11 * Notes on the allocation strategy:
12 *
13 * The dcache is a master of the icache - whenever a dcache entry
14 * exists, the inode will always exist. "iput()" is done either when
15 * the dcache entry is deleted or garbage collected.
16 */
17
18 #include <linux/ratelimit.h>
19 #include <linux/string.h>
20 #include <linux/mm.h>
21 #include <linux/fs.h>
22 #include <linux/fscrypt.h>
23 #include <linux/fsnotify.h>
24 #include <linux/slab.h>
25 #include <linux/init.h>
26 #include <linux/hash.h>
27 #include <linux/cache.h>
28 #include <linux/export.h>
29 #include <linux/security.h>
30 #include <linux/seqlock.h>
31 #include <linux/memblock.h>
32 #include <linux/bit_spinlock.h>
33 #include <linux/rculist_bl.h>
34 #include <linux/list_lru.h>
35 #include "internal.h"
36 #include "mount.h"
37
38 /*
39 * Usage:
40 * dcache->d_inode->i_lock protects:
41 * - i_dentry, d_u.d_alias, d_inode of aliases
42 * dcache_hash_bucket lock protects:
43 * - the dcache hash table
44 * s_roots bl list spinlock protects:
45 * - the s_roots list (see __d_drop)
46 * dentry->d_sb->s_dentry_lru_lock protects:
47 * - the dcache lru lists and counters
48 * d_lock protects:
49 * - d_flags
50 * - d_name
51 * - d_lru
52 * - d_count
53 * - d_unhashed()
54 * - d_parent and d_subdirs
55 * - childrens' d_child and d_parent
56 * - d_u.d_alias, d_inode
57 *
58 * Ordering:
59 * dentry->d_inode->i_lock
60 * dentry->d_lock
61 * dentry->d_sb->s_dentry_lru_lock
62 * dcache_hash_bucket lock
63 * s_roots lock
64 *
65 * If there is an ancestor relationship:
66 * dentry->d_parent->...->d_parent->d_lock
67 * ...
68 * dentry->d_parent->d_lock
69 * dentry->d_lock
70 *
71 * If no ancestor relationship:
72 * arbitrary, since it's serialized on rename_lock
73 */
74 int sysctl_vfs_cache_pressure __read_mostly = 100;
75 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
76
77 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
78
79 EXPORT_SYMBOL(rename_lock);
80
81 static struct kmem_cache *dentry_cache __read_mostly;
82
83 const struct qstr empty_name = QSTR_INIT("", 0);
84 EXPORT_SYMBOL(empty_name);
85 const struct qstr slash_name = QSTR_INIT("/", 1);
86 EXPORT_SYMBOL(slash_name);
87 const struct qstr dotdot_name = QSTR_INIT("..", 2);
88 EXPORT_SYMBOL(dotdot_name);
89
90 /*
91 * This is the single most critical data structure when it comes
92 * to the dcache: the hashtable for lookups. Somebody should try
93 * to make this good - I've just made it work.
94 *
95 * This hash-function tries to avoid losing too many bits of hash
96 * information, yet avoid using a prime hash-size or similar.
97 */
98
99 static unsigned int d_hash_shift __read_mostly;
100
101 static struct hlist_bl_head *dentry_hashtable __read_mostly;
102
d_hash(unsigned int hash)103 static inline struct hlist_bl_head *d_hash(unsigned int hash)
104 {
105 return dentry_hashtable + (hash >> d_hash_shift);
106 }
107
108 #define IN_LOOKUP_SHIFT 10
109 static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
110
in_lookup_hash(const struct dentry * parent,unsigned int hash)111 static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
112 unsigned int hash)
113 {
114 hash += (unsigned long) parent / L1_CACHE_BYTES;
115 return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
116 }
117
118 struct dentry_stat_t {
119 long nr_dentry;
120 long nr_unused;
121 long age_limit; /* age in seconds */
122 long want_pages; /* pages requested by system */
123 long nr_negative; /* # of unused negative dentries */
124 long dummy; /* Reserved for future use */
125 };
126
127 static DEFINE_PER_CPU(long, nr_dentry);
128 static DEFINE_PER_CPU(long, nr_dentry_unused);
129 static DEFINE_PER_CPU(long, nr_dentry_negative);
130
131 #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
132 /* Statistics gathering. */
133 static struct dentry_stat_t dentry_stat = {
134 .age_limit = 45,
135 };
136
137 /*
138 * Here we resort to our own counters instead of using generic per-cpu counters
139 * for consistency with what the vfs inode code does. We are expected to harvest
140 * better code and performance by having our own specialized counters.
141 *
142 * Please note that the loop is done over all possible CPUs, not over all online
143 * CPUs. The reason for this is that we don't want to play games with CPUs going
144 * on and off. If one of them goes off, we will just keep their counters.
145 *
146 * glommer: See cffbc8a for details, and if you ever intend to change this,
147 * please update all vfs counters to match.
148 */
get_nr_dentry(void)149 static long get_nr_dentry(void)
150 {
151 int i;
152 long sum = 0;
153 for_each_possible_cpu(i)
154 sum += per_cpu(nr_dentry, i);
155 return sum < 0 ? 0 : sum;
156 }
157
get_nr_dentry_unused(void)158 static long get_nr_dentry_unused(void)
159 {
160 int i;
161 long sum = 0;
162 for_each_possible_cpu(i)
163 sum += per_cpu(nr_dentry_unused, i);
164 return sum < 0 ? 0 : sum;
165 }
166
get_nr_dentry_negative(void)167 static long get_nr_dentry_negative(void)
168 {
169 int i;
170 long sum = 0;
171
172 for_each_possible_cpu(i)
173 sum += per_cpu(nr_dentry_negative, i);
174 return sum < 0 ? 0 : sum;
175 }
176
proc_nr_dentry(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)177 static int proc_nr_dentry(struct ctl_table *table, int write, void *buffer,
178 size_t *lenp, loff_t *ppos)
179 {
180 dentry_stat.nr_dentry = get_nr_dentry();
181 dentry_stat.nr_unused = get_nr_dentry_unused();
182 dentry_stat.nr_negative = get_nr_dentry_negative();
183 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
184 }
185
186 static struct ctl_table fs_dcache_sysctls[] = {
187 {
188 .procname = "dentry-state",
189 .data = &dentry_stat,
190 .maxlen = 6*sizeof(long),
191 .mode = 0444,
192 .proc_handler = proc_nr_dentry,
193 },
194 { }
195 };
196
init_fs_dcache_sysctls(void)197 static int __init init_fs_dcache_sysctls(void)
198 {
199 register_sysctl_init("fs", fs_dcache_sysctls);
200 return 0;
201 }
202 fs_initcall(init_fs_dcache_sysctls);
203 #endif
204
205 /*
206 * Compare 2 name strings, return 0 if they match, otherwise non-zero.
207 * The strings are both count bytes long, and count is non-zero.
208 */
209 #ifdef CONFIG_DCACHE_WORD_ACCESS
210
211 #include <asm/word-at-a-time.h>
212 /*
213 * NOTE! 'cs' and 'scount' come from a dentry, so it has a
214 * aligned allocation for this particular component. We don't
215 * strictly need the load_unaligned_zeropad() safety, but it
216 * doesn't hurt either.
217 *
218 * In contrast, 'ct' and 'tcount' can be from a pathname, and do
219 * need the careful unaligned handling.
220 */
dentry_string_cmp(const unsigned char * cs,const unsigned char * ct,unsigned tcount)221 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
222 {
223 unsigned long a,b,mask;
224
225 for (;;) {
226 a = read_word_at_a_time(cs);
227 b = load_unaligned_zeropad(ct);
228 if (tcount < sizeof(unsigned long))
229 break;
230 if (unlikely(a != b))
231 return 1;
232 cs += sizeof(unsigned long);
233 ct += sizeof(unsigned long);
234 tcount -= sizeof(unsigned long);
235 if (!tcount)
236 return 0;
237 }
238 mask = bytemask_from_count(tcount);
239 return unlikely(!!((a ^ b) & mask));
240 }
241
242 #else
243
dentry_string_cmp(const unsigned char * cs,const unsigned char * ct,unsigned tcount)244 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
245 {
246 do {
247 if (*cs != *ct)
248 return 1;
249 cs++;
250 ct++;
251 tcount--;
252 } while (tcount);
253 return 0;
254 }
255
256 #endif
257
dentry_cmp(const struct dentry * dentry,const unsigned char * ct,unsigned tcount)258 static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
259 {
260 /*
261 * Be careful about RCU walk racing with rename:
262 * use 'READ_ONCE' to fetch the name pointer.
263 *
264 * NOTE! Even if a rename will mean that the length
265 * was not loaded atomically, we don't care. The
266 * RCU walk will check the sequence count eventually,
267 * and catch it. And we won't overrun the buffer,
268 * because we're reading the name pointer atomically,
269 * and a dentry name is guaranteed to be properly
270 * terminated with a NUL byte.
271 *
272 * End result: even if 'len' is wrong, we'll exit
273 * early because the data cannot match (there can
274 * be no NUL in the ct/tcount data)
275 */
276 const unsigned char *cs = READ_ONCE(dentry->d_name.name);
277
278 return dentry_string_cmp(cs, ct, tcount);
279 }
280
281 struct external_name {
282 union {
283 atomic_t count;
284 struct rcu_head head;
285 } u;
286 unsigned char name[];
287 };
288
external_name(struct dentry * dentry)289 static inline struct external_name *external_name(struct dentry *dentry)
290 {
291 return container_of(dentry->d_name.name, struct external_name, name[0]);
292 }
293
__d_free(struct rcu_head * head)294 static void __d_free(struct rcu_head *head)
295 {
296 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
297
298 kmem_cache_free(dentry_cache, dentry);
299 }
300
__d_free_external(struct rcu_head * head)301 static void __d_free_external(struct rcu_head *head)
302 {
303 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
304 kfree(external_name(dentry));
305 kmem_cache_free(dentry_cache, dentry);
306 }
307
dname_external(const struct dentry * dentry)308 static inline int dname_external(const struct dentry *dentry)
309 {
310 return dentry->d_name.name != dentry->d_iname;
311 }
312
take_dentry_name_snapshot(struct name_snapshot * name,struct dentry * dentry)313 void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
314 {
315 spin_lock(&dentry->d_lock);
316 name->name = dentry->d_name;
317 if (unlikely(dname_external(dentry))) {
318 atomic_inc(&external_name(dentry)->u.count);
319 } else {
320 memcpy(name->inline_name, dentry->d_iname,
321 dentry->d_name.len + 1);
322 name->name.name = name->inline_name;
323 }
324 spin_unlock(&dentry->d_lock);
325 }
326 EXPORT_SYMBOL(take_dentry_name_snapshot);
327
release_dentry_name_snapshot(struct name_snapshot * name)328 void release_dentry_name_snapshot(struct name_snapshot *name)
329 {
330 if (unlikely(name->name.name != name->inline_name)) {
331 struct external_name *p;
332 p = container_of(name->name.name, struct external_name, name[0]);
333 if (unlikely(atomic_dec_and_test(&p->u.count)))
334 kfree_rcu(p, u.head);
335 }
336 }
337 EXPORT_SYMBOL(release_dentry_name_snapshot);
338
__d_set_inode_and_type(struct dentry * dentry,struct inode * inode,unsigned type_flags)339 static inline void __d_set_inode_and_type(struct dentry *dentry,
340 struct inode *inode,
341 unsigned type_flags)
342 {
343 unsigned flags;
344
345 dentry->d_inode = inode;
346 flags = READ_ONCE(dentry->d_flags);
347 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
348 flags |= type_flags;
349 smp_store_release(&dentry->d_flags, flags);
350 }
351
__d_clear_type_and_inode(struct dentry * dentry)352 static inline void __d_clear_type_and_inode(struct dentry *dentry)
353 {
354 unsigned flags = READ_ONCE(dentry->d_flags);
355
356 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
357 WRITE_ONCE(dentry->d_flags, flags);
358 dentry->d_inode = NULL;
359 if (dentry->d_flags & DCACHE_LRU_LIST)
360 this_cpu_inc(nr_dentry_negative);
361 }
362
dentry_free(struct dentry * dentry)363 static void dentry_free(struct dentry *dentry)
364 {
365 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias));
366 if (unlikely(dname_external(dentry))) {
367 struct external_name *p = external_name(dentry);
368 if (likely(atomic_dec_and_test(&p->u.count))) {
369 call_rcu(&dentry->d_u.d_rcu, __d_free_external);
370 return;
371 }
372 }
373 /* if dentry was never visible to RCU, immediate free is OK */
374 if (dentry->d_flags & DCACHE_NORCU)
375 __d_free(&dentry->d_u.d_rcu);
376 else
377 call_rcu(&dentry->d_u.d_rcu, __d_free);
378 }
379
380 /*
381 * Release the dentry's inode, using the filesystem
382 * d_iput() operation if defined.
383 */
dentry_unlink_inode(struct dentry * dentry)384 static void dentry_unlink_inode(struct dentry * dentry)
385 __releases(dentry->d_lock)
386 __releases(dentry->d_inode->i_lock)
387 {
388 struct inode *inode = dentry->d_inode;
389
390 raw_write_seqcount_begin(&dentry->d_seq);
391 __d_clear_type_and_inode(dentry);
392 hlist_del_init(&dentry->d_u.d_alias);
393 raw_write_seqcount_end(&dentry->d_seq);
394 spin_unlock(&dentry->d_lock);
395 spin_unlock(&inode->i_lock);
396 if (!inode->i_nlink)
397 fsnotify_inoderemove(inode);
398 if (dentry->d_op && dentry->d_op->d_iput)
399 dentry->d_op->d_iput(dentry, inode);
400 else
401 iput(inode);
402 }
403
404 /*
405 * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
406 * is in use - which includes both the "real" per-superblock
407 * LRU list _and_ the DCACHE_SHRINK_LIST use.
408 *
409 * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
410 * on the shrink list (ie not on the superblock LRU list).
411 *
412 * The per-cpu "nr_dentry_unused" counters are updated with
413 * the DCACHE_LRU_LIST bit.
414 *
415 * The per-cpu "nr_dentry_negative" counters are only updated
416 * when deleted from or added to the per-superblock LRU list, not
417 * from/to the shrink list. That is to avoid an unneeded dec/inc
418 * pair when moving from LRU to shrink list in select_collect().
419 *
420 * These helper functions make sure we always follow the
421 * rules. d_lock must be held by the caller.
422 */
423 #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
d_lru_add(struct dentry * dentry)424 static void d_lru_add(struct dentry *dentry)
425 {
426 D_FLAG_VERIFY(dentry, 0);
427 dentry->d_flags |= DCACHE_LRU_LIST;
428 this_cpu_inc(nr_dentry_unused);
429 if (d_is_negative(dentry))
430 this_cpu_inc(nr_dentry_negative);
431 WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
432 }
433
d_lru_del(struct dentry * dentry)434 static void d_lru_del(struct dentry *dentry)
435 {
436 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
437 dentry->d_flags &= ~DCACHE_LRU_LIST;
438 this_cpu_dec(nr_dentry_unused);
439 if (d_is_negative(dentry))
440 this_cpu_dec(nr_dentry_negative);
441 WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
442 }
443
d_shrink_del(struct dentry * dentry)444 static void d_shrink_del(struct dentry *dentry)
445 {
446 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
447 list_del_init(&dentry->d_lru);
448 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
449 this_cpu_dec(nr_dentry_unused);
450 }
451
d_shrink_add(struct dentry * dentry,struct list_head * list)452 static void d_shrink_add(struct dentry *dentry, struct list_head *list)
453 {
454 D_FLAG_VERIFY(dentry, 0);
455 list_add(&dentry->d_lru, list);
456 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
457 this_cpu_inc(nr_dentry_unused);
458 }
459
460 /*
461 * These can only be called under the global LRU lock, ie during the
462 * callback for freeing the LRU list. "isolate" removes it from the
463 * LRU lists entirely, while shrink_move moves it to the indicated
464 * private list.
465 */
d_lru_isolate(struct list_lru_one * lru,struct dentry * dentry)466 static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)
467 {
468 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
469 dentry->d_flags &= ~DCACHE_LRU_LIST;
470 this_cpu_dec(nr_dentry_unused);
471 if (d_is_negative(dentry))
472 this_cpu_dec(nr_dentry_negative);
473 list_lru_isolate(lru, &dentry->d_lru);
474 }
475
d_lru_shrink_move(struct list_lru_one * lru,struct dentry * dentry,struct list_head * list)476 static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,
477 struct list_head *list)
478 {
479 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
480 dentry->d_flags |= DCACHE_SHRINK_LIST;
481 if (d_is_negative(dentry))
482 this_cpu_dec(nr_dentry_negative);
483 list_lru_isolate_move(lru, &dentry->d_lru, list);
484 }
485
___d_drop(struct dentry * dentry)486 static void ___d_drop(struct dentry *dentry)
487 {
488 struct hlist_bl_head *b;
489 /*
490 * Hashed dentries are normally on the dentry hashtable,
491 * with the exception of those newly allocated by
492 * d_obtain_root, which are always IS_ROOT:
493 */
494 if (unlikely(IS_ROOT(dentry)))
495 b = &dentry->d_sb->s_roots;
496 else
497 b = d_hash(dentry->d_name.hash);
498
499 hlist_bl_lock(b);
500 __hlist_bl_del(&dentry->d_hash);
501 hlist_bl_unlock(b);
502 }
503
__d_drop(struct dentry * dentry)504 void __d_drop(struct dentry *dentry)
505 {
506 if (!d_unhashed(dentry)) {
507 ___d_drop(dentry);
508 dentry->d_hash.pprev = NULL;
509 write_seqcount_invalidate(&dentry->d_seq);
510 }
511 }
512 EXPORT_SYMBOL(__d_drop);
513
514 /**
515 * d_drop - drop a dentry
516 * @dentry: dentry to drop
517 *
518 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
519 * be found through a VFS lookup any more. Note that this is different from
520 * deleting the dentry - d_delete will try to mark the dentry negative if
521 * possible, giving a successful _negative_ lookup, while d_drop will
522 * just make the cache lookup fail.
523 *
524 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
525 * reason (NFS timeouts or autofs deletes).
526 *
527 * __d_drop requires dentry->d_lock
528 *
529 * ___d_drop doesn't mark dentry as "unhashed"
530 * (dentry->d_hash.pprev will be LIST_POISON2, not NULL).
531 */
d_drop(struct dentry * dentry)532 void d_drop(struct dentry *dentry)
533 {
534 spin_lock(&dentry->d_lock);
535 __d_drop(dentry);
536 spin_unlock(&dentry->d_lock);
537 }
538 EXPORT_SYMBOL(d_drop);
539
dentry_unlist(struct dentry * dentry,struct dentry * parent)540 static inline void dentry_unlist(struct dentry *dentry, struct dentry *parent)
541 {
542 struct dentry *next;
543 /*
544 * Inform d_walk() and shrink_dentry_list() that we are no longer
545 * attached to the dentry tree
546 */
547 dentry->d_flags |= DCACHE_DENTRY_KILLED;
548 if (unlikely(list_empty(&dentry->d_child)))
549 return;
550 __list_del_entry(&dentry->d_child);
551 /*
552 * Cursors can move around the list of children. While we'd been
553 * a normal list member, it didn't matter - ->d_child.next would've
554 * been updated. However, from now on it won't be and for the
555 * things like d_walk() it might end up with a nasty surprise.
556 * Normally d_walk() doesn't care about cursors moving around -
557 * ->d_lock on parent prevents that and since a cursor has no children
558 * of its own, we get through it without ever unlocking the parent.
559 * There is one exception, though - if we ascend from a child that
560 * gets killed as soon as we unlock it, the next sibling is found
561 * using the value left in its ->d_child.next. And if _that_
562 * pointed to a cursor, and cursor got moved (e.g. by lseek())
563 * before d_walk() regains parent->d_lock, we'll end up skipping
564 * everything the cursor had been moved past.
565 *
566 * Solution: make sure that the pointer left behind in ->d_child.next
567 * points to something that won't be moving around. I.e. skip the
568 * cursors.
569 */
570 while (dentry->d_child.next != &parent->d_subdirs) {
571 next = list_entry(dentry->d_child.next, struct dentry, d_child);
572 if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))
573 break;
574 dentry->d_child.next = next->d_child.next;
575 }
576 }
577
__dentry_kill(struct dentry * dentry)578 static void __dentry_kill(struct dentry *dentry)
579 {
580 struct dentry *parent = NULL;
581 bool can_free = true;
582 if (!IS_ROOT(dentry))
583 parent = dentry->d_parent;
584
585 /*
586 * The dentry is now unrecoverably dead to the world.
587 */
588 lockref_mark_dead(&dentry->d_lockref);
589
590 /*
591 * inform the fs via d_prune that this dentry is about to be
592 * unhashed and destroyed.
593 */
594 if (dentry->d_flags & DCACHE_OP_PRUNE)
595 dentry->d_op->d_prune(dentry);
596
597 if (dentry->d_flags & DCACHE_LRU_LIST) {
598 if (!(dentry->d_flags & DCACHE_SHRINK_LIST))
599 d_lru_del(dentry);
600 }
601 /* if it was on the hash then remove it */
602 __d_drop(dentry);
603 dentry_unlist(dentry, parent);
604 if (parent)
605 spin_unlock(&parent->d_lock);
606 if (dentry->d_inode)
607 dentry_unlink_inode(dentry);
608 else
609 spin_unlock(&dentry->d_lock);
610 this_cpu_dec(nr_dentry);
611 if (dentry->d_op && dentry->d_op->d_release)
612 dentry->d_op->d_release(dentry);
613
614 spin_lock(&dentry->d_lock);
615 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
616 dentry->d_flags |= DCACHE_MAY_FREE;
617 can_free = false;
618 }
619 spin_unlock(&dentry->d_lock);
620 if (likely(can_free))
621 dentry_free(dentry);
622 cond_resched();
623 }
624
__lock_parent(struct dentry * dentry)625 static struct dentry *__lock_parent(struct dentry *dentry)
626 {
627 struct dentry *parent;
628 rcu_read_lock();
629 spin_unlock(&dentry->d_lock);
630 again:
631 parent = READ_ONCE(dentry->d_parent);
632 spin_lock(&parent->d_lock);
633 /*
634 * We can't blindly lock dentry until we are sure
635 * that we won't violate the locking order.
636 * Any changes of dentry->d_parent must have
637 * been done with parent->d_lock held, so
638 * spin_lock() above is enough of a barrier
639 * for checking if it's still our child.
640 */
641 if (unlikely(parent != dentry->d_parent)) {
642 spin_unlock(&parent->d_lock);
643 goto again;
644 }
645 rcu_read_unlock();
646 if (parent != dentry)
647 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
648 else
649 parent = NULL;
650 return parent;
651 }
652
lock_parent(struct dentry * dentry)653 static inline struct dentry *lock_parent(struct dentry *dentry)
654 {
655 struct dentry *parent = dentry->d_parent;
656 if (IS_ROOT(dentry))
657 return NULL;
658 if (likely(spin_trylock(&parent->d_lock)))
659 return parent;
660 return __lock_parent(dentry);
661 }
662
retain_dentry(struct dentry * dentry)663 static inline bool retain_dentry(struct dentry *dentry)
664 {
665 WARN_ON(d_in_lookup(dentry));
666
667 /* Unreachable? Get rid of it */
668 if (unlikely(d_unhashed(dentry)))
669 return false;
670
671 if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED))
672 return false;
673
674 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
675 if (dentry->d_op->d_delete(dentry))
676 return false;
677 }
678
679 if (unlikely(dentry->d_flags & DCACHE_DONTCACHE))
680 return false;
681
682 /* retain; LRU fodder */
683 dentry->d_lockref.count--;
684 if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
685 d_lru_add(dentry);
686 else if (unlikely(!(dentry->d_flags & DCACHE_REFERENCED)))
687 dentry->d_flags |= DCACHE_REFERENCED;
688 return true;
689 }
690
d_mark_dontcache(struct inode * inode)691 void d_mark_dontcache(struct inode *inode)
692 {
693 struct dentry *de;
694
695 spin_lock(&inode->i_lock);
696 hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
697 spin_lock(&de->d_lock);
698 de->d_flags |= DCACHE_DONTCACHE;
699 spin_unlock(&de->d_lock);
700 }
701 inode->i_state |= I_DONTCACHE;
702 spin_unlock(&inode->i_lock);
703 }
704 EXPORT_SYMBOL(d_mark_dontcache);
705
706 /*
707 * Finish off a dentry we've decided to kill.
708 * dentry->d_lock must be held, returns with it unlocked.
709 * Returns dentry requiring refcount drop, or NULL if we're done.
710 */
dentry_kill(struct dentry * dentry)711 static struct dentry *dentry_kill(struct dentry *dentry)
712 __releases(dentry->d_lock)
713 {
714 struct inode *inode = dentry->d_inode;
715 struct dentry *parent = NULL;
716
717 if (inode && unlikely(!spin_trylock(&inode->i_lock)))
718 goto slow_positive;
719
720 if (!IS_ROOT(dentry)) {
721 parent = dentry->d_parent;
722 if (unlikely(!spin_trylock(&parent->d_lock))) {
723 parent = __lock_parent(dentry);
724 if (likely(inode || !dentry->d_inode))
725 goto got_locks;
726 /* negative that became positive */
727 if (parent)
728 spin_unlock(&parent->d_lock);
729 inode = dentry->d_inode;
730 goto slow_positive;
731 }
732 }
733 __dentry_kill(dentry);
734 return parent;
735
736 slow_positive:
737 spin_unlock(&dentry->d_lock);
738 spin_lock(&inode->i_lock);
739 spin_lock(&dentry->d_lock);
740 parent = lock_parent(dentry);
741 got_locks:
742 if (unlikely(dentry->d_lockref.count != 1)) {
743 dentry->d_lockref.count--;
744 } else if (likely(!retain_dentry(dentry))) {
745 __dentry_kill(dentry);
746 return parent;
747 }
748 /* we are keeping it, after all */
749 if (inode)
750 spin_unlock(&inode->i_lock);
751 if (parent)
752 spin_unlock(&parent->d_lock);
753 spin_unlock(&dentry->d_lock);
754 return NULL;
755 }
756
757 /*
758 * Try to do a lockless dput(), and return whether that was successful.
759 *
760 * If unsuccessful, we return false, having already taken the dentry lock.
761 *
762 * The caller needs to hold the RCU read lock, so that the dentry is
763 * guaranteed to stay around even if the refcount goes down to zero!
764 */
fast_dput(struct dentry * dentry)765 static inline bool fast_dput(struct dentry *dentry)
766 {
767 int ret;
768 unsigned int d_flags;
769
770 /*
771 * If we have a d_op->d_delete() operation, we sould not
772 * let the dentry count go to zero, so use "put_or_lock".
773 */
774 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE))
775 return lockref_put_or_lock(&dentry->d_lockref);
776
777 /*
778 * .. otherwise, we can try to just decrement the
779 * lockref optimistically.
780 */
781 ret = lockref_put_return(&dentry->d_lockref);
782
783 /*
784 * If the lockref_put_return() failed due to the lock being held
785 * by somebody else, the fast path has failed. We will need to
786 * get the lock, and then check the count again.
787 */
788 if (unlikely(ret < 0)) {
789 spin_lock(&dentry->d_lock);
790 if (WARN_ON_ONCE(dentry->d_lockref.count <= 0)) {
791 spin_unlock(&dentry->d_lock);
792 return true;
793 }
794 dentry->d_lockref.count--;
795 goto locked;
796 }
797
798 /*
799 * If we weren't the last ref, we're done.
800 */
801 if (ret)
802 return true;
803
804 /*
805 * Careful, careful. The reference count went down
806 * to zero, but we don't hold the dentry lock, so
807 * somebody else could get it again, and do another
808 * dput(), and we need to not race with that.
809 *
810 * However, there is a very special and common case
811 * where we don't care, because there is nothing to
812 * do: the dentry is still hashed, it does not have
813 * a 'delete' op, and it's referenced and already on
814 * the LRU list.
815 *
816 * NOTE! Since we aren't locked, these values are
817 * not "stable". However, it is sufficient that at
818 * some point after we dropped the reference the
819 * dentry was hashed and the flags had the proper
820 * value. Other dentry users may have re-gotten
821 * a reference to the dentry and change that, but
822 * our work is done - we can leave the dentry
823 * around with a zero refcount.
824 *
825 * Nevertheless, there are two cases that we should kill
826 * the dentry anyway.
827 * 1. free disconnected dentries as soon as their refcount
828 * reached zero.
829 * 2. free dentries if they should not be cached.
830 */
831 smp_rmb();
832 d_flags = READ_ONCE(dentry->d_flags);
833 d_flags &= DCACHE_REFERENCED | DCACHE_LRU_LIST |
834 DCACHE_DISCONNECTED | DCACHE_DONTCACHE;
835
836 /* Nothing to do? Dropping the reference was all we needed? */
837 if (d_flags == (DCACHE_REFERENCED | DCACHE_LRU_LIST) && !d_unhashed(dentry))
838 return true;
839
840 /*
841 * Not the fast normal case? Get the lock. We've already decremented
842 * the refcount, but we'll need to re-check the situation after
843 * getting the lock.
844 */
845 spin_lock(&dentry->d_lock);
846
847 /*
848 * Did somebody else grab a reference to it in the meantime, and
849 * we're no longer the last user after all? Alternatively, somebody
850 * else could have killed it and marked it dead. Either way, we
851 * don't need to do anything else.
852 */
853 locked:
854 if (dentry->d_lockref.count) {
855 spin_unlock(&dentry->d_lock);
856 return true;
857 }
858
859 /*
860 * Re-get the reference we optimistically dropped. We hold the
861 * lock, and we just tested that it was zero, so we can just
862 * set it to 1.
863 */
864 dentry->d_lockref.count = 1;
865 return false;
866 }
867
868
869 /*
870 * This is dput
871 *
872 * This is complicated by the fact that we do not want to put
873 * dentries that are no longer on any hash chain on the unused
874 * list: we'd much rather just get rid of them immediately.
875 *
876 * However, that implies that we have to traverse the dentry
877 * tree upwards to the parents which might _also_ now be
878 * scheduled for deletion (it may have been only waiting for
879 * its last child to go away).
880 *
881 * This tail recursion is done by hand as we don't want to depend
882 * on the compiler to always get this right (gcc generally doesn't).
883 * Real recursion would eat up our stack space.
884 */
885
886 /*
887 * dput - release a dentry
888 * @dentry: dentry to release
889 *
890 * Release a dentry. This will drop the usage count and if appropriate
891 * call the dentry unlink method as well as removing it from the queues and
892 * releasing its resources. If the parent dentries were scheduled for release
893 * they too may now get deleted.
894 */
dput(struct dentry * dentry)895 void dput(struct dentry *dentry)
896 {
897 while (dentry) {
898 might_sleep();
899
900 rcu_read_lock();
901 if (likely(fast_dput(dentry))) {
902 rcu_read_unlock();
903 return;
904 }
905
906 /* Slow case: now with the dentry lock held */
907 rcu_read_unlock();
908
909 if (likely(retain_dentry(dentry))) {
910 spin_unlock(&dentry->d_lock);
911 return;
912 }
913
914 dentry = dentry_kill(dentry);
915 }
916 }
917 EXPORT_SYMBOL(dput);
918
__dput_to_list(struct dentry * dentry,struct list_head * list)919 static void __dput_to_list(struct dentry *dentry, struct list_head *list)
920 __must_hold(&dentry->d_lock)
921 {
922 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
923 /* let the owner of the list it's on deal with it */
924 --dentry->d_lockref.count;
925 } else {
926 if (dentry->d_flags & DCACHE_LRU_LIST)
927 d_lru_del(dentry);
928 if (!--dentry->d_lockref.count)
929 d_shrink_add(dentry, list);
930 }
931 }
932
dput_to_list(struct dentry * dentry,struct list_head * list)933 void dput_to_list(struct dentry *dentry, struct list_head *list)
934 {
935 rcu_read_lock();
936 if (likely(fast_dput(dentry))) {
937 rcu_read_unlock();
938 return;
939 }
940 rcu_read_unlock();
941 if (!retain_dentry(dentry))
942 __dput_to_list(dentry, list);
943 spin_unlock(&dentry->d_lock);
944 }
945
946 /* This must be called with d_lock held */
__dget_dlock(struct dentry * dentry)947 static inline void __dget_dlock(struct dentry *dentry)
948 {
949 dentry->d_lockref.count++;
950 }
951
__dget(struct dentry * dentry)952 static inline void __dget(struct dentry *dentry)
953 {
954 lockref_get(&dentry->d_lockref);
955 }
956
dget_parent(struct dentry * dentry)957 struct dentry *dget_parent(struct dentry *dentry)
958 {
959 int gotref;
960 struct dentry *ret;
961 unsigned seq;
962
963 /*
964 * Do optimistic parent lookup without any
965 * locking.
966 */
967 rcu_read_lock();
968 seq = raw_seqcount_begin(&dentry->d_seq);
969 ret = READ_ONCE(dentry->d_parent);
970 gotref = lockref_get_not_zero(&ret->d_lockref);
971 rcu_read_unlock();
972 if (likely(gotref)) {
973 if (!read_seqcount_retry(&dentry->d_seq, seq))
974 return ret;
975 dput(ret);
976 }
977
978 repeat:
979 /*
980 * Don't need rcu_dereference because we re-check it was correct under
981 * the lock.
982 */
983 rcu_read_lock();
984 ret = dentry->d_parent;
985 spin_lock(&ret->d_lock);
986 if (unlikely(ret != dentry->d_parent)) {
987 spin_unlock(&ret->d_lock);
988 rcu_read_unlock();
989 goto repeat;
990 }
991 rcu_read_unlock();
992 BUG_ON(!ret->d_lockref.count);
993 ret->d_lockref.count++;
994 spin_unlock(&ret->d_lock);
995 return ret;
996 }
997 EXPORT_SYMBOL(dget_parent);
998
__d_find_any_alias(struct inode * inode)999 static struct dentry * __d_find_any_alias(struct inode *inode)
1000 {
1001 struct dentry *alias;
1002
1003 if (hlist_empty(&inode->i_dentry))
1004 return NULL;
1005 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
1006 __dget(alias);
1007 return alias;
1008 }
1009
1010 /**
1011 * d_find_any_alias - find any alias for a given inode
1012 * @inode: inode to find an alias for
1013 *
1014 * If any aliases exist for the given inode, take and return a
1015 * reference for one of them. If no aliases exist, return %NULL.
1016 */
d_find_any_alias(struct inode * inode)1017 struct dentry *d_find_any_alias(struct inode *inode)
1018 {
1019 struct dentry *de;
1020
1021 spin_lock(&inode->i_lock);
1022 de = __d_find_any_alias(inode);
1023 spin_unlock(&inode->i_lock);
1024 return de;
1025 }
1026 EXPORT_SYMBOL(d_find_any_alias);
1027
__d_find_alias(struct inode * inode)1028 static struct dentry *__d_find_alias(struct inode *inode)
1029 {
1030 struct dentry *alias;
1031
1032 if (S_ISDIR(inode->i_mode))
1033 return __d_find_any_alias(inode);
1034
1035 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
1036 spin_lock(&alias->d_lock);
1037 if (!d_unhashed(alias)) {
1038 __dget_dlock(alias);
1039 spin_unlock(&alias->d_lock);
1040 return alias;
1041 }
1042 spin_unlock(&alias->d_lock);
1043 }
1044 return NULL;
1045 }
1046
1047 /**
1048 * d_find_alias - grab a hashed alias of inode
1049 * @inode: inode in question
1050 *
1051 * If inode has a hashed alias, or is a directory and has any alias,
1052 * acquire the reference to alias and return it. Otherwise return NULL.
1053 * Notice that if inode is a directory there can be only one alias and
1054 * it can be unhashed only if it has no children, or if it is the root
1055 * of a filesystem, or if the directory was renamed and d_revalidate
1056 * was the first vfs operation to notice.
1057 *
1058 * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
1059 * any other hashed alias over that one.
1060 */
d_find_alias(struct inode * inode)1061 struct dentry *d_find_alias(struct inode *inode)
1062 {
1063 struct dentry *de = NULL;
1064
1065 if (!hlist_empty(&inode->i_dentry)) {
1066 spin_lock(&inode->i_lock);
1067 de = __d_find_alias(inode);
1068 spin_unlock(&inode->i_lock);
1069 }
1070 return de;
1071 }
1072 EXPORT_SYMBOL(d_find_alias);
1073
1074 /*
1075 * Caller MUST be holding rcu_read_lock() and be guaranteed
1076 * that inode won't get freed until rcu_read_unlock().
1077 */
d_find_alias_rcu(struct inode * inode)1078 struct dentry *d_find_alias_rcu(struct inode *inode)
1079 {
1080 struct hlist_head *l = &inode->i_dentry;
1081 struct dentry *de = NULL;
1082
1083 spin_lock(&inode->i_lock);
1084 // ->i_dentry and ->i_rcu are colocated, but the latter won't be
1085 // used without having I_FREEING set, which means no aliases left
1086 if (likely(!(inode->i_state & I_FREEING) && !hlist_empty(l))) {
1087 if (S_ISDIR(inode->i_mode)) {
1088 de = hlist_entry(l->first, struct dentry, d_u.d_alias);
1089 } else {
1090 hlist_for_each_entry(de, l, d_u.d_alias)
1091 if (!d_unhashed(de))
1092 break;
1093 }
1094 }
1095 spin_unlock(&inode->i_lock);
1096 return de;
1097 }
1098
1099 /*
1100 * Try to kill dentries associated with this inode.
1101 * WARNING: you must own a reference to inode.
1102 */
d_prune_aliases(struct inode * inode)1103 void d_prune_aliases(struct inode *inode)
1104 {
1105 struct dentry *dentry;
1106 restart:
1107 spin_lock(&inode->i_lock);
1108 hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
1109 spin_lock(&dentry->d_lock);
1110 if (!dentry->d_lockref.count) {
1111 struct dentry *parent = lock_parent(dentry);
1112 if (likely(!dentry->d_lockref.count)) {
1113 __dentry_kill(dentry);
1114 dput(parent);
1115 goto restart;
1116 }
1117 if (parent)
1118 spin_unlock(&parent->d_lock);
1119 }
1120 spin_unlock(&dentry->d_lock);
1121 }
1122 spin_unlock(&inode->i_lock);
1123 }
1124 EXPORT_SYMBOL(d_prune_aliases);
1125
1126 /*
1127 * Lock a dentry from shrink list.
1128 * Called under rcu_read_lock() and dentry->d_lock; the former
1129 * guarantees that nothing we access will be freed under us.
1130 * Note that dentry is *not* protected from concurrent dentry_kill(),
1131 * d_delete(), etc.
1132 *
1133 * Return false if dentry has been disrupted or grabbed, leaving
1134 * the caller to kick it off-list. Otherwise, return true and have
1135 * that dentry's inode and parent both locked.
1136 */
shrink_lock_dentry(struct dentry * dentry)1137 static bool shrink_lock_dentry(struct dentry *dentry)
1138 {
1139 struct inode *inode;
1140 struct dentry *parent;
1141
1142 if (dentry->d_lockref.count)
1143 return false;
1144
1145 inode = dentry->d_inode;
1146 if (inode && unlikely(!spin_trylock(&inode->i_lock))) {
1147 spin_unlock(&dentry->d_lock);
1148 spin_lock(&inode->i_lock);
1149 spin_lock(&dentry->d_lock);
1150 if (unlikely(dentry->d_lockref.count))
1151 goto out;
1152 /* changed inode means that somebody had grabbed it */
1153 if (unlikely(inode != dentry->d_inode))
1154 goto out;
1155 }
1156
1157 parent = dentry->d_parent;
1158 if (IS_ROOT(dentry) || likely(spin_trylock(&parent->d_lock)))
1159 return true;
1160
1161 spin_unlock(&dentry->d_lock);
1162 spin_lock(&parent->d_lock);
1163 if (unlikely(parent != dentry->d_parent)) {
1164 spin_unlock(&parent->d_lock);
1165 spin_lock(&dentry->d_lock);
1166 goto out;
1167 }
1168 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1169 if (likely(!dentry->d_lockref.count))
1170 return true;
1171 spin_unlock(&parent->d_lock);
1172 out:
1173 if (inode)
1174 spin_unlock(&inode->i_lock);
1175 return false;
1176 }
1177
shrink_dentry_list(struct list_head * list)1178 void shrink_dentry_list(struct list_head *list)
1179 {
1180 while (!list_empty(list)) {
1181 struct dentry *dentry, *parent;
1182
1183 dentry = list_entry(list->prev, struct dentry, d_lru);
1184 spin_lock(&dentry->d_lock);
1185 rcu_read_lock();
1186 if (!shrink_lock_dentry(dentry)) {
1187 bool can_free = false;
1188 rcu_read_unlock();
1189 d_shrink_del(dentry);
1190 if (dentry->d_lockref.count < 0)
1191 can_free = dentry->d_flags & DCACHE_MAY_FREE;
1192 spin_unlock(&dentry->d_lock);
1193 if (can_free)
1194 dentry_free(dentry);
1195 continue;
1196 }
1197 rcu_read_unlock();
1198 d_shrink_del(dentry);
1199 parent = dentry->d_parent;
1200 if (parent != dentry)
1201 __dput_to_list(parent, list);
1202 __dentry_kill(dentry);
1203 }
1204 }
1205
dentry_lru_isolate(struct list_head * item,struct list_lru_one * lru,spinlock_t * lru_lock,void * arg)1206 static enum lru_status dentry_lru_isolate(struct list_head *item,
1207 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1208 {
1209 struct list_head *freeable = arg;
1210 struct dentry *dentry = container_of(item, struct dentry, d_lru);
1211
1212
1213 /*
1214 * we are inverting the lru lock/dentry->d_lock here,
1215 * so use a trylock. If we fail to get the lock, just skip
1216 * it
1217 */
1218 if (!spin_trylock(&dentry->d_lock))
1219 return LRU_SKIP;
1220
1221 /*
1222 * Referenced dentries are still in use. If they have active
1223 * counts, just remove them from the LRU. Otherwise give them
1224 * another pass through the LRU.
1225 */
1226 if (dentry->d_lockref.count) {
1227 d_lru_isolate(lru, dentry);
1228 spin_unlock(&dentry->d_lock);
1229 return LRU_REMOVED;
1230 }
1231
1232 if (dentry->d_flags & DCACHE_REFERENCED) {
1233 dentry->d_flags &= ~DCACHE_REFERENCED;
1234 spin_unlock(&dentry->d_lock);
1235
1236 /*
1237 * The list move itself will be made by the common LRU code. At
1238 * this point, we've dropped the dentry->d_lock but keep the
1239 * lru lock. This is safe to do, since every list movement is
1240 * protected by the lru lock even if both locks are held.
1241 *
1242 * This is guaranteed by the fact that all LRU management
1243 * functions are intermediated by the LRU API calls like
1244 * list_lru_add and list_lru_del. List movement in this file
1245 * only ever occur through this functions or through callbacks
1246 * like this one, that are called from the LRU API.
1247 *
1248 * The only exceptions to this are functions like
1249 * shrink_dentry_list, and code that first checks for the
1250 * DCACHE_SHRINK_LIST flag. Those are guaranteed to be
1251 * operating only with stack provided lists after they are
1252 * properly isolated from the main list. It is thus, always a
1253 * local access.
1254 */
1255 return LRU_ROTATE;
1256 }
1257
1258 d_lru_shrink_move(lru, dentry, freeable);
1259 spin_unlock(&dentry->d_lock);
1260
1261 return LRU_REMOVED;
1262 }
1263
1264 /**
1265 * prune_dcache_sb - shrink the dcache
1266 * @sb: superblock
1267 * @sc: shrink control, passed to list_lru_shrink_walk()
1268 *
1269 * Attempt to shrink the superblock dcache LRU by @sc->nr_to_scan entries. This
1270 * is done when we need more memory and called from the superblock shrinker
1271 * function.
1272 *
1273 * This function may fail to free any resources if all the dentries are in
1274 * use.
1275 */
prune_dcache_sb(struct super_block * sb,struct shrink_control * sc)1276 long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc)
1277 {
1278 LIST_HEAD(dispose);
1279 long freed;
1280
1281 freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc,
1282 dentry_lru_isolate, &dispose);
1283 shrink_dentry_list(&dispose);
1284 return freed;
1285 }
1286
dentry_lru_isolate_shrink(struct list_head * item,struct list_lru_one * lru,spinlock_t * lru_lock,void * arg)1287 static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
1288 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1289 {
1290 struct list_head *freeable = arg;
1291 struct dentry *dentry = container_of(item, struct dentry, d_lru);
1292
1293 /*
1294 * we are inverting the lru lock/dentry->d_lock here,
1295 * so use a trylock. If we fail to get the lock, just skip
1296 * it
1297 */
1298 if (!spin_trylock(&dentry->d_lock))
1299 return LRU_SKIP;
1300
1301 d_lru_shrink_move(lru, dentry, freeable);
1302 spin_unlock(&dentry->d_lock);
1303
1304 return LRU_REMOVED;
1305 }
1306
1307
1308 /**
1309 * shrink_dcache_sb - shrink dcache for a superblock
1310 * @sb: superblock
1311 *
1312 * Shrink the dcache for the specified super block. This is used to free
1313 * the dcache before unmounting a file system.
1314 */
shrink_dcache_sb(struct super_block * sb)1315 void shrink_dcache_sb(struct super_block *sb)
1316 {
1317 do {
1318 LIST_HEAD(dispose);
1319
1320 list_lru_walk(&sb->s_dentry_lru,
1321 dentry_lru_isolate_shrink, &dispose, 1024);
1322 shrink_dentry_list(&dispose);
1323 } while (list_lru_count(&sb->s_dentry_lru) > 0);
1324 }
1325 EXPORT_SYMBOL(shrink_dcache_sb);
1326
1327 /**
1328 * enum d_walk_ret - action to talke during tree walk
1329 * @D_WALK_CONTINUE: contrinue walk
1330 * @D_WALK_QUIT: quit walk
1331 * @D_WALK_NORETRY: quit when retry is needed
1332 * @D_WALK_SKIP: skip this dentry and its children
1333 */
1334 enum d_walk_ret {
1335 D_WALK_CONTINUE,
1336 D_WALK_QUIT,
1337 D_WALK_NORETRY,
1338 D_WALK_SKIP,
1339 };
1340
1341 /**
1342 * d_walk - walk the dentry tree
1343 * @parent: start of walk
1344 * @data: data passed to @enter() and @finish()
1345 * @enter: callback when first entering the dentry
1346 *
1347 * The @enter() callbacks are called with d_lock held.
1348 */
d_walk(struct dentry * parent,void * data,enum d_walk_ret (* enter)(void *,struct dentry *))1349 static void d_walk(struct dentry *parent, void *data,
1350 enum d_walk_ret (*enter)(void *, struct dentry *))
1351 {
1352 struct dentry *this_parent;
1353 struct list_head *next;
1354 unsigned seq = 0;
1355 enum d_walk_ret ret;
1356 bool retry = true;
1357
1358 again:
1359 read_seqbegin_or_lock(&rename_lock, &seq);
1360 this_parent = parent;
1361 spin_lock(&this_parent->d_lock);
1362
1363 ret = enter(data, this_parent);
1364 switch (ret) {
1365 case D_WALK_CONTINUE:
1366 break;
1367 case D_WALK_QUIT:
1368 case D_WALK_SKIP:
1369 goto out_unlock;
1370 case D_WALK_NORETRY:
1371 retry = false;
1372 break;
1373 }
1374 repeat:
1375 next = this_parent->d_subdirs.next;
1376 resume:
1377 while (next != &this_parent->d_subdirs) {
1378 struct list_head *tmp = next;
1379 struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
1380 next = tmp->next;
1381
1382 if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR))
1383 continue;
1384
1385 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1386
1387 ret = enter(data, dentry);
1388 switch (ret) {
1389 case D_WALK_CONTINUE:
1390 break;
1391 case D_WALK_QUIT:
1392 spin_unlock(&dentry->d_lock);
1393 goto out_unlock;
1394 case D_WALK_NORETRY:
1395 retry = false;
1396 break;
1397 case D_WALK_SKIP:
1398 spin_unlock(&dentry->d_lock);
1399 continue;
1400 }
1401
1402 if (!list_empty(&dentry->d_subdirs)) {
1403 spin_unlock(&this_parent->d_lock);
1404 spin_release(&dentry->d_lock.dep_map, _RET_IP_);
1405 this_parent = dentry;
1406 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
1407 goto repeat;
1408 }
1409 spin_unlock(&dentry->d_lock);
1410 }
1411 /*
1412 * All done at this level ... ascend and resume the search.
1413 */
1414 rcu_read_lock();
1415 ascend:
1416 if (this_parent != parent) {
1417 struct dentry *child = this_parent;
1418 this_parent = child->d_parent;
1419
1420 spin_unlock(&child->d_lock);
1421 spin_lock(&this_parent->d_lock);
1422
1423 /* might go back up the wrong parent if we have had a rename. */
1424 if (need_seqretry(&rename_lock, seq))
1425 goto rename_retry;
1426 /* go into the first sibling still alive */
1427 do {
1428 next = child->d_child.next;
1429 if (next == &this_parent->d_subdirs)
1430 goto ascend;
1431 child = list_entry(next, struct dentry, d_child);
1432 } while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED));
1433 rcu_read_unlock();
1434 goto resume;
1435 }
1436 if (need_seqretry(&rename_lock, seq))
1437 goto rename_retry;
1438 rcu_read_unlock();
1439
1440 out_unlock:
1441 spin_unlock(&this_parent->d_lock);
1442 done_seqretry(&rename_lock, seq);
1443 return;
1444
1445 rename_retry:
1446 spin_unlock(&this_parent->d_lock);
1447 rcu_read_unlock();
1448 BUG_ON(seq & 1);
1449 if (!retry)
1450 return;
1451 seq = 1;
1452 goto again;
1453 }
1454
1455 struct check_mount {
1456 struct vfsmount *mnt;
1457 unsigned int mounted;
1458 };
1459
path_check_mount(void * data,struct dentry * dentry)1460 static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry)
1461 {
1462 struct check_mount *info = data;
1463 struct path path = { .mnt = info->mnt, .dentry = dentry };
1464
1465 if (likely(!d_mountpoint(dentry)))
1466 return D_WALK_CONTINUE;
1467 if (__path_is_mountpoint(&path)) {
1468 info->mounted = 1;
1469 return D_WALK_QUIT;
1470 }
1471 return D_WALK_CONTINUE;
1472 }
1473
1474 /**
1475 * path_has_submounts - check for mounts over a dentry in the
1476 * current namespace.
1477 * @parent: path to check.
1478 *
1479 * Return true if the parent or its subdirectories contain
1480 * a mount point in the current namespace.
1481 */
path_has_submounts(const struct path * parent)1482 int path_has_submounts(const struct path *parent)
1483 {
1484 struct check_mount data = { .mnt = parent->mnt, .mounted = 0 };
1485
1486 read_seqlock_excl(&mount_lock);
1487 d_walk(parent->dentry, &data, path_check_mount);
1488 read_sequnlock_excl(&mount_lock);
1489
1490 return data.mounted;
1491 }
1492 EXPORT_SYMBOL(path_has_submounts);
1493
1494 /*
1495 * Called by mount code to set a mountpoint and check if the mountpoint is
1496 * reachable (e.g. NFS can unhash a directory dentry and then the complete
1497 * subtree can become unreachable).
1498 *
1499 * Only one of d_invalidate() and d_set_mounted() must succeed. For
1500 * this reason take rename_lock and d_lock on dentry and ancestors.
1501 */
d_set_mounted(struct dentry * dentry)1502 int d_set_mounted(struct dentry *dentry)
1503 {
1504 struct dentry *p;
1505 int ret = -ENOENT;
1506 write_seqlock(&rename_lock);
1507 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
1508 /* Need exclusion wrt. d_invalidate() */
1509 spin_lock(&p->d_lock);
1510 if (unlikely(d_unhashed(p))) {
1511 spin_unlock(&p->d_lock);
1512 goto out;
1513 }
1514 spin_unlock(&p->d_lock);
1515 }
1516 spin_lock(&dentry->d_lock);
1517 if (!d_unlinked(dentry)) {
1518 ret = -EBUSY;
1519 if (!d_mountpoint(dentry)) {
1520 dentry->d_flags |= DCACHE_MOUNTED;
1521 ret = 0;
1522 }
1523 }
1524 spin_unlock(&dentry->d_lock);
1525 out:
1526 write_sequnlock(&rename_lock);
1527 return ret;
1528 }
1529
1530 /*
1531 * Search the dentry child list of the specified parent,
1532 * and move any unused dentries to the end of the unused
1533 * list for prune_dcache(). We descend to the next level
1534 * whenever the d_subdirs list is non-empty and continue
1535 * searching.
1536 *
1537 * It returns zero iff there are no unused children,
1538 * otherwise it returns the number of children moved to
1539 * the end of the unused list. This may not be the total
1540 * number of unused children, because select_parent can
1541 * drop the lock and return early due to latency
1542 * constraints.
1543 */
1544
1545 struct select_data {
1546 struct dentry *start;
1547 union {
1548 long found;
1549 struct dentry *victim;
1550 };
1551 struct list_head dispose;
1552 };
1553
select_collect(void * _data,struct dentry * dentry)1554 static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
1555 {
1556 struct select_data *data = _data;
1557 enum d_walk_ret ret = D_WALK_CONTINUE;
1558
1559 if (data->start == dentry)
1560 goto out;
1561
1562 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1563 data->found++;
1564 } else {
1565 if (dentry->d_flags & DCACHE_LRU_LIST)
1566 d_lru_del(dentry);
1567 if (!dentry->d_lockref.count) {
1568 d_shrink_add(dentry, &data->dispose);
1569 data->found++;
1570 }
1571 }
1572 /*
1573 * We can return to the caller if we have found some (this
1574 * ensures forward progress). We'll be coming back to find
1575 * the rest.
1576 */
1577 if (!list_empty(&data->dispose))
1578 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1579 out:
1580 return ret;
1581 }
1582
select_collect2(void * _data,struct dentry * dentry)1583 static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
1584 {
1585 struct select_data *data = _data;
1586 enum d_walk_ret ret = D_WALK_CONTINUE;
1587
1588 if (data->start == dentry)
1589 goto out;
1590
1591 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1592 if (!dentry->d_lockref.count) {
1593 rcu_read_lock();
1594 data->victim = dentry;
1595 return D_WALK_QUIT;
1596 }
1597 } else {
1598 if (dentry->d_flags & DCACHE_LRU_LIST)
1599 d_lru_del(dentry);
1600 if (!dentry->d_lockref.count)
1601 d_shrink_add(dentry, &data->dispose);
1602 }
1603 /*
1604 * We can return to the caller if we have found some (this
1605 * ensures forward progress). We'll be coming back to find
1606 * the rest.
1607 */
1608 if (!list_empty(&data->dispose))
1609 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1610 out:
1611 return ret;
1612 }
1613
1614 /**
1615 * shrink_dcache_parent - prune dcache
1616 * @parent: parent of entries to prune
1617 *
1618 * Prune the dcache to remove unused children of the parent dentry.
1619 */
shrink_dcache_parent(struct dentry * parent)1620 void shrink_dcache_parent(struct dentry *parent)
1621 {
1622 for (;;) {
1623 struct select_data data = {.start = parent};
1624
1625 INIT_LIST_HEAD(&data.dispose);
1626 d_walk(parent, &data, select_collect);
1627
1628 if (!list_empty(&data.dispose)) {
1629 shrink_dentry_list(&data.dispose);
1630 continue;
1631 }
1632
1633 cond_resched();
1634 if (!data.found)
1635 break;
1636 data.victim = NULL;
1637 d_walk(parent, &data, select_collect2);
1638 if (data.victim) {
1639 struct dentry *parent;
1640 spin_lock(&data.victim->d_lock);
1641 if (!shrink_lock_dentry(data.victim)) {
1642 spin_unlock(&data.victim->d_lock);
1643 rcu_read_unlock();
1644 } else {
1645 rcu_read_unlock();
1646 parent = data.victim->d_parent;
1647 if (parent != data.victim)
1648 __dput_to_list(parent, &data.dispose);
1649 __dentry_kill(data.victim);
1650 }
1651 }
1652 if (!list_empty(&data.dispose))
1653 shrink_dentry_list(&data.dispose);
1654 }
1655 }
1656 EXPORT_SYMBOL(shrink_dcache_parent);
1657
umount_check(void * _data,struct dentry * dentry)1658 static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
1659 {
1660 /* it has busy descendents; complain about those instead */
1661 if (!list_empty(&dentry->d_subdirs))
1662 return D_WALK_CONTINUE;
1663
1664 /* root with refcount 1 is fine */
1665 if (dentry == _data && dentry->d_lockref.count == 1)
1666 return D_WALK_CONTINUE;
1667
1668 printk(KERN_ERR "BUG: Dentry %p{i=%lx,n=%pd} "
1669 " still in use (%d) [unmount of %s %s]\n",
1670 dentry,
1671 dentry->d_inode ?
1672 dentry->d_inode->i_ino : 0UL,
1673 dentry,
1674 dentry->d_lockref.count,
1675 dentry->d_sb->s_type->name,
1676 dentry->d_sb->s_id);
1677 WARN_ON(1);
1678 return D_WALK_CONTINUE;
1679 }
1680
do_one_tree(struct dentry * dentry)1681 static void do_one_tree(struct dentry *dentry)
1682 {
1683 shrink_dcache_parent(dentry);
1684 d_walk(dentry, dentry, umount_check);
1685 d_drop(dentry);
1686 dput(dentry);
1687 }
1688
1689 /*
1690 * destroy the dentries attached to a superblock on unmounting
1691 */
shrink_dcache_for_umount(struct super_block * sb)1692 void shrink_dcache_for_umount(struct super_block *sb)
1693 {
1694 struct dentry *dentry;
1695
1696 WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked");
1697
1698 dentry = sb->s_root;
1699 sb->s_root = NULL;
1700 do_one_tree(dentry);
1701
1702 while (!hlist_bl_empty(&sb->s_roots)) {
1703 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
1704 do_one_tree(dentry);
1705 }
1706 }
1707
find_submount(void * _data,struct dentry * dentry)1708 static enum d_walk_ret find_submount(void *_data, struct dentry *dentry)
1709 {
1710 struct dentry **victim = _data;
1711 if (d_mountpoint(dentry)) {
1712 __dget_dlock(dentry);
1713 *victim = dentry;
1714 return D_WALK_QUIT;
1715 }
1716 return D_WALK_CONTINUE;
1717 }
1718
1719 /**
1720 * d_invalidate - detach submounts, prune dcache, and drop
1721 * @dentry: dentry to invalidate (aka detach, prune and drop)
1722 */
d_invalidate(struct dentry * dentry)1723 void d_invalidate(struct dentry *dentry)
1724 {
1725 bool had_submounts = false;
1726 spin_lock(&dentry->d_lock);
1727 if (d_unhashed(dentry)) {
1728 spin_unlock(&dentry->d_lock);
1729 return;
1730 }
1731 __d_drop(dentry);
1732 spin_unlock(&dentry->d_lock);
1733
1734 /* Negative dentries can be dropped without further checks */
1735 if (!dentry->d_inode)
1736 return;
1737
1738 shrink_dcache_parent(dentry);
1739 for (;;) {
1740 struct dentry *victim = NULL;
1741 d_walk(dentry, &victim, find_submount);
1742 if (!victim) {
1743 if (had_submounts)
1744 shrink_dcache_parent(dentry);
1745 return;
1746 }
1747 had_submounts = true;
1748 detach_mounts(victim);
1749 dput(victim);
1750 }
1751 }
1752 EXPORT_SYMBOL(d_invalidate);
1753
1754 /**
1755 * __d_alloc - allocate a dcache entry
1756 * @sb: filesystem it will belong to
1757 * @name: qstr of the name
1758 *
1759 * Allocates a dentry. It returns %NULL if there is insufficient memory
1760 * available. On a success the dentry is returned. The name passed in is
1761 * copied and the copy passed in may be reused after this call.
1762 */
1763
__d_alloc(struct super_block * sb,const struct qstr * name)1764 static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
1765 {
1766 struct dentry *dentry;
1767 char *dname;
1768 int err;
1769
1770 dentry = kmem_cache_alloc_lru(dentry_cache, &sb->s_dentry_lru,
1771 GFP_KERNEL);
1772 if (!dentry)
1773 return NULL;
1774
1775 /*
1776 * We guarantee that the inline name is always NUL-terminated.
1777 * This way the memcpy() done by the name switching in rename
1778 * will still always have a NUL at the end, even if we might
1779 * be overwriting an internal NUL character
1780 */
1781 dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
1782 if (unlikely(!name)) {
1783 name = &slash_name;
1784 dname = dentry->d_iname;
1785 } else if (name->len > DNAME_INLINE_LEN-1) {
1786 size_t size = offsetof(struct external_name, name[1]);
1787 struct external_name *p = kmalloc(size + name->len,
1788 GFP_KERNEL_ACCOUNT |
1789 __GFP_RECLAIMABLE);
1790 if (!p) {
1791 kmem_cache_free(dentry_cache, dentry);
1792 return NULL;
1793 }
1794 atomic_set(&p->u.count, 1);
1795 dname = p->name;
1796 } else {
1797 dname = dentry->d_iname;
1798 }
1799
1800 dentry->d_name.len = name->len;
1801 dentry->d_name.hash = name->hash;
1802 memcpy(dname, name->name, name->len);
1803 dname[name->len] = 0;
1804
1805 /* Make sure we always see the terminating NUL character */
1806 smp_store_release(&dentry->d_name.name, dname); /* ^^^ */
1807
1808 dentry->d_lockref.count = 1;
1809 dentry->d_flags = 0;
1810 spin_lock_init(&dentry->d_lock);
1811 seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
1812 dentry->d_inode = NULL;
1813 dentry->d_parent = dentry;
1814 dentry->d_sb = sb;
1815 dentry->d_op = NULL;
1816 dentry->d_fsdata = NULL;
1817 INIT_HLIST_BL_NODE(&dentry->d_hash);
1818 INIT_LIST_HEAD(&dentry->d_lru);
1819 INIT_LIST_HEAD(&dentry->d_subdirs);
1820 INIT_HLIST_NODE(&dentry->d_u.d_alias);
1821 INIT_LIST_HEAD(&dentry->d_child);
1822 d_set_d_op(dentry, dentry->d_sb->s_d_op);
1823
1824 if (dentry->d_op && dentry->d_op->d_init) {
1825 err = dentry->d_op->d_init(dentry);
1826 if (err) {
1827 if (dname_external(dentry))
1828 kfree(external_name(dentry));
1829 kmem_cache_free(dentry_cache, dentry);
1830 return NULL;
1831 }
1832 }
1833
1834 this_cpu_inc(nr_dentry);
1835
1836 return dentry;
1837 }
1838
1839 /**
1840 * d_alloc - allocate a dcache entry
1841 * @parent: parent of entry to allocate
1842 * @name: qstr of the name
1843 *
1844 * Allocates a dentry. It returns %NULL if there is insufficient memory
1845 * available. On a success the dentry is returned. The name passed in is
1846 * copied and the copy passed in may be reused after this call.
1847 */
d_alloc(struct dentry * parent,const struct qstr * name)1848 struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
1849 {
1850 struct dentry *dentry = __d_alloc(parent->d_sb, name);
1851 if (!dentry)
1852 return NULL;
1853 spin_lock(&parent->d_lock);
1854 /*
1855 * don't need child lock because it is not subject
1856 * to concurrency here
1857 */
1858 __dget_dlock(parent);
1859 dentry->d_parent = parent;
1860 list_add(&dentry->d_child, &parent->d_subdirs);
1861 spin_unlock(&parent->d_lock);
1862
1863 return dentry;
1864 }
1865 EXPORT_SYMBOL(d_alloc);
1866
d_alloc_anon(struct super_block * sb)1867 struct dentry *d_alloc_anon(struct super_block *sb)
1868 {
1869 return __d_alloc(sb, NULL);
1870 }
1871 EXPORT_SYMBOL(d_alloc_anon);
1872
d_alloc_cursor(struct dentry * parent)1873 struct dentry *d_alloc_cursor(struct dentry * parent)
1874 {
1875 struct dentry *dentry = d_alloc_anon(parent->d_sb);
1876 if (dentry) {
1877 dentry->d_flags |= DCACHE_DENTRY_CURSOR;
1878 dentry->d_parent = dget(parent);
1879 }
1880 return dentry;
1881 }
1882
1883 /**
1884 * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
1885 * @sb: the superblock
1886 * @name: qstr of the name
1887 *
1888 * For a filesystem that just pins its dentries in memory and never
1889 * performs lookups at all, return an unhashed IS_ROOT dentry.
1890 * This is used for pipes, sockets et.al. - the stuff that should
1891 * never be anyone's children or parents. Unlike all other
1892 * dentries, these will not have RCU delay between dropping the
1893 * last reference and freeing them.
1894 *
1895 * The only user is alloc_file_pseudo() and that's what should
1896 * be considered a public interface. Don't use directly.
1897 */
d_alloc_pseudo(struct super_block * sb,const struct qstr * name)1898 struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
1899 {
1900 struct dentry *dentry = __d_alloc(sb, name);
1901 if (likely(dentry))
1902 dentry->d_flags |= DCACHE_NORCU;
1903 return dentry;
1904 }
1905
d_alloc_name(struct dentry * parent,const char * name)1906 struct dentry *d_alloc_name(struct dentry *parent, const char *name)
1907 {
1908 struct qstr q;
1909
1910 q.name = name;
1911 q.hash_len = hashlen_string(parent, name);
1912 return d_alloc(parent, &q);
1913 }
1914 EXPORT_SYMBOL(d_alloc_name);
1915
d_set_d_op(struct dentry * dentry,const struct dentry_operations * op)1916 void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
1917 {
1918 WARN_ON_ONCE(dentry->d_op);
1919 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
1920 DCACHE_OP_COMPARE |
1921 DCACHE_OP_REVALIDATE |
1922 DCACHE_OP_WEAK_REVALIDATE |
1923 DCACHE_OP_DELETE |
1924 DCACHE_OP_REAL));
1925 dentry->d_op = op;
1926 if (!op)
1927 return;
1928 if (op->d_hash)
1929 dentry->d_flags |= DCACHE_OP_HASH;
1930 if (op->d_compare)
1931 dentry->d_flags |= DCACHE_OP_COMPARE;
1932 if (op->d_revalidate)
1933 dentry->d_flags |= DCACHE_OP_REVALIDATE;
1934 if (op->d_weak_revalidate)
1935 dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
1936 if (op->d_delete)
1937 dentry->d_flags |= DCACHE_OP_DELETE;
1938 if (op->d_prune)
1939 dentry->d_flags |= DCACHE_OP_PRUNE;
1940 if (op->d_real)
1941 dentry->d_flags |= DCACHE_OP_REAL;
1942
1943 }
1944 EXPORT_SYMBOL(d_set_d_op);
1945
1946
1947 /*
1948 * d_set_fallthru - Mark a dentry as falling through to a lower layer
1949 * @dentry - The dentry to mark
1950 *
1951 * Mark a dentry as falling through to the lower layer (as set with
1952 * d_pin_lower()). This flag may be recorded on the medium.
1953 */
d_set_fallthru(struct dentry * dentry)1954 void d_set_fallthru(struct dentry *dentry)
1955 {
1956 spin_lock(&dentry->d_lock);
1957 dentry->d_flags |= DCACHE_FALLTHRU;
1958 spin_unlock(&dentry->d_lock);
1959 }
1960 EXPORT_SYMBOL(d_set_fallthru);
1961
d_flags_for_inode(struct inode * inode)1962 static unsigned d_flags_for_inode(struct inode *inode)
1963 {
1964 unsigned add_flags = DCACHE_REGULAR_TYPE;
1965
1966 if (!inode)
1967 return DCACHE_MISS_TYPE;
1968
1969 if (S_ISDIR(inode->i_mode)) {
1970 add_flags = DCACHE_DIRECTORY_TYPE;
1971 if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
1972 if (unlikely(!inode->i_op->lookup))
1973 add_flags = DCACHE_AUTODIR_TYPE;
1974 else
1975 inode->i_opflags |= IOP_LOOKUP;
1976 }
1977 goto type_determined;
1978 }
1979
1980 if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
1981 if (unlikely(inode->i_op->get_link)) {
1982 add_flags = DCACHE_SYMLINK_TYPE;
1983 goto type_determined;
1984 }
1985 inode->i_opflags |= IOP_NOFOLLOW;
1986 }
1987
1988 if (unlikely(!S_ISREG(inode->i_mode)))
1989 add_flags = DCACHE_SPECIAL_TYPE;
1990
1991 type_determined:
1992 if (unlikely(IS_AUTOMOUNT(inode)))
1993 add_flags |= DCACHE_NEED_AUTOMOUNT;
1994 return add_flags;
1995 }
1996
__d_instantiate(struct dentry * dentry,struct inode * inode)1997 static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1998 {
1999 unsigned add_flags = d_flags_for_inode(inode);
2000 WARN_ON(d_in_lookup(dentry));
2001
2002 spin_lock(&dentry->d_lock);
2003 /*
2004 * Decrement negative dentry count if it was in the LRU list.
2005 */
2006 if (dentry->d_flags & DCACHE_LRU_LIST)
2007 this_cpu_dec(nr_dentry_negative);
2008 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2009 raw_write_seqcount_begin(&dentry->d_seq);
2010 __d_set_inode_and_type(dentry, inode, add_flags);
2011 raw_write_seqcount_end(&dentry->d_seq);
2012 fsnotify_update_flags(dentry);
2013 spin_unlock(&dentry->d_lock);
2014 }
2015
2016 /**
2017 * d_instantiate - fill in inode information for a dentry
2018 * @entry: dentry to complete
2019 * @inode: inode to attach to this dentry
2020 *
2021 * Fill in inode information in the entry.
2022 *
2023 * This turns negative dentries into productive full members
2024 * of society.
2025 *
2026 * NOTE! This assumes that the inode count has been incremented
2027 * (or otherwise set) by the caller to indicate that it is now
2028 * in use by the dcache.
2029 */
2030
d_instantiate(struct dentry * entry,struct inode * inode)2031 void d_instantiate(struct dentry *entry, struct inode * inode)
2032 {
2033 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
2034 if (inode) {
2035 security_d_instantiate(entry, inode);
2036 spin_lock(&inode->i_lock);
2037 __d_instantiate(entry, inode);
2038 spin_unlock(&inode->i_lock);
2039 }
2040 }
2041 EXPORT_SYMBOL(d_instantiate);
2042
2043 /*
2044 * This should be equivalent to d_instantiate() + unlock_new_inode(),
2045 * with lockdep-related part of unlock_new_inode() done before
2046 * anything else. Use that instead of open-coding d_instantiate()/
2047 * unlock_new_inode() combinations.
2048 */
d_instantiate_new(struct dentry * entry,struct inode * inode)2049 void d_instantiate_new(struct dentry *entry, struct inode *inode)
2050 {
2051 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
2052 BUG_ON(!inode);
2053 lockdep_annotate_inode_mutex_key(inode);
2054 security_d_instantiate(entry, inode);
2055 spin_lock(&inode->i_lock);
2056 __d_instantiate(entry, inode);
2057 WARN_ON(!(inode->i_state & I_NEW));
2058 inode->i_state &= ~I_NEW & ~I_CREATING;
2059 smp_mb();
2060 wake_up_bit(&inode->i_state, __I_NEW);
2061 spin_unlock(&inode->i_lock);
2062 }
2063 EXPORT_SYMBOL(d_instantiate_new);
2064
d_make_root(struct inode * root_inode)2065 struct dentry *d_make_root(struct inode *root_inode)
2066 {
2067 struct dentry *res = NULL;
2068
2069 if (root_inode) {
2070 res = d_alloc_anon(root_inode->i_sb);
2071 if (res)
2072 d_instantiate(res, root_inode);
2073 else
2074 iput(root_inode);
2075 }
2076 return res;
2077 }
2078 EXPORT_SYMBOL(d_make_root);
2079
__d_instantiate_anon(struct dentry * dentry,struct inode * inode,bool disconnected)2080 static struct dentry *__d_instantiate_anon(struct dentry *dentry,
2081 struct inode *inode,
2082 bool disconnected)
2083 {
2084 struct dentry *res;
2085 unsigned add_flags;
2086
2087 security_d_instantiate(dentry, inode);
2088 spin_lock(&inode->i_lock);
2089 res = __d_find_any_alias(inode);
2090 if (res) {
2091 spin_unlock(&inode->i_lock);
2092 dput(dentry);
2093 goto out_iput;
2094 }
2095
2096 /* attach a disconnected dentry */
2097 add_flags = d_flags_for_inode(inode);
2098
2099 if (disconnected)
2100 add_flags |= DCACHE_DISCONNECTED;
2101
2102 spin_lock(&dentry->d_lock);
2103 __d_set_inode_and_type(dentry, inode, add_flags);
2104 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2105 if (!disconnected) {
2106 hlist_bl_lock(&dentry->d_sb->s_roots);
2107 hlist_bl_add_head(&dentry->d_hash, &dentry->d_sb->s_roots);
2108 hlist_bl_unlock(&dentry->d_sb->s_roots);
2109 }
2110 spin_unlock(&dentry->d_lock);
2111 spin_unlock(&inode->i_lock);
2112
2113 return dentry;
2114
2115 out_iput:
2116 iput(inode);
2117 return res;
2118 }
2119
d_instantiate_anon(struct dentry * dentry,struct inode * inode)2120 struct dentry *d_instantiate_anon(struct dentry *dentry, struct inode *inode)
2121 {
2122 return __d_instantiate_anon(dentry, inode, true);
2123 }
2124 EXPORT_SYMBOL(d_instantiate_anon);
2125
__d_obtain_alias(struct inode * inode,bool disconnected)2126 static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
2127 {
2128 struct dentry *tmp;
2129 struct dentry *res;
2130
2131 if (!inode)
2132 return ERR_PTR(-ESTALE);
2133 if (IS_ERR(inode))
2134 return ERR_CAST(inode);
2135
2136 res = d_find_any_alias(inode);
2137 if (res)
2138 goto out_iput;
2139
2140 tmp = d_alloc_anon(inode->i_sb);
2141 if (!tmp) {
2142 res = ERR_PTR(-ENOMEM);
2143 goto out_iput;
2144 }
2145
2146 return __d_instantiate_anon(tmp, inode, disconnected);
2147
2148 out_iput:
2149 iput(inode);
2150 return res;
2151 }
2152
2153 /**
2154 * d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode
2155 * @inode: inode to allocate the dentry for
2156 *
2157 * Obtain a dentry for an inode resulting from NFS filehandle conversion or
2158 * similar open by handle operations. The returned dentry may be anonymous,
2159 * or may have a full name (if the inode was already in the cache).
2160 *
2161 * When called on a directory inode, we must ensure that the inode only ever
2162 * has one dentry. If a dentry is found, that is returned instead of
2163 * allocating a new one.
2164 *
2165 * On successful return, the reference to the inode has been transferred
2166 * to the dentry. In case of an error the reference on the inode is released.
2167 * To make it easier to use in export operations a %NULL or IS_ERR inode may
2168 * be passed in and the error will be propagated to the return value,
2169 * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
2170 */
d_obtain_alias(struct inode * inode)2171 struct dentry *d_obtain_alias(struct inode *inode)
2172 {
2173 return __d_obtain_alias(inode, true);
2174 }
2175 EXPORT_SYMBOL(d_obtain_alias);
2176
2177 /**
2178 * d_obtain_root - find or allocate a dentry for a given inode
2179 * @inode: inode to allocate the dentry for
2180 *
2181 * Obtain an IS_ROOT dentry for the root of a filesystem.
2182 *
2183 * We must ensure that directory inodes only ever have one dentry. If a
2184 * dentry is found, that is returned instead of allocating a new one.
2185 *
2186 * On successful return, the reference to the inode has been transferred
2187 * to the dentry. In case of an error the reference on the inode is
2188 * released. A %NULL or IS_ERR inode may be passed in and will be the
2189 * error will be propagate to the return value, with a %NULL @inode
2190 * replaced by ERR_PTR(-ESTALE).
2191 */
d_obtain_root(struct inode * inode)2192 struct dentry *d_obtain_root(struct inode *inode)
2193 {
2194 return __d_obtain_alias(inode, false);
2195 }
2196 EXPORT_SYMBOL(d_obtain_root);
2197
2198 /**
2199 * d_add_ci - lookup or allocate new dentry with case-exact name
2200 * @inode: the inode case-insensitive lookup has found
2201 * @dentry: the negative dentry that was passed to the parent's lookup func
2202 * @name: the case-exact name to be associated with the returned dentry
2203 *
2204 * This is to avoid filling the dcache with case-insensitive names to the
2205 * same inode, only the actual correct case is stored in the dcache for
2206 * case-insensitive filesystems.
2207 *
2208 * For a case-insensitive lookup match and if the case-exact dentry
2209 * already exists in the dcache, use it and return it.
2210 *
2211 * If no entry exists with the exact case name, allocate new dentry with
2212 * the exact case, and return the spliced entry.
2213 */
d_add_ci(struct dentry * dentry,struct inode * inode,struct qstr * name)2214 struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
2215 struct qstr *name)
2216 {
2217 struct dentry *found, *res;
2218
2219 /*
2220 * First check if a dentry matching the name already exists,
2221 * if not go ahead and create it now.
2222 */
2223 found = d_hash_and_lookup(dentry->d_parent, name);
2224 if (found) {
2225 iput(inode);
2226 return found;
2227 }
2228 if (d_in_lookup(dentry)) {
2229 found = d_alloc_parallel(dentry->d_parent, name,
2230 dentry->d_wait);
2231 if (IS_ERR(found) || !d_in_lookup(found)) {
2232 iput(inode);
2233 return found;
2234 }
2235 } else {
2236 found = d_alloc(dentry->d_parent, name);
2237 if (!found) {
2238 iput(inode);
2239 return ERR_PTR(-ENOMEM);
2240 }
2241 }
2242 res = d_splice_alias(inode, found);
2243 if (res) {
2244 d_lookup_done(found);
2245 dput(found);
2246 return res;
2247 }
2248 return found;
2249 }
2250 EXPORT_SYMBOL(d_add_ci);
2251
2252 /**
2253 * d_same_name - compare dentry name with case-exact name
2254 * @parent: parent dentry
2255 * @dentry: the negative dentry that was passed to the parent's lookup func
2256 * @name: the case-exact name to be associated with the returned dentry
2257 *
2258 * Return: true if names are same, or false
2259 */
d_same_name(const struct dentry * dentry,const struct dentry * parent,const struct qstr * name)2260 bool d_same_name(const struct dentry *dentry, const struct dentry *parent,
2261 const struct qstr *name)
2262 {
2263 if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) {
2264 if (dentry->d_name.len != name->len)
2265 return false;
2266 return dentry_cmp(dentry, name->name, name->len) == 0;
2267 }
2268 return parent->d_op->d_compare(dentry,
2269 dentry->d_name.len, dentry->d_name.name,
2270 name) == 0;
2271 }
2272 EXPORT_SYMBOL_GPL(d_same_name);
2273
2274 /*
2275 * This is __d_lookup_rcu() when the parent dentry has
2276 * DCACHE_OP_COMPARE, which makes things much nastier.
2277 */
__d_lookup_rcu_op_compare(const struct dentry * parent,const struct qstr * name,unsigned * seqp)2278 static noinline struct dentry *__d_lookup_rcu_op_compare(
2279 const struct dentry *parent,
2280 const struct qstr *name,
2281 unsigned *seqp)
2282 {
2283 u64 hashlen = name->hash_len;
2284 struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
2285 struct hlist_bl_node *node;
2286 struct dentry *dentry;
2287
2288 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2289 int tlen;
2290 const char *tname;
2291 unsigned seq;
2292
2293 seqretry:
2294 seq = raw_seqcount_begin(&dentry->d_seq);
2295 if (dentry->d_parent != parent)
2296 continue;
2297 if (d_unhashed(dentry))
2298 continue;
2299 if (dentry->d_name.hash != hashlen_hash(hashlen))
2300 continue;
2301 tlen = dentry->d_name.len;
2302 tname = dentry->d_name.name;
2303 /* we want a consistent (name,len) pair */
2304 if (read_seqcount_retry(&dentry->d_seq, seq)) {
2305 cpu_relax();
2306 goto seqretry;
2307 }
2308 if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0)
2309 continue;
2310 *seqp = seq;
2311 return dentry;
2312 }
2313 return NULL;
2314 }
2315
2316 /**
2317 * __d_lookup_rcu - search for a dentry (racy, store-free)
2318 * @parent: parent dentry
2319 * @name: qstr of name we wish to find
2320 * @seqp: returns d_seq value at the point where the dentry was found
2321 * Returns: dentry, or NULL
2322 *
2323 * __d_lookup_rcu is the dcache lookup function for rcu-walk name
2324 * resolution (store-free path walking) design described in
2325 * Documentation/filesystems/path-lookup.txt.
2326 *
2327 * This is not to be used outside core vfs.
2328 *
2329 * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
2330 * held, and rcu_read_lock held. The returned dentry must not be stored into
2331 * without taking d_lock and checking d_seq sequence count against @seq
2332 * returned here.
2333 *
2334 * A refcount may be taken on the found dentry with the d_rcu_to_refcount
2335 * function.
2336 *
2337 * Alternatively, __d_lookup_rcu may be called again to look up the child of
2338 * the returned dentry, so long as its parent's seqlock is checked after the
2339 * child is looked up. Thus, an interlocking stepping of sequence lock checks
2340 * is formed, giving integrity down the path walk.
2341 *
2342 * NOTE! The caller *has* to check the resulting dentry against the sequence
2343 * number we've returned before using any of the resulting dentry state!
2344 */
__d_lookup_rcu(const struct dentry * parent,const struct qstr * name,unsigned * seqp)2345 struct dentry *__d_lookup_rcu(const struct dentry *parent,
2346 const struct qstr *name,
2347 unsigned *seqp)
2348 {
2349 u64 hashlen = name->hash_len;
2350 const unsigned char *str = name->name;
2351 struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
2352 struct hlist_bl_node *node;
2353 struct dentry *dentry;
2354
2355 /*
2356 * Note: There is significant duplication with __d_lookup_rcu which is
2357 * required to prevent single threaded performance regressions
2358 * especially on architectures where smp_rmb (in seqcounts) are costly.
2359 * Keep the two functions in sync.
2360 */
2361
2362 if (unlikely(parent->d_flags & DCACHE_OP_COMPARE))
2363 return __d_lookup_rcu_op_compare(parent, name, seqp);
2364
2365 /*
2366 * The hash list is protected using RCU.
2367 *
2368 * Carefully use d_seq when comparing a candidate dentry, to avoid
2369 * races with d_move().
2370 *
2371 * It is possible that concurrent renames can mess up our list
2372 * walk here and result in missing our dentry, resulting in the
2373 * false-negative result. d_lookup() protects against concurrent
2374 * renames using rename_lock seqlock.
2375 *
2376 * See Documentation/filesystems/path-lookup.txt for more details.
2377 */
2378 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2379 unsigned seq;
2380
2381 /*
2382 * The dentry sequence count protects us from concurrent
2383 * renames, and thus protects parent and name fields.
2384 *
2385 * The caller must perform a seqcount check in order
2386 * to do anything useful with the returned dentry.
2387 *
2388 * NOTE! We do a "raw" seqcount_begin here. That means that
2389 * we don't wait for the sequence count to stabilize if it
2390 * is in the middle of a sequence change. If we do the slow
2391 * dentry compare, we will do seqretries until it is stable,
2392 * and if we end up with a successful lookup, we actually
2393 * want to exit RCU lookup anyway.
2394 *
2395 * Note that raw_seqcount_begin still *does* smp_rmb(), so
2396 * we are still guaranteed NUL-termination of ->d_name.name.
2397 */
2398 seq = raw_seqcount_begin(&dentry->d_seq);
2399 if (dentry->d_parent != parent)
2400 continue;
2401 if (d_unhashed(dentry))
2402 continue;
2403 if (dentry->d_name.hash_len != hashlen)
2404 continue;
2405 if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
2406 continue;
2407 *seqp = seq;
2408 return dentry;
2409 }
2410 return NULL;
2411 }
2412
2413 /**
2414 * d_lookup - search for a dentry
2415 * @parent: parent dentry
2416 * @name: qstr of name we wish to find
2417 * Returns: dentry, or NULL
2418 *
2419 * d_lookup searches the children of the parent dentry for the name in
2420 * question. If the dentry is found its reference count is incremented and the
2421 * dentry is returned. The caller must use dput to free the entry when it has
2422 * finished using it. %NULL is returned if the dentry does not exist.
2423 */
d_lookup(const struct dentry * parent,const struct qstr * name)2424 struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
2425 {
2426 struct dentry *dentry;
2427 unsigned seq;
2428
2429 do {
2430 seq = read_seqbegin(&rename_lock);
2431 dentry = __d_lookup(parent, name);
2432 if (dentry)
2433 break;
2434 } while (read_seqretry(&rename_lock, seq));
2435 return dentry;
2436 }
2437 EXPORT_SYMBOL(d_lookup);
2438
2439 /**
2440 * __d_lookup - search for a dentry (racy)
2441 * @parent: parent dentry
2442 * @name: qstr of name we wish to find
2443 * Returns: dentry, or NULL
2444 *
2445 * __d_lookup is like d_lookup, however it may (rarely) return a
2446 * false-negative result due to unrelated rename activity.
2447 *
2448 * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
2449 * however it must be used carefully, eg. with a following d_lookup in
2450 * the case of failure.
2451 *
2452 * __d_lookup callers must be commented.
2453 */
__d_lookup(const struct dentry * parent,const struct qstr * name)2454 struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
2455 {
2456 unsigned int hash = name->hash;
2457 struct hlist_bl_head *b = d_hash(hash);
2458 struct hlist_bl_node *node;
2459 struct dentry *found = NULL;
2460 struct dentry *dentry;
2461
2462 /*
2463 * Note: There is significant duplication with __d_lookup_rcu which is
2464 * required to prevent single threaded performance regressions
2465 * especially on architectures where smp_rmb (in seqcounts) are costly.
2466 * Keep the two functions in sync.
2467 */
2468
2469 /*
2470 * The hash list is protected using RCU.
2471 *
2472 * Take d_lock when comparing a candidate dentry, to avoid races
2473 * with d_move().
2474 *
2475 * It is possible that concurrent renames can mess up our list
2476 * walk here and result in missing our dentry, resulting in the
2477 * false-negative result. d_lookup() protects against concurrent
2478 * renames using rename_lock seqlock.
2479 *
2480 * See Documentation/filesystems/path-lookup.txt for more details.
2481 */
2482 rcu_read_lock();
2483
2484 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2485
2486 if (dentry->d_name.hash != hash)
2487 continue;
2488
2489 spin_lock(&dentry->d_lock);
2490 if (dentry->d_parent != parent)
2491 goto next;
2492 if (d_unhashed(dentry))
2493 goto next;
2494
2495 if (!d_same_name(dentry, parent, name))
2496 goto next;
2497
2498 dentry->d_lockref.count++;
2499 found = dentry;
2500 spin_unlock(&dentry->d_lock);
2501 break;
2502 next:
2503 spin_unlock(&dentry->d_lock);
2504 }
2505 rcu_read_unlock();
2506
2507 return found;
2508 }
2509
2510 /**
2511 * d_hash_and_lookup - hash the qstr then search for a dentry
2512 * @dir: Directory to search in
2513 * @name: qstr of name we wish to find
2514 *
2515 * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
2516 */
d_hash_and_lookup(struct dentry * dir,struct qstr * name)2517 struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
2518 {
2519 /*
2520 * Check for a fs-specific hash function. Note that we must
2521 * calculate the standard hash first, as the d_op->d_hash()
2522 * routine may choose to leave the hash value unchanged.
2523 */
2524 name->hash = full_name_hash(dir, name->name, name->len);
2525 if (dir->d_flags & DCACHE_OP_HASH) {
2526 int err = dir->d_op->d_hash(dir, name);
2527 if (unlikely(err < 0))
2528 return ERR_PTR(err);
2529 }
2530 return d_lookup(dir, name);
2531 }
2532 EXPORT_SYMBOL(d_hash_and_lookup);
2533
2534 /*
2535 * When a file is deleted, we have two options:
2536 * - turn this dentry into a negative dentry
2537 * - unhash this dentry and free it.
2538 *
2539 * Usually, we want to just turn this into
2540 * a negative dentry, but if anybody else is
2541 * currently using the dentry or the inode
2542 * we can't do that and we fall back on removing
2543 * it from the hash queues and waiting for
2544 * it to be deleted later when it has no users
2545 */
2546
2547 /**
2548 * d_delete - delete a dentry
2549 * @dentry: The dentry to delete
2550 *
2551 * Turn the dentry into a negative dentry if possible, otherwise
2552 * remove it from the hash queues so it can be deleted later
2553 */
2554
d_delete(struct dentry * dentry)2555 void d_delete(struct dentry * dentry)
2556 {
2557 struct inode *inode = dentry->d_inode;
2558
2559 spin_lock(&inode->i_lock);
2560 spin_lock(&dentry->d_lock);
2561 /*
2562 * Are we the only user?
2563 */
2564 if (dentry->d_lockref.count == 1) {
2565 dentry->d_flags &= ~DCACHE_CANT_MOUNT;
2566 dentry_unlink_inode(dentry);
2567 } else {
2568 __d_drop(dentry);
2569 spin_unlock(&dentry->d_lock);
2570 spin_unlock(&inode->i_lock);
2571 }
2572 }
2573 EXPORT_SYMBOL(d_delete);
2574
__d_rehash(struct dentry * entry)2575 static void __d_rehash(struct dentry *entry)
2576 {
2577 struct hlist_bl_head *b = d_hash(entry->d_name.hash);
2578
2579 hlist_bl_lock(b);
2580 hlist_bl_add_head_rcu(&entry->d_hash, b);
2581 hlist_bl_unlock(b);
2582 }
2583
2584 /**
2585 * d_rehash - add an entry back to the hash
2586 * @entry: dentry to add to the hash
2587 *
2588 * Adds a dentry to the hash according to its name.
2589 */
2590
d_rehash(struct dentry * entry)2591 void d_rehash(struct dentry * entry)
2592 {
2593 spin_lock(&entry->d_lock);
2594 __d_rehash(entry);
2595 spin_unlock(&entry->d_lock);
2596 }
2597 EXPORT_SYMBOL(d_rehash);
2598
start_dir_add(struct inode * dir)2599 static inline unsigned start_dir_add(struct inode *dir)
2600 {
2601 preempt_disable_nested();
2602 for (;;) {
2603 unsigned n = dir->i_dir_seq;
2604 if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
2605 return n;
2606 cpu_relax();
2607 }
2608 }
2609
end_dir_add(struct inode * dir,unsigned int n,wait_queue_head_t * d_wait)2610 static inline void end_dir_add(struct inode *dir, unsigned int n,
2611 wait_queue_head_t *d_wait)
2612 {
2613 smp_store_release(&dir->i_dir_seq, n + 2);
2614 preempt_enable_nested();
2615 wake_up_all(d_wait);
2616 }
2617
d_wait_lookup(struct dentry * dentry)2618 static void d_wait_lookup(struct dentry *dentry)
2619 {
2620 if (d_in_lookup(dentry)) {
2621 DECLARE_WAITQUEUE(wait, current);
2622 add_wait_queue(dentry->d_wait, &wait);
2623 do {
2624 set_current_state(TASK_UNINTERRUPTIBLE);
2625 spin_unlock(&dentry->d_lock);
2626 schedule();
2627 spin_lock(&dentry->d_lock);
2628 } while (d_in_lookup(dentry));
2629 }
2630 }
2631
d_alloc_parallel(struct dentry * parent,const struct qstr * name,wait_queue_head_t * wq)2632 struct dentry *d_alloc_parallel(struct dentry *parent,
2633 const struct qstr *name,
2634 wait_queue_head_t *wq)
2635 {
2636 unsigned int hash = name->hash;
2637 struct hlist_bl_head *b = in_lookup_hash(parent, hash);
2638 struct hlist_bl_node *node;
2639 struct dentry *new = d_alloc(parent, name);
2640 struct dentry *dentry;
2641 unsigned seq, r_seq, d_seq;
2642
2643 if (unlikely(!new))
2644 return ERR_PTR(-ENOMEM);
2645
2646 retry:
2647 rcu_read_lock();
2648 seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
2649 r_seq = read_seqbegin(&rename_lock);
2650 dentry = __d_lookup_rcu(parent, name, &d_seq);
2651 if (unlikely(dentry)) {
2652 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2653 rcu_read_unlock();
2654 goto retry;
2655 }
2656 if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
2657 rcu_read_unlock();
2658 dput(dentry);
2659 goto retry;
2660 }
2661 rcu_read_unlock();
2662 dput(new);
2663 return dentry;
2664 }
2665 if (unlikely(read_seqretry(&rename_lock, r_seq))) {
2666 rcu_read_unlock();
2667 goto retry;
2668 }
2669
2670 if (unlikely(seq & 1)) {
2671 rcu_read_unlock();
2672 goto retry;
2673 }
2674
2675 hlist_bl_lock(b);
2676 if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
2677 hlist_bl_unlock(b);
2678 rcu_read_unlock();
2679 goto retry;
2680 }
2681 /*
2682 * No changes for the parent since the beginning of d_lookup().
2683 * Since all removals from the chain happen with hlist_bl_lock(),
2684 * any potential in-lookup matches are going to stay here until
2685 * we unlock the chain. All fields are stable in everything
2686 * we encounter.
2687 */
2688 hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
2689 if (dentry->d_name.hash != hash)
2690 continue;
2691 if (dentry->d_parent != parent)
2692 continue;
2693 if (!d_same_name(dentry, parent, name))
2694 continue;
2695 hlist_bl_unlock(b);
2696 /* now we can try to grab a reference */
2697 if (!lockref_get_not_dead(&dentry->d_lockref)) {
2698 rcu_read_unlock();
2699 goto retry;
2700 }
2701
2702 rcu_read_unlock();
2703 /*
2704 * somebody is likely to be still doing lookup for it;
2705 * wait for them to finish
2706 */
2707 spin_lock(&dentry->d_lock);
2708 d_wait_lookup(dentry);
2709 /*
2710 * it's not in-lookup anymore; in principle we should repeat
2711 * everything from dcache lookup, but it's likely to be what
2712 * d_lookup() would've found anyway. If it is, just return it;
2713 * otherwise we really have to repeat the whole thing.
2714 */
2715 if (unlikely(dentry->d_name.hash != hash))
2716 goto mismatch;
2717 if (unlikely(dentry->d_parent != parent))
2718 goto mismatch;
2719 if (unlikely(d_unhashed(dentry)))
2720 goto mismatch;
2721 if (unlikely(!d_same_name(dentry, parent, name)))
2722 goto mismatch;
2723 /* OK, it *is* a hashed match; return it */
2724 spin_unlock(&dentry->d_lock);
2725 dput(new);
2726 return dentry;
2727 }
2728 rcu_read_unlock();
2729 /* we can't take ->d_lock here; it's OK, though. */
2730 new->d_flags |= DCACHE_PAR_LOOKUP;
2731 new->d_wait = wq;
2732 hlist_bl_add_head_rcu(&new->d_u.d_in_lookup_hash, b);
2733 hlist_bl_unlock(b);
2734 return new;
2735 mismatch:
2736 spin_unlock(&dentry->d_lock);
2737 dput(dentry);
2738 goto retry;
2739 }
2740 EXPORT_SYMBOL(d_alloc_parallel);
2741
2742 /*
2743 * - Unhash the dentry
2744 * - Retrieve and clear the waitqueue head in dentry
2745 * - Return the waitqueue head
2746 */
__d_lookup_unhash(struct dentry * dentry)2747 static wait_queue_head_t *__d_lookup_unhash(struct dentry *dentry)
2748 {
2749 wait_queue_head_t *d_wait;
2750 struct hlist_bl_head *b;
2751
2752 lockdep_assert_held(&dentry->d_lock);
2753
2754 b = in_lookup_hash(dentry->d_parent, dentry->d_name.hash);
2755 hlist_bl_lock(b);
2756 dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
2757 __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
2758 d_wait = dentry->d_wait;
2759 dentry->d_wait = NULL;
2760 hlist_bl_unlock(b);
2761 INIT_HLIST_NODE(&dentry->d_u.d_alias);
2762 INIT_LIST_HEAD(&dentry->d_lru);
2763 return d_wait;
2764 }
2765
__d_lookup_unhash_wake(struct dentry * dentry)2766 void __d_lookup_unhash_wake(struct dentry *dentry)
2767 {
2768 spin_lock(&dentry->d_lock);
2769 wake_up_all(__d_lookup_unhash(dentry));
2770 spin_unlock(&dentry->d_lock);
2771 }
2772 EXPORT_SYMBOL(__d_lookup_unhash_wake);
2773
2774 /* inode->i_lock held if inode is non-NULL */
2775
__d_add(struct dentry * dentry,struct inode * inode)2776 static inline void __d_add(struct dentry *dentry, struct inode *inode)
2777 {
2778 wait_queue_head_t *d_wait;
2779 struct inode *dir = NULL;
2780 unsigned n;
2781 spin_lock(&dentry->d_lock);
2782 if (unlikely(d_in_lookup(dentry))) {
2783 dir = dentry->d_parent->d_inode;
2784 n = start_dir_add(dir);
2785 d_wait = __d_lookup_unhash(dentry);
2786 }
2787 if (inode) {
2788 unsigned add_flags = d_flags_for_inode(inode);
2789 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2790 raw_write_seqcount_begin(&dentry->d_seq);
2791 __d_set_inode_and_type(dentry, inode, add_flags);
2792 raw_write_seqcount_end(&dentry->d_seq);
2793 fsnotify_update_flags(dentry);
2794 }
2795 __d_rehash(dentry);
2796 if (dir)
2797 end_dir_add(dir, n, d_wait);
2798 spin_unlock(&dentry->d_lock);
2799 if (inode)
2800 spin_unlock(&inode->i_lock);
2801 }
2802
2803 /**
2804 * d_add - add dentry to hash queues
2805 * @entry: dentry to add
2806 * @inode: The inode to attach to this dentry
2807 *
2808 * This adds the entry to the hash queues and initializes @inode.
2809 * The entry was actually filled in earlier during d_alloc().
2810 */
2811
d_add(struct dentry * entry,struct inode * inode)2812 void d_add(struct dentry *entry, struct inode *inode)
2813 {
2814 if (inode) {
2815 security_d_instantiate(entry, inode);
2816 spin_lock(&inode->i_lock);
2817 }
2818 __d_add(entry, inode);
2819 }
2820 EXPORT_SYMBOL(d_add);
2821
2822 /**
2823 * d_exact_alias - find and hash an exact unhashed alias
2824 * @entry: dentry to add
2825 * @inode: The inode to go with this dentry
2826 *
2827 * If an unhashed dentry with the same name/parent and desired
2828 * inode already exists, hash and return it. Otherwise, return
2829 * NULL.
2830 *
2831 * Parent directory should be locked.
2832 */
d_exact_alias(struct dentry * entry,struct inode * inode)2833 struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode)
2834 {
2835 struct dentry *alias;
2836 unsigned int hash = entry->d_name.hash;
2837
2838 spin_lock(&inode->i_lock);
2839 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
2840 /*
2841 * Don't need alias->d_lock here, because aliases with
2842 * d_parent == entry->d_parent are not subject to name or
2843 * parent changes, because the parent inode i_mutex is held.
2844 */
2845 if (alias->d_name.hash != hash)
2846 continue;
2847 if (alias->d_parent != entry->d_parent)
2848 continue;
2849 if (!d_same_name(alias, entry->d_parent, &entry->d_name))
2850 continue;
2851 spin_lock(&alias->d_lock);
2852 if (!d_unhashed(alias)) {
2853 spin_unlock(&alias->d_lock);
2854 alias = NULL;
2855 } else {
2856 __dget_dlock(alias);
2857 __d_rehash(alias);
2858 spin_unlock(&alias->d_lock);
2859 }
2860 spin_unlock(&inode->i_lock);
2861 return alias;
2862 }
2863 spin_unlock(&inode->i_lock);
2864 return NULL;
2865 }
2866 EXPORT_SYMBOL(d_exact_alias);
2867
swap_names(struct dentry * dentry,struct dentry * target)2868 static void swap_names(struct dentry *dentry, struct dentry *target)
2869 {
2870 if (unlikely(dname_external(target))) {
2871 if (unlikely(dname_external(dentry))) {
2872 /*
2873 * Both external: swap the pointers
2874 */
2875 swap(target->d_name.name, dentry->d_name.name);
2876 } else {
2877 /*
2878 * dentry:internal, target:external. Steal target's
2879 * storage and make target internal.
2880 */
2881 memcpy(target->d_iname, dentry->d_name.name,
2882 dentry->d_name.len + 1);
2883 dentry->d_name.name = target->d_name.name;
2884 target->d_name.name = target->d_iname;
2885 }
2886 } else {
2887 if (unlikely(dname_external(dentry))) {
2888 /*
2889 * dentry:external, target:internal. Give dentry's
2890 * storage to target and make dentry internal
2891 */
2892 memcpy(dentry->d_iname, target->d_name.name,
2893 target->d_name.len + 1);
2894 target->d_name.name = dentry->d_name.name;
2895 dentry->d_name.name = dentry->d_iname;
2896 } else {
2897 /*
2898 * Both are internal.
2899 */
2900 unsigned int i;
2901 BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
2902 for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
2903 swap(((long *) &dentry->d_iname)[i],
2904 ((long *) &target->d_iname)[i]);
2905 }
2906 }
2907 }
2908 swap(dentry->d_name.hash_len, target->d_name.hash_len);
2909 }
2910
copy_name(struct dentry * dentry,struct dentry * target)2911 static void copy_name(struct dentry *dentry, struct dentry *target)
2912 {
2913 struct external_name *old_name = NULL;
2914 if (unlikely(dname_external(dentry)))
2915 old_name = external_name(dentry);
2916 if (unlikely(dname_external(target))) {
2917 atomic_inc(&external_name(target)->u.count);
2918 dentry->d_name = target->d_name;
2919 } else {
2920 memcpy(dentry->d_iname, target->d_name.name,
2921 target->d_name.len + 1);
2922 dentry->d_name.name = dentry->d_iname;
2923 dentry->d_name.hash_len = target->d_name.hash_len;
2924 }
2925 if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
2926 kfree_rcu(old_name, u.head);
2927 }
2928
2929 /*
2930 * __d_move - move a dentry
2931 * @dentry: entry to move
2932 * @target: new dentry
2933 * @exchange: exchange the two dentries
2934 *
2935 * Update the dcache to reflect the move of a file name. Negative
2936 * dcache entries should not be moved in this way. Caller must hold
2937 * rename_lock, the i_mutex of the source and target directories,
2938 * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
2939 */
__d_move(struct dentry * dentry,struct dentry * target,bool exchange)2940 static void __d_move(struct dentry *dentry, struct dentry *target,
2941 bool exchange)
2942 {
2943 struct dentry *old_parent, *p;
2944 wait_queue_head_t *d_wait;
2945 struct inode *dir = NULL;
2946 unsigned n;
2947
2948 WARN_ON(!dentry->d_inode);
2949 if (WARN_ON(dentry == target))
2950 return;
2951
2952 BUG_ON(d_ancestor(target, dentry));
2953 old_parent = dentry->d_parent;
2954 p = d_ancestor(old_parent, target);
2955 if (IS_ROOT(dentry)) {
2956 BUG_ON(p);
2957 spin_lock(&target->d_parent->d_lock);
2958 } else if (!p) {
2959 /* target is not a descendent of dentry->d_parent */
2960 spin_lock(&target->d_parent->d_lock);
2961 spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED);
2962 } else {
2963 BUG_ON(p == dentry);
2964 spin_lock(&old_parent->d_lock);
2965 if (p != target)
2966 spin_lock_nested(&target->d_parent->d_lock,
2967 DENTRY_D_LOCK_NESTED);
2968 }
2969 spin_lock_nested(&dentry->d_lock, 2);
2970 spin_lock_nested(&target->d_lock, 3);
2971
2972 if (unlikely(d_in_lookup(target))) {
2973 dir = target->d_parent->d_inode;
2974 n = start_dir_add(dir);
2975 d_wait = __d_lookup_unhash(target);
2976 }
2977
2978 write_seqcount_begin(&dentry->d_seq);
2979 write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
2980
2981 /* unhash both */
2982 if (!d_unhashed(dentry))
2983 ___d_drop(dentry);
2984 if (!d_unhashed(target))
2985 ___d_drop(target);
2986
2987 /* ... and switch them in the tree */
2988 dentry->d_parent = target->d_parent;
2989 if (!exchange) {
2990 copy_name(dentry, target);
2991 target->d_hash.pprev = NULL;
2992 dentry->d_parent->d_lockref.count++;
2993 if (dentry != old_parent) /* wasn't IS_ROOT */
2994 WARN_ON(!--old_parent->d_lockref.count);
2995 } else {
2996 target->d_parent = old_parent;
2997 swap_names(dentry, target);
2998 list_move(&target->d_child, &target->d_parent->d_subdirs);
2999 __d_rehash(target);
3000 fsnotify_update_flags(target);
3001 }
3002 list_move(&dentry->d_child, &dentry->d_parent->d_subdirs);
3003 __d_rehash(dentry);
3004 fsnotify_update_flags(dentry);
3005 fscrypt_handle_d_move(dentry);
3006
3007 write_seqcount_end(&target->d_seq);
3008 write_seqcount_end(&dentry->d_seq);
3009
3010 if (dir)
3011 end_dir_add(dir, n, d_wait);
3012
3013 if (dentry->d_parent != old_parent)
3014 spin_unlock(&dentry->d_parent->d_lock);
3015 if (dentry != old_parent)
3016 spin_unlock(&old_parent->d_lock);
3017 spin_unlock(&target->d_lock);
3018 spin_unlock(&dentry->d_lock);
3019 }
3020
3021 /*
3022 * d_move - move a dentry
3023 * @dentry: entry to move
3024 * @target: new dentry
3025 *
3026 * Update the dcache to reflect the move of a file name. Negative
3027 * dcache entries should not be moved in this way. See the locking
3028 * requirements for __d_move.
3029 */
d_move(struct dentry * dentry,struct dentry * target)3030 void d_move(struct dentry *dentry, struct dentry *target)
3031 {
3032 write_seqlock(&rename_lock);
3033 __d_move(dentry, target, false);
3034 write_sequnlock(&rename_lock);
3035 }
3036 EXPORT_SYMBOL(d_move);
3037
3038 /*
3039 * d_exchange - exchange two dentries
3040 * @dentry1: first dentry
3041 * @dentry2: second dentry
3042 */
d_exchange(struct dentry * dentry1,struct dentry * dentry2)3043 void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
3044 {
3045 write_seqlock(&rename_lock);
3046
3047 WARN_ON(!dentry1->d_inode);
3048 WARN_ON(!dentry2->d_inode);
3049 WARN_ON(IS_ROOT(dentry1));
3050 WARN_ON(IS_ROOT(dentry2));
3051
3052 __d_move(dentry1, dentry2, true);
3053
3054 write_sequnlock(&rename_lock);
3055 }
3056
3057 /**
3058 * d_ancestor - search for an ancestor
3059 * @p1: ancestor dentry
3060 * @p2: child dentry
3061 *
3062 * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
3063 * an ancestor of p2, else NULL.
3064 */
d_ancestor(struct dentry * p1,struct dentry * p2)3065 struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
3066 {
3067 struct dentry *p;
3068
3069 for (p = p2; !IS_ROOT(p); p = p->d_parent) {
3070 if (p->d_parent == p1)
3071 return p;
3072 }
3073 return NULL;
3074 }
3075
3076 /*
3077 * This helper attempts to cope with remotely renamed directories
3078 *
3079 * It assumes that the caller is already holding
3080 * dentry->d_parent->d_inode->i_mutex, and rename_lock
3081 *
3082 * Note: If ever the locking in lock_rename() changes, then please
3083 * remember to update this too...
3084 */
__d_unalias(struct inode * inode,struct dentry * dentry,struct dentry * alias)3085 static int __d_unalias(struct inode *inode,
3086 struct dentry *dentry, struct dentry *alias)
3087 {
3088 struct mutex *m1 = NULL;
3089 struct rw_semaphore *m2 = NULL;
3090 int ret = -ESTALE;
3091
3092 /* If alias and dentry share a parent, then no extra locks required */
3093 if (alias->d_parent == dentry->d_parent)
3094 goto out_unalias;
3095
3096 /* See lock_rename() */
3097 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
3098 goto out_err;
3099 m1 = &dentry->d_sb->s_vfs_rename_mutex;
3100 if (!inode_trylock_shared(alias->d_parent->d_inode))
3101 goto out_err;
3102 m2 = &alias->d_parent->d_inode->i_rwsem;
3103 out_unalias:
3104 __d_move(alias, dentry, false);
3105 ret = 0;
3106 out_err:
3107 if (m2)
3108 up_read(m2);
3109 if (m1)
3110 mutex_unlock(m1);
3111 return ret;
3112 }
3113
3114 /**
3115 * d_splice_alias - splice a disconnected dentry into the tree if one exists
3116 * @inode: the inode which may have a disconnected dentry
3117 * @dentry: a negative dentry which we want to point to the inode.
3118 *
3119 * If inode is a directory and has an IS_ROOT alias, then d_move that in
3120 * place of the given dentry and return it, else simply d_add the inode
3121 * to the dentry and return NULL.
3122 *
3123 * If a non-IS_ROOT directory is found, the filesystem is corrupt, and
3124 * we should error out: directories can't have multiple aliases.
3125 *
3126 * This is needed in the lookup routine of any filesystem that is exportable
3127 * (via knfsd) so that we can build dcache paths to directories effectively.
3128 *
3129 * If a dentry was found and moved, then it is returned. Otherwise NULL
3130 * is returned. This matches the expected return value of ->lookup.
3131 *
3132 * Cluster filesystems may call this function with a negative, hashed dentry.
3133 * In that case, we know that the inode will be a regular file, and also this
3134 * will only occur during atomic_open. So we need to check for the dentry
3135 * being already hashed only in the final case.
3136 */
d_splice_alias(struct inode * inode,struct dentry * dentry)3137 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
3138 {
3139 if (IS_ERR(inode))
3140 return ERR_CAST(inode);
3141
3142 BUG_ON(!d_unhashed(dentry));
3143
3144 if (!inode)
3145 goto out;
3146
3147 security_d_instantiate(dentry, inode);
3148 spin_lock(&inode->i_lock);
3149 if (S_ISDIR(inode->i_mode)) {
3150 struct dentry *new = __d_find_any_alias(inode);
3151 if (unlikely(new)) {
3152 /* The reference to new ensures it remains an alias */
3153 spin_unlock(&inode->i_lock);
3154 write_seqlock(&rename_lock);
3155 if (unlikely(d_ancestor(new, dentry))) {
3156 write_sequnlock(&rename_lock);
3157 dput(new);
3158 new = ERR_PTR(-ELOOP);
3159 pr_warn_ratelimited(
3160 "VFS: Lookup of '%s' in %s %s"
3161 " would have caused loop\n",
3162 dentry->d_name.name,
3163 inode->i_sb->s_type->name,
3164 inode->i_sb->s_id);
3165 } else if (!IS_ROOT(new)) {
3166 struct dentry *old_parent = dget(new->d_parent);
3167 int err = __d_unalias(inode, dentry, new);
3168 write_sequnlock(&rename_lock);
3169 if (err) {
3170 dput(new);
3171 new = ERR_PTR(err);
3172 }
3173 dput(old_parent);
3174 } else {
3175 __d_move(new, dentry, false);
3176 write_sequnlock(&rename_lock);
3177 }
3178 iput(inode);
3179 return new;
3180 }
3181 }
3182 out:
3183 __d_add(dentry, inode);
3184 return NULL;
3185 }
3186 EXPORT_SYMBOL(d_splice_alias);
3187
3188 /*
3189 * Test whether new_dentry is a subdirectory of old_dentry.
3190 *
3191 * Trivially implemented using the dcache structure
3192 */
3193
3194 /**
3195 * is_subdir - is new dentry a subdirectory of old_dentry
3196 * @new_dentry: new dentry
3197 * @old_dentry: old dentry
3198 *
3199 * Returns true if new_dentry is a subdirectory of the parent (at any depth).
3200 * Returns false otherwise.
3201 * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
3202 */
3203
is_subdir(struct dentry * new_dentry,struct dentry * old_dentry)3204 bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
3205 {
3206 bool result;
3207 unsigned seq;
3208
3209 if (new_dentry == old_dentry)
3210 return true;
3211
3212 do {
3213 /* for restarting inner loop in case of seq retry */
3214 seq = read_seqbegin(&rename_lock);
3215 /*
3216 * Need rcu_readlock to protect against the d_parent trashing
3217 * due to d_move
3218 */
3219 rcu_read_lock();
3220 if (d_ancestor(old_dentry, new_dentry))
3221 result = true;
3222 else
3223 result = false;
3224 rcu_read_unlock();
3225 } while (read_seqretry(&rename_lock, seq));
3226
3227 return result;
3228 }
3229 EXPORT_SYMBOL(is_subdir);
3230
d_genocide_kill(void * data,struct dentry * dentry)3231 static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
3232 {
3233 struct dentry *root = data;
3234 if (dentry != root) {
3235 if (d_unhashed(dentry) || !dentry->d_inode)
3236 return D_WALK_SKIP;
3237
3238 if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
3239 dentry->d_flags |= DCACHE_GENOCIDE;
3240 dentry->d_lockref.count--;
3241 }
3242 }
3243 return D_WALK_CONTINUE;
3244 }
3245
d_genocide(struct dentry * parent)3246 void d_genocide(struct dentry *parent)
3247 {
3248 d_walk(parent, parent, d_genocide_kill);
3249 }
3250
3251 EXPORT_SYMBOL(d_genocide);
3252
d_tmpfile(struct file * file,struct inode * inode)3253 void d_tmpfile(struct file *file, struct inode *inode)
3254 {
3255 struct dentry *dentry = file->f_path.dentry;
3256
3257 inode_dec_link_count(inode);
3258 BUG_ON(dentry->d_name.name != dentry->d_iname ||
3259 !hlist_unhashed(&dentry->d_u.d_alias) ||
3260 !d_unlinked(dentry));
3261 spin_lock(&dentry->d_parent->d_lock);
3262 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
3263 dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
3264 (unsigned long long)inode->i_ino);
3265 spin_unlock(&dentry->d_lock);
3266 spin_unlock(&dentry->d_parent->d_lock);
3267 d_instantiate(dentry, inode);
3268 }
3269 EXPORT_SYMBOL(d_tmpfile);
3270
3271 static __initdata unsigned long dhash_entries;
set_dhash_entries(char * str)3272 static int __init set_dhash_entries(char *str)
3273 {
3274 if (!str)
3275 return 0;
3276 dhash_entries = simple_strtoul(str, &str, 0);
3277 return 1;
3278 }
3279 __setup("dhash_entries=", set_dhash_entries);
3280
dcache_init_early(void)3281 static void __init dcache_init_early(void)
3282 {
3283 /* If hashes are distributed across NUMA nodes, defer
3284 * hash allocation until vmalloc space is available.
3285 */
3286 if (hashdist)
3287 return;
3288
3289 dentry_hashtable =
3290 alloc_large_system_hash("Dentry cache",
3291 sizeof(struct hlist_bl_head),
3292 dhash_entries,
3293 13,
3294 HASH_EARLY | HASH_ZERO,
3295 &d_hash_shift,
3296 NULL,
3297 0,
3298 0);
3299 d_hash_shift = 32 - d_hash_shift;
3300 }
3301
dcache_init(void)3302 static void __init dcache_init(void)
3303 {
3304 /*
3305 * A constructor could be added for stable state like the lists,
3306 * but it is probably not worth it because of the cache nature
3307 * of the dcache.
3308 */
3309 dentry_cache = KMEM_CACHE_USERCOPY(dentry,
3310 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD|SLAB_ACCOUNT,
3311 d_iname);
3312
3313 /* Hash may have been set up in dcache_init_early */
3314 if (!hashdist)
3315 return;
3316
3317 dentry_hashtable =
3318 alloc_large_system_hash("Dentry cache",
3319 sizeof(struct hlist_bl_head),
3320 dhash_entries,
3321 13,
3322 HASH_ZERO,
3323 &d_hash_shift,
3324 NULL,
3325 0,
3326 0);
3327 d_hash_shift = 32 - d_hash_shift;
3328 }
3329
3330 /* SLAB cache for __getname() consumers */
3331 struct kmem_cache *names_cachep __read_mostly;
3332 EXPORT_SYMBOL(names_cachep);
3333
vfs_caches_init_early(void)3334 void __init vfs_caches_init_early(void)
3335 {
3336 int i;
3337
3338 for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
3339 INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
3340
3341 dcache_init_early();
3342 inode_init_early();
3343 }
3344
vfs_caches_init(void)3345 void __init vfs_caches_init(void)
3346 {
3347 names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
3348 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
3349
3350 dcache_init();
3351 inode_init();
3352 files_init();
3353 files_maxfiles_init();
3354 mnt_init();
3355 bdev_cache_init();
3356 chrdev_init();
3357 }
3358