• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * (C) 1997 Linus Torvalds
4  * (C) 1999 Andrea Arcangeli <andrea@suse.de> (dynamic inode allocation)
5  */
6 #include <linux/export.h>
7 #include <linux/fs.h>
8 #include <linux/mm.h>
9 #include <linux/backing-dev.h>
10 #include <linux/hash.h>
11 #include <linux/swap.h>
12 #include <linux/security.h>
13 #include <linux/cdev.h>
14 #include <linux/memblock.h>
15 #include <linux/fscrypt.h>
16 #include <linux/fsnotify.h>
17 #include <linux/mount.h>
18 #include <linux/posix_acl.h>
19 #include <linux/prefetch.h>
20 #include <linux/buffer_head.h> /* for inode_has_buffers */
21 #include <linux/ratelimit.h>
22 #include <linux/list_lru.h>
23 #include <linux/iversion.h>
24 #include <linux/xpm.h>
25 #include <trace/events/writeback.h>
26 #include "internal.h"
27 
28 /*
29  * Inode locking rules:
30  *
31  * inode->i_lock protects:
32  *   inode->i_state, inode->i_hash, __iget()
33  * Inode LRU list locks protect:
34  *   inode->i_sb->s_inode_lru, inode->i_lru
35  * inode->i_sb->s_inode_list_lock protects:
36  *   inode->i_sb->s_inodes, inode->i_sb_list
37  * bdi->wb.list_lock protects:
38  *   bdi->wb.b_{dirty,io,more_io,dirty_time}, inode->i_io_list
39  * inode_hash_lock protects:
40  *   inode_hashtable, inode->i_hash
41  *
42  * Lock ordering:
43  *
44  * inode->i_sb->s_inode_list_lock
45  *   inode->i_lock
46  *     Inode LRU list locks
47  *
48  * bdi->wb.list_lock
49  *   inode->i_lock
50  *
51  * inode_hash_lock
52  *   inode->i_sb->s_inode_list_lock
53  *   inode->i_lock
54  *
55  * iunique_lock
56  *   inode_hash_lock
57  */
58 
59 static unsigned int i_hash_mask __read_mostly;
60 static unsigned int i_hash_shift __read_mostly;
61 static struct hlist_head *inode_hashtable __read_mostly;
62 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
63 
64 /*
65  * Empty aops. Can be used for the cases where the user does not
66  * define any of the address_space operations.
67  */
68 const struct address_space_operations empty_aops = {
69 };
70 EXPORT_SYMBOL(empty_aops);
71 
72 /*
73  * Statistics gathering..
74  */
75 struct inodes_stat_t inodes_stat;
76 
77 static DEFINE_PER_CPU(unsigned long, nr_inodes);
78 static DEFINE_PER_CPU(unsigned long, nr_unused);
79 
80 static struct kmem_cache *inode_cachep __read_mostly;
81 
get_nr_inodes(void)82 static long get_nr_inodes(void)
83 {
84 	int i;
85 	long sum = 0;
86 	for_each_possible_cpu(i)
87 		sum += per_cpu(nr_inodes, i);
88 	return sum < 0 ? 0 : sum;
89 }
90 
get_nr_inodes_unused(void)91 static inline long get_nr_inodes_unused(void)
92 {
93 	int i;
94 	long sum = 0;
95 	for_each_possible_cpu(i)
96 		sum += per_cpu(nr_unused, i);
97 	return sum < 0 ? 0 : sum;
98 }
99 
get_nr_dirty_inodes(void)100 long get_nr_dirty_inodes(void)
101 {
102 	/* not actually dirty inodes, but a wild approximation */
103 	long nr_dirty = get_nr_inodes() - get_nr_inodes_unused();
104 	return nr_dirty > 0 ? nr_dirty : 0;
105 }
106 
107 /*
108  * Handle nr_inode sysctl
109  */
110 #ifdef CONFIG_SYSCTL
proc_nr_inodes(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)111 int proc_nr_inodes(struct ctl_table *table, int write,
112 		   void *buffer, size_t *lenp, loff_t *ppos)
113 {
114 	inodes_stat.nr_inodes = get_nr_inodes();
115 	inodes_stat.nr_unused = get_nr_inodes_unused();
116 	return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
117 }
118 #endif
119 
no_open(struct inode * inode,struct file * file)120 static int no_open(struct inode *inode, struct file *file)
121 {
122 	return -ENXIO;
123 }
124 
125 /**
126  * inode_init_always - perform inode structure initialisation
127  * @sb: superblock inode belongs to
128  * @inode: inode to initialise
129  *
130  * These are initializations that need to be done on every inode
131  * allocation as the fields are not initialised by slab allocation.
132  */
inode_init_always(struct super_block * sb,struct inode * inode)133 int inode_init_always(struct super_block *sb, struct inode *inode)
134 {
135 	static const struct inode_operations empty_iops;
136 	static const struct file_operations no_open_fops = {.open = no_open};
137 	struct address_space *const mapping = &inode->i_data;
138 
139 	inode->i_sb = sb;
140 	inode->i_blkbits = sb->s_blocksize_bits;
141 	inode->i_flags = 0;
142 	atomic64_set(&inode->i_sequence, 0);
143 	atomic_set(&inode->i_count, 1);
144 	inode->i_op = &empty_iops;
145 	inode->i_fop = &no_open_fops;
146 	inode->__i_nlink = 1;
147 	inode->i_opflags = 0;
148 	if (sb->s_xattr)
149 		inode->i_opflags |= IOP_XATTR;
150 	i_uid_write(inode, 0);
151 	i_gid_write(inode, 0);
152 	atomic_set(&inode->i_writecount, 0);
153 	inode->i_size = 0;
154 	inode->i_write_hint = WRITE_LIFE_NOT_SET;
155 	inode->i_blocks = 0;
156 	inode->i_bytes = 0;
157 	inode->i_generation = 0;
158 	inode->i_pipe = NULL;
159 	inode->i_bdev = NULL;
160 	inode->i_cdev = NULL;
161 	inode->i_link = NULL;
162 	inode->i_dir_seq = 0;
163 	inode->i_rdev = 0;
164 	inode->dirtied_when = 0;
165 
166 #ifdef CONFIG_CGROUP_WRITEBACK
167 	inode->i_wb_frn_winner = 0;
168 	inode->i_wb_frn_avg_time = 0;
169 	inode->i_wb_frn_history = 0;
170 #endif
171 
172 	spin_lock_init(&inode->i_lock);
173 	lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
174 
175 	init_rwsem(&inode->i_rwsem);
176 	lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key);
177 
178 	atomic_set(&inode->i_dio_count, 0);
179 
180 	mapping->a_ops = &empty_aops;
181 	mapping->host = inode;
182 	mapping->flags = 0;
183 	if (sb->s_type->fs_flags & FS_THP_SUPPORT)
184 		__set_bit(AS_THP_SUPPORT, &mapping->flags);
185 	mapping->wb_err = 0;
186 	atomic_set(&mapping->i_mmap_writable, 0);
187 #ifdef CONFIG_READ_ONLY_THP_FOR_FS
188 	atomic_set(&mapping->nr_thps, 0);
189 #endif
190 	mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
191 	mapping->private_data = NULL;
192 	mapping->writeback_index = 0;
193 	inode->i_private = NULL;
194 	inode->i_mapping = mapping;
195 	INIT_HLIST_HEAD(&inode->i_dentry);	/* buggered by rcu freeing */
196 #ifdef CONFIG_FS_POSIX_ACL
197 	inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED;
198 #endif
199 
200 #ifdef CONFIG_FSNOTIFY
201 	inode->i_fsnotify_mask = 0;
202 #endif
203 	inode->i_flctx = NULL;
204 
205 	if (unlikely(security_inode_alloc(inode)))
206 		return -ENOMEM;
207 	this_cpu_inc(nr_inodes);
208 
209 	return 0;
210 }
211 EXPORT_SYMBOL(inode_init_always);
212 
free_inode_nonrcu(struct inode * inode)213 void free_inode_nonrcu(struct inode *inode)
214 {
215 	kmem_cache_free(inode_cachep, inode);
216 }
217 EXPORT_SYMBOL(free_inode_nonrcu);
218 
i_callback(struct rcu_head * head)219 static void i_callback(struct rcu_head *head)
220 {
221 	struct inode *inode = container_of(head, struct inode, i_rcu);
222 	if (inode->free_inode)
223 		inode->free_inode(inode);
224 	else
225 		free_inode_nonrcu(inode);
226 }
227 
alloc_inode(struct super_block * sb)228 static struct inode *alloc_inode(struct super_block *sb)
229 {
230 	const struct super_operations *ops = sb->s_op;
231 	struct inode *inode;
232 
233 	if (ops->alloc_inode)
234 		inode = ops->alloc_inode(sb);
235 	else
236 		inode = kmem_cache_alloc(inode_cachep, GFP_KERNEL);
237 
238 	if (!inode)
239 		return NULL;
240 
241 	if (unlikely(inode_init_always(sb, inode))) {
242 		if (ops->destroy_inode) {
243 			ops->destroy_inode(inode);
244 			if (!ops->free_inode)
245 				return NULL;
246 		}
247 		inode->free_inode = ops->free_inode;
248 		i_callback(&inode->i_rcu);
249 		return NULL;
250 	}
251 
252 	return inode;
253 }
254 
__destroy_inode(struct inode * inode)255 void __destroy_inode(struct inode *inode)
256 {
257 	BUG_ON(inode_has_buffers(inode));
258 	inode_detach_wb(inode);
259 	security_inode_free(inode);
260 	fsnotify_inode_delete(inode);
261 	locks_free_lock_context(inode);
262 	xpm_delete_cache_node_hook(inode);
263 	if (!inode->i_nlink) {
264 		WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0);
265 		atomic_long_dec(&inode->i_sb->s_remove_count);
266 	}
267 
268 #ifdef CONFIG_FS_POSIX_ACL
269 	if (inode->i_acl && !is_uncached_acl(inode->i_acl))
270 		posix_acl_release(inode->i_acl);
271 	if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl))
272 		posix_acl_release(inode->i_default_acl);
273 #endif
274 	this_cpu_dec(nr_inodes);
275 }
276 EXPORT_SYMBOL(__destroy_inode);
277 
destroy_inode(struct inode * inode)278 static void destroy_inode(struct inode *inode)
279 {
280 	const struct super_operations *ops = inode->i_sb->s_op;
281 
282 	BUG_ON(!list_empty(&inode->i_lru));
283 	__destroy_inode(inode);
284 	if (ops->destroy_inode) {
285 		ops->destroy_inode(inode);
286 		if (!ops->free_inode)
287 			return;
288 	}
289 	inode->free_inode = ops->free_inode;
290 	call_rcu(&inode->i_rcu, i_callback);
291 }
292 
293 /**
294  * drop_nlink - directly drop an inode's link count
295  * @inode: inode
296  *
297  * This is a low-level filesystem helper to replace any
298  * direct filesystem manipulation of i_nlink.  In cases
299  * where we are attempting to track writes to the
300  * filesystem, a decrement to zero means an imminent
301  * write when the file is truncated and actually unlinked
302  * on the filesystem.
303  */
drop_nlink(struct inode * inode)304 void drop_nlink(struct inode *inode)
305 {
306 	WARN_ON(inode->i_nlink == 0);
307 	inode->__i_nlink--;
308 	if (!inode->i_nlink)
309 		atomic_long_inc(&inode->i_sb->s_remove_count);
310 }
311 EXPORT_SYMBOL(drop_nlink);
312 
313 /**
314  * clear_nlink - directly zero an inode's link count
315  * @inode: inode
316  *
317  * This is a low-level filesystem helper to replace any
318  * direct filesystem manipulation of i_nlink.  See
319  * drop_nlink() for why we care about i_nlink hitting zero.
320  */
clear_nlink(struct inode * inode)321 void clear_nlink(struct inode *inode)
322 {
323 	if (inode->i_nlink) {
324 		inode->__i_nlink = 0;
325 		atomic_long_inc(&inode->i_sb->s_remove_count);
326 	}
327 }
328 EXPORT_SYMBOL(clear_nlink);
329 
330 /**
331  * set_nlink - directly set an inode's link count
332  * @inode: inode
333  * @nlink: new nlink (should be non-zero)
334  *
335  * This is a low-level filesystem helper to replace any
336  * direct filesystem manipulation of i_nlink.
337  */
set_nlink(struct inode * inode,unsigned int nlink)338 void set_nlink(struct inode *inode, unsigned int nlink)
339 {
340 	if (!nlink) {
341 		clear_nlink(inode);
342 	} else {
343 		/* Yes, some filesystems do change nlink from zero to one */
344 		if (inode->i_nlink == 0)
345 			atomic_long_dec(&inode->i_sb->s_remove_count);
346 
347 		inode->__i_nlink = nlink;
348 	}
349 }
350 EXPORT_SYMBOL(set_nlink);
351 
352 /**
353  * inc_nlink - directly increment an inode's link count
354  * @inode: inode
355  *
356  * This is a low-level filesystem helper to replace any
357  * direct filesystem manipulation of i_nlink.  Currently,
358  * it is only here for parity with dec_nlink().
359  */
inc_nlink(struct inode * inode)360 void inc_nlink(struct inode *inode)
361 {
362 	if (unlikely(inode->i_nlink == 0)) {
363 		WARN_ON(!(inode->i_state & I_LINKABLE));
364 		atomic_long_dec(&inode->i_sb->s_remove_count);
365 	}
366 
367 	inode->__i_nlink++;
368 }
369 EXPORT_SYMBOL(inc_nlink);
370 
__address_space_init_once(struct address_space * mapping)371 static void __address_space_init_once(struct address_space *mapping)
372 {
373 	xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT);
374 	init_rwsem(&mapping->i_mmap_rwsem);
375 	INIT_LIST_HEAD(&mapping->private_list);
376 	spin_lock_init(&mapping->private_lock);
377 	mapping->i_mmap = RB_ROOT_CACHED;
378 }
379 
address_space_init_once(struct address_space * mapping)380 void address_space_init_once(struct address_space *mapping)
381 {
382 	memset(mapping, 0, sizeof(*mapping));
383 	__address_space_init_once(mapping);
384 }
385 EXPORT_SYMBOL(address_space_init_once);
386 
387 /*
388  * These are initializations that only need to be done
389  * once, because the fields are idempotent across use
390  * of the inode, so let the slab aware of that.
391  */
inode_init_once(struct inode * inode)392 void inode_init_once(struct inode *inode)
393 {
394 	memset(inode, 0, sizeof(*inode));
395 	INIT_HLIST_NODE(&inode->i_hash);
396 	INIT_LIST_HEAD(&inode->i_devices);
397 	INIT_LIST_HEAD(&inode->i_io_list);
398 	INIT_LIST_HEAD(&inode->i_wb_list);
399 	INIT_LIST_HEAD(&inode->i_lru);
400 	__address_space_init_once(&inode->i_data);
401 	i_size_ordered_init(inode);
402 }
403 EXPORT_SYMBOL(inode_init_once);
404 
init_once(void * foo)405 static void init_once(void *foo)
406 {
407 	struct inode *inode = (struct inode *) foo;
408 
409 	inode_init_once(inode);
410 }
411 
412 /*
413  * inode->i_lock must be held
414  */
__iget(struct inode * inode)415 void __iget(struct inode *inode)
416 {
417 	atomic_inc(&inode->i_count);
418 }
419 
420 /*
421  * get additional reference to inode; caller must already hold one.
422  */
ihold(struct inode * inode)423 void ihold(struct inode *inode)
424 {
425 	WARN_ON(atomic_inc_return(&inode->i_count) < 2);
426 }
427 EXPORT_SYMBOL(ihold);
428 
inode_lru_list_add(struct inode * inode)429 static void inode_lru_list_add(struct inode *inode)
430 {
431 	if (list_lru_add(&inode->i_sb->s_inode_lru, &inode->i_lru))
432 		this_cpu_inc(nr_unused);
433 	else
434 		inode->i_state |= I_REFERENCED;
435 }
436 
437 /*
438  * Add inode to LRU if needed (inode is unused and clean).
439  *
440  * Needs inode->i_lock held.
441  */
inode_add_lru(struct inode * inode)442 void inode_add_lru(struct inode *inode)
443 {
444 	if (!(inode->i_state & (I_DIRTY_ALL | I_SYNC |
445 				I_FREEING | I_WILL_FREE)) &&
446 	    !atomic_read(&inode->i_count) && inode->i_sb->s_flags & SB_ACTIVE)
447 		inode_lru_list_add(inode);
448 }
449 
450 
inode_lru_list_del(struct inode * inode)451 static void inode_lru_list_del(struct inode *inode)
452 {
453 
454 	if (list_lru_del(&inode->i_sb->s_inode_lru, &inode->i_lru))
455 		this_cpu_dec(nr_unused);
456 }
457 
458 /**
459  * inode_sb_list_add - add inode to the superblock list of inodes
460  * @inode: inode to add
461  */
inode_sb_list_add(struct inode * inode)462 void inode_sb_list_add(struct inode *inode)
463 {
464 	spin_lock(&inode->i_sb->s_inode_list_lock);
465 	list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
466 	spin_unlock(&inode->i_sb->s_inode_list_lock);
467 }
468 EXPORT_SYMBOL_GPL(inode_sb_list_add);
469 
inode_sb_list_del(struct inode * inode)470 static inline void inode_sb_list_del(struct inode *inode)
471 {
472 	if (!list_empty(&inode->i_sb_list)) {
473 		spin_lock(&inode->i_sb->s_inode_list_lock);
474 		list_del_init(&inode->i_sb_list);
475 		spin_unlock(&inode->i_sb->s_inode_list_lock);
476 	}
477 }
478 
hash(struct super_block * sb,unsigned long hashval)479 static unsigned long hash(struct super_block *sb, unsigned long hashval)
480 {
481 	unsigned long tmp;
482 
483 	tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
484 			L1_CACHE_BYTES;
485 	tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
486 	return tmp & i_hash_mask;
487 }
488 
489 /**
490  *	__insert_inode_hash - hash an inode
491  *	@inode: unhashed inode
492  *	@hashval: unsigned long value used to locate this object in the
493  *		inode_hashtable.
494  *
495  *	Add an inode to the inode hash for this superblock.
496  */
__insert_inode_hash(struct inode * inode,unsigned long hashval)497 void __insert_inode_hash(struct inode *inode, unsigned long hashval)
498 {
499 	struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
500 
501 	spin_lock(&inode_hash_lock);
502 	spin_lock(&inode->i_lock);
503 	hlist_add_head_rcu(&inode->i_hash, b);
504 	spin_unlock(&inode->i_lock);
505 	spin_unlock(&inode_hash_lock);
506 }
507 EXPORT_SYMBOL(__insert_inode_hash);
508 
509 /**
510  *	__remove_inode_hash - remove an inode from the hash
511  *	@inode: inode to unhash
512  *
513  *	Remove an inode from the superblock.
514  */
__remove_inode_hash(struct inode * inode)515 void __remove_inode_hash(struct inode *inode)
516 {
517 	spin_lock(&inode_hash_lock);
518 	spin_lock(&inode->i_lock);
519 	hlist_del_init_rcu(&inode->i_hash);
520 	spin_unlock(&inode->i_lock);
521 	spin_unlock(&inode_hash_lock);
522 }
523 EXPORT_SYMBOL(__remove_inode_hash);
524 
clear_inode(struct inode * inode)525 void clear_inode(struct inode *inode)
526 {
527 	/*
528 	 * We have to cycle the i_pages lock here because reclaim can be in the
529 	 * process of removing the last page (in __delete_from_page_cache())
530 	 * and we must not free the mapping under it.
531 	 */
532 	xa_lock_irq(&inode->i_data.i_pages);
533 	BUG_ON(inode->i_data.nrpages);
534 	BUG_ON(inode->i_data.nrexceptional);
535 	xa_unlock_irq(&inode->i_data.i_pages);
536 	BUG_ON(!list_empty(&inode->i_data.private_list));
537 	BUG_ON(!(inode->i_state & I_FREEING));
538 	BUG_ON(inode->i_state & I_CLEAR);
539 	BUG_ON(!list_empty(&inode->i_wb_list));
540 	/* don't need i_lock here, no concurrent mods to i_state */
541 	inode->i_state = I_FREEING | I_CLEAR;
542 }
543 EXPORT_SYMBOL(clear_inode);
544 
545 /*
546  * Free the inode passed in, removing it from the lists it is still connected
547  * to. We remove any pages still attached to the inode and wait for any IO that
548  * is still in progress before finally destroying the inode.
549  *
550  * An inode must already be marked I_FREEING so that we avoid the inode being
551  * moved back onto lists if we race with other code that manipulates the lists
552  * (e.g. writeback_single_inode). The caller is responsible for setting this.
553  *
554  * An inode must already be removed from the LRU list before being evicted from
555  * the cache. This should occur atomically with setting the I_FREEING state
556  * flag, so no inodes here should ever be on the LRU when being evicted.
557  */
evict(struct inode * inode)558 static void evict(struct inode *inode)
559 {
560 	const struct super_operations *op = inode->i_sb->s_op;
561 
562 	BUG_ON(!(inode->i_state & I_FREEING));
563 	BUG_ON(!list_empty(&inode->i_lru));
564 
565 	if (!list_empty(&inode->i_io_list))
566 		inode_io_list_del(inode);
567 
568 	inode_sb_list_del(inode);
569 
570 	/*
571 	 * Wait for flusher thread to be done with the inode so that filesystem
572 	 * does not start destroying it while writeback is still running. Since
573 	 * the inode has I_FREEING set, flusher thread won't start new work on
574 	 * the inode.  We just have to wait for running writeback to finish.
575 	 */
576 	inode_wait_for_writeback(inode);
577 
578 	if (op->evict_inode) {
579 		op->evict_inode(inode);
580 	} else {
581 		truncate_inode_pages_final(&inode->i_data);
582 		clear_inode(inode);
583 	}
584 	if (S_ISBLK(inode->i_mode) && inode->i_bdev)
585 		bd_forget(inode);
586 	if (S_ISCHR(inode->i_mode) && inode->i_cdev)
587 		cd_forget(inode);
588 
589 	remove_inode_hash(inode);
590 
591 	spin_lock(&inode->i_lock);
592 	wake_up_bit(&inode->i_state, __I_NEW);
593 	BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
594 	spin_unlock(&inode->i_lock);
595 
596 	destroy_inode(inode);
597 }
598 
599 /*
600  * dispose_list - dispose of the contents of a local list
601  * @head: the head of the list to free
602  *
603  * Dispose-list gets a local list with local inodes in it, so it doesn't
604  * need to worry about list corruption and SMP locks.
605  */
dispose_list(struct list_head * head)606 static void dispose_list(struct list_head *head)
607 {
608 	while (!list_empty(head)) {
609 		struct inode *inode;
610 
611 		inode = list_first_entry(head, struct inode, i_lru);
612 		list_del_init(&inode->i_lru);
613 
614 		evict(inode);
615 		cond_resched();
616 	}
617 }
618 
619 /**
620  * evict_inodes	- evict all evictable inodes for a superblock
621  * @sb:		superblock to operate on
622  *
623  * Make sure that no inodes with zero refcount are retained.  This is
624  * called by superblock shutdown after having SB_ACTIVE flag removed,
625  * so any inode reaching zero refcount during or after that call will
626  * be immediately evicted.
627  */
evict_inodes(struct super_block * sb)628 void evict_inodes(struct super_block *sb)
629 {
630 	struct inode *inode, *next;
631 	LIST_HEAD(dispose);
632 
633 again:
634 	spin_lock(&sb->s_inode_list_lock);
635 	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
636 		if (atomic_read(&inode->i_count))
637 			continue;
638 
639 		spin_lock(&inode->i_lock);
640 		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
641 			spin_unlock(&inode->i_lock);
642 			continue;
643 		}
644 
645 		inode->i_state |= I_FREEING;
646 		inode_lru_list_del(inode);
647 		spin_unlock(&inode->i_lock);
648 		list_add(&inode->i_lru, &dispose);
649 
650 		/*
651 		 * We can have a ton of inodes to evict at unmount time given
652 		 * enough memory, check to see if we need to go to sleep for a
653 		 * bit so we don't livelock.
654 		 */
655 		if (need_resched()) {
656 			spin_unlock(&sb->s_inode_list_lock);
657 			cond_resched();
658 			dispose_list(&dispose);
659 			goto again;
660 		}
661 	}
662 	spin_unlock(&sb->s_inode_list_lock);
663 
664 	dispose_list(&dispose);
665 }
666 EXPORT_SYMBOL_GPL(evict_inodes);
667 
668 /**
669  * invalidate_inodes	- attempt to free all inodes on a superblock
670  * @sb:		superblock to operate on
671  * @kill_dirty: flag to guide handling of dirty inodes
672  *
673  * Attempts to free all inodes for a given superblock.  If there were any
674  * busy inodes return a non-zero value, else zero.
675  * If @kill_dirty is set, discard dirty inodes too, otherwise treat
676  * them as busy.
677  */
invalidate_inodes(struct super_block * sb,bool kill_dirty)678 int invalidate_inodes(struct super_block *sb, bool kill_dirty)
679 {
680 	int busy = 0;
681 	struct inode *inode, *next;
682 	LIST_HEAD(dispose);
683 
684 again:
685 	spin_lock(&sb->s_inode_list_lock);
686 	list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
687 		spin_lock(&inode->i_lock);
688 		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
689 			spin_unlock(&inode->i_lock);
690 			continue;
691 		}
692 		if (inode->i_state & I_DIRTY_ALL && !kill_dirty) {
693 			spin_unlock(&inode->i_lock);
694 			busy = 1;
695 			continue;
696 		}
697 		if (atomic_read(&inode->i_count)) {
698 			spin_unlock(&inode->i_lock);
699 			busy = 1;
700 			continue;
701 		}
702 
703 		inode->i_state |= I_FREEING;
704 		inode_lru_list_del(inode);
705 		spin_unlock(&inode->i_lock);
706 		list_add(&inode->i_lru, &dispose);
707 		if (need_resched()) {
708 			spin_unlock(&sb->s_inode_list_lock);
709 			cond_resched();
710 			dispose_list(&dispose);
711 			goto again;
712 		}
713 	}
714 	spin_unlock(&sb->s_inode_list_lock);
715 
716 	dispose_list(&dispose);
717 
718 	return busy;
719 }
720 
721 /*
722  * Isolate the inode from the LRU in preparation for freeing it.
723  *
724  * Any inodes which are pinned purely because of attached pagecache have their
725  * pagecache removed.  If the inode has metadata buffers attached to
726  * mapping->private_list then try to remove them.
727  *
728  * If the inode has the I_REFERENCED flag set, then it means that it has been
729  * used recently - the flag is set in iput_final(). When we encounter such an
730  * inode, clear the flag and move it to the back of the LRU so it gets another
731  * pass through the LRU before it gets reclaimed. This is necessary because of
732  * the fact we are doing lazy LRU updates to minimise lock contention so the
733  * LRU does not have strict ordering. Hence we don't want to reclaim inodes
734  * with this flag set because they are the inodes that are out of order.
735  */
inode_lru_isolate(struct list_head * item,struct list_lru_one * lru,spinlock_t * lru_lock,void * arg)736 static enum lru_status inode_lru_isolate(struct list_head *item,
737 		struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
738 {
739 	struct list_head *freeable = arg;
740 	struct inode	*inode = container_of(item, struct inode, i_lru);
741 
742 	/*
743 	 * we are inverting the lru lock/inode->i_lock here, so use a trylock.
744 	 * If we fail to get the lock, just skip it.
745 	 */
746 	if (!spin_trylock(&inode->i_lock))
747 		return LRU_SKIP;
748 
749 	/*
750 	 * Referenced or dirty inodes are still in use. Give them another pass
751 	 * through the LRU as we canot reclaim them now.
752 	 */
753 	if (atomic_read(&inode->i_count) ||
754 	    (inode->i_state & ~I_REFERENCED)) {
755 		list_lru_isolate(lru, &inode->i_lru);
756 		spin_unlock(&inode->i_lock);
757 		this_cpu_dec(nr_unused);
758 		return LRU_REMOVED;
759 	}
760 
761 	/* recently referenced inodes get one more pass */
762 	if (inode->i_state & I_REFERENCED) {
763 		inode->i_state &= ~I_REFERENCED;
764 		spin_unlock(&inode->i_lock);
765 		return LRU_ROTATE;
766 	}
767 
768 	if (inode_has_buffers(inode) || inode->i_data.nrpages) {
769 		__iget(inode);
770 		spin_unlock(&inode->i_lock);
771 		spin_unlock(lru_lock);
772 		if (remove_inode_buffers(inode)) {
773 			unsigned long reap;
774 			reap = invalidate_mapping_pages(&inode->i_data, 0, -1);
775 			if (current_is_kswapd())
776 				__count_vm_events(KSWAPD_INODESTEAL, reap);
777 			else
778 				__count_vm_events(PGINODESTEAL, reap);
779 			if (current->reclaim_state)
780 				current->reclaim_state->reclaimed_slab += reap;
781 		}
782 		iput(inode);
783 		spin_lock(lru_lock);
784 		return LRU_RETRY;
785 	}
786 
787 	WARN_ON(inode->i_state & I_NEW);
788 	inode->i_state |= I_FREEING;
789 	list_lru_isolate_move(lru, &inode->i_lru, freeable);
790 	spin_unlock(&inode->i_lock);
791 
792 	this_cpu_dec(nr_unused);
793 	return LRU_REMOVED;
794 }
795 
796 /*
797  * Walk the superblock inode LRU for freeable inodes and attempt to free them.
798  * This is called from the superblock shrinker function with a number of inodes
799  * to trim from the LRU. Inodes to be freed are moved to a temporary list and
800  * then are freed outside inode_lock by dispose_list().
801  */
prune_icache_sb(struct super_block * sb,struct shrink_control * sc)802 long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
803 {
804 	LIST_HEAD(freeable);
805 	long freed;
806 
807 	freed = list_lru_shrink_walk(&sb->s_inode_lru, sc,
808 				     inode_lru_isolate, &freeable);
809 	dispose_list(&freeable);
810 	return freed;
811 }
812 
813 static void __wait_on_freeing_inode(struct inode *inode);
814 /*
815  * Called with the inode lock held.
816  */
find_inode(struct super_block * sb,struct hlist_head * head,int (* test)(struct inode *,void *),void * data)817 static struct inode *find_inode(struct super_block *sb,
818 				struct hlist_head *head,
819 				int (*test)(struct inode *, void *),
820 				void *data)
821 {
822 	struct inode *inode = NULL;
823 
824 repeat:
825 	hlist_for_each_entry(inode, head, i_hash) {
826 		if (inode->i_sb != sb)
827 			continue;
828 		if (!test(inode, data))
829 			continue;
830 		spin_lock(&inode->i_lock);
831 		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
832 			__wait_on_freeing_inode(inode);
833 			goto repeat;
834 		}
835 		if (unlikely(inode->i_state & I_CREATING)) {
836 			spin_unlock(&inode->i_lock);
837 			return ERR_PTR(-ESTALE);
838 		}
839 		__iget(inode);
840 		spin_unlock(&inode->i_lock);
841 		return inode;
842 	}
843 	return NULL;
844 }
845 
846 /*
847  * find_inode_fast is the fast path version of find_inode, see the comment at
848  * iget_locked for details.
849  */
find_inode_fast(struct super_block * sb,struct hlist_head * head,unsigned long ino)850 static struct inode *find_inode_fast(struct super_block *sb,
851 				struct hlist_head *head, unsigned long ino)
852 {
853 	struct inode *inode = NULL;
854 
855 repeat:
856 	hlist_for_each_entry(inode, head, i_hash) {
857 		if (inode->i_ino != ino)
858 			continue;
859 		if (inode->i_sb != sb)
860 			continue;
861 		spin_lock(&inode->i_lock);
862 		if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
863 			__wait_on_freeing_inode(inode);
864 			goto repeat;
865 		}
866 		if (unlikely(inode->i_state & I_CREATING)) {
867 			spin_unlock(&inode->i_lock);
868 			return ERR_PTR(-ESTALE);
869 		}
870 		__iget(inode);
871 		spin_unlock(&inode->i_lock);
872 		return inode;
873 	}
874 	return NULL;
875 }
876 
877 /*
878  * Each cpu owns a range of LAST_INO_BATCH numbers.
879  * 'shared_last_ino' is dirtied only once out of LAST_INO_BATCH allocations,
880  * to renew the exhausted range.
881  *
882  * This does not significantly increase overflow rate because every CPU can
883  * consume at most LAST_INO_BATCH-1 unused inode numbers. So there is
884  * NR_CPUS*(LAST_INO_BATCH-1) wastage. At 4096 and 1024, this is ~0.1% of the
885  * 2^32 range, and is a worst-case. Even a 50% wastage would only increase
886  * overflow rate by 2x, which does not seem too significant.
887  *
888  * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
889  * error if st_ino won't fit in target struct field. Use 32bit counter
890  * here to attempt to avoid that.
891  */
892 #define LAST_INO_BATCH 1024
893 static DEFINE_PER_CPU(unsigned int, last_ino);
894 
get_next_ino(void)895 unsigned int get_next_ino(void)
896 {
897 	unsigned int *p = &get_cpu_var(last_ino);
898 	unsigned int res = *p;
899 
900 #ifdef CONFIG_SMP
901 	if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) {
902 		static atomic_t shared_last_ino;
903 		int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino);
904 
905 		res = next - LAST_INO_BATCH;
906 	}
907 #endif
908 
909 	res++;
910 	/* get_next_ino should not provide a 0 inode number */
911 	if (unlikely(!res))
912 		res++;
913 	*p = res;
914 	put_cpu_var(last_ino);
915 	return res;
916 }
917 EXPORT_SYMBOL(get_next_ino);
918 
919 /**
920  *	new_inode_pseudo 	- obtain an inode
921  *	@sb: superblock
922  *
923  *	Allocates a new inode for given superblock.
924  *	Inode wont be chained in superblock s_inodes list
925  *	This means :
926  *	- fs can't be unmount
927  *	- quotas, fsnotify, writeback can't work
928  */
new_inode_pseudo(struct super_block * sb)929 struct inode *new_inode_pseudo(struct super_block *sb)
930 {
931 	struct inode *inode = alloc_inode(sb);
932 
933 	if (inode) {
934 		spin_lock(&inode->i_lock);
935 		inode->i_state = 0;
936 		spin_unlock(&inode->i_lock);
937 		INIT_LIST_HEAD(&inode->i_sb_list);
938 	}
939 	return inode;
940 }
941 
942 /**
943  *	new_inode 	- obtain an inode
944  *	@sb: superblock
945  *
946  *	Allocates a new inode for given superblock. The default gfp_mask
947  *	for allocations related to inode->i_mapping is GFP_HIGHUSER_MOVABLE.
948  *	If HIGHMEM pages are unsuitable or it is known that pages allocated
949  *	for the page cache are not reclaimable or migratable,
950  *	mapping_set_gfp_mask() must be called with suitable flags on the
951  *	newly created inode's mapping
952  *
953  */
new_inode(struct super_block * sb)954 struct inode *new_inode(struct super_block *sb)
955 {
956 	struct inode *inode;
957 
958 	spin_lock_prefetch(&sb->s_inode_list_lock);
959 
960 	inode = new_inode_pseudo(sb);
961 	if (inode)
962 		inode_sb_list_add(inode);
963 	return inode;
964 }
965 EXPORT_SYMBOL(new_inode);
966 
967 #ifdef CONFIG_DEBUG_LOCK_ALLOC
lockdep_annotate_inode_mutex_key(struct inode * inode)968 void lockdep_annotate_inode_mutex_key(struct inode *inode)
969 {
970 	if (S_ISDIR(inode->i_mode)) {
971 		struct file_system_type *type = inode->i_sb->s_type;
972 
973 		/* Set new key only if filesystem hasn't already changed it */
974 		if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) {
975 			/*
976 			 * ensure nobody is actually holding i_mutex
977 			 */
978 			// mutex_destroy(&inode->i_mutex);
979 			init_rwsem(&inode->i_rwsem);
980 			lockdep_set_class(&inode->i_rwsem,
981 					  &type->i_mutex_dir_key);
982 		}
983 	}
984 }
985 EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key);
986 #endif
987 
988 /**
989  * unlock_new_inode - clear the I_NEW state and wake up any waiters
990  * @inode:	new inode to unlock
991  *
992  * Called when the inode is fully initialised to clear the new state of the
993  * inode and wake up anyone waiting for the inode to finish initialisation.
994  */
unlock_new_inode(struct inode * inode)995 void unlock_new_inode(struct inode *inode)
996 {
997 	lockdep_annotate_inode_mutex_key(inode);
998 	spin_lock(&inode->i_lock);
999 	WARN_ON(!(inode->i_state & I_NEW));
1000 	inode->i_state &= ~I_NEW & ~I_CREATING;
1001 	smp_mb();
1002 	wake_up_bit(&inode->i_state, __I_NEW);
1003 	spin_unlock(&inode->i_lock);
1004 }
1005 EXPORT_SYMBOL(unlock_new_inode);
1006 
discard_new_inode(struct inode * inode)1007 void discard_new_inode(struct inode *inode)
1008 {
1009 	lockdep_annotate_inode_mutex_key(inode);
1010 	spin_lock(&inode->i_lock);
1011 	WARN_ON(!(inode->i_state & I_NEW));
1012 	inode->i_state &= ~I_NEW;
1013 	smp_mb();
1014 	wake_up_bit(&inode->i_state, __I_NEW);
1015 	spin_unlock(&inode->i_lock);
1016 	iput(inode);
1017 }
1018 EXPORT_SYMBOL(discard_new_inode);
1019 
1020 /**
1021  * lock_two_nondirectories - take two i_mutexes on non-directory objects
1022  *
1023  * Lock any non-NULL argument that is not a directory.
1024  * Zero, one or two objects may be locked by this function.
1025  *
1026  * @inode1: first inode to lock
1027  * @inode2: second inode to lock
1028  */
lock_two_nondirectories(struct inode * inode1,struct inode * inode2)1029 void lock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1030 {
1031 	if (inode1 > inode2)
1032 		swap(inode1, inode2);
1033 
1034 	if (inode1 && !S_ISDIR(inode1->i_mode))
1035 		inode_lock(inode1);
1036 	if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1037 		inode_lock_nested(inode2, I_MUTEX_NONDIR2);
1038 }
1039 EXPORT_SYMBOL(lock_two_nondirectories);
1040 
1041 /**
1042  * unlock_two_nondirectories - release locks from lock_two_nondirectories()
1043  * @inode1: first inode to unlock
1044  * @inode2: second inode to unlock
1045  */
unlock_two_nondirectories(struct inode * inode1,struct inode * inode2)1046 void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2)
1047 {
1048 	if (inode1 && !S_ISDIR(inode1->i_mode))
1049 		inode_unlock(inode1);
1050 	if (inode2 && !S_ISDIR(inode2->i_mode) && inode2 != inode1)
1051 		inode_unlock(inode2);
1052 }
1053 EXPORT_SYMBOL(unlock_two_nondirectories);
1054 
1055 /**
1056  * inode_insert5 - obtain an inode from a mounted file system
1057  * @inode:	pre-allocated inode to use for insert to cache
1058  * @hashval:	hash value (usually inode number) to get
1059  * @test:	callback used for comparisons between inodes
1060  * @set:	callback used to initialize a new struct inode
1061  * @data:	opaque data pointer to pass to @test and @set
1062  *
1063  * Search for the inode specified by @hashval and @data in the inode cache,
1064  * and if present it is return it with an increased reference count. This is
1065  * a variant of iget5_locked() for callers that don't want to fail on memory
1066  * allocation of inode.
1067  *
1068  * If the inode is not in cache, insert the pre-allocated inode to cache and
1069  * return it locked, hashed, and with the I_NEW flag set. The file system gets
1070  * to fill it in before unlocking it via unlock_new_inode().
1071  *
1072  * Note both @test and @set are called with the inode_hash_lock held, so can't
1073  * sleep.
1074  */
inode_insert5(struct inode * inode,unsigned long hashval,int (* test)(struct inode *,void *),int (* set)(struct inode *,void *),void * data)1075 struct inode *inode_insert5(struct inode *inode, unsigned long hashval,
1076 			    int (*test)(struct inode *, void *),
1077 			    int (*set)(struct inode *, void *), void *data)
1078 {
1079 	struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval);
1080 	struct inode *old;
1081 	bool creating = inode->i_state & I_CREATING;
1082 
1083 again:
1084 	spin_lock(&inode_hash_lock);
1085 	old = find_inode(inode->i_sb, head, test, data);
1086 	if (unlikely(old)) {
1087 		/*
1088 		 * Uhhuh, somebody else created the same inode under us.
1089 		 * Use the old inode instead of the preallocated one.
1090 		 */
1091 		spin_unlock(&inode_hash_lock);
1092 		if (IS_ERR(old))
1093 			return NULL;
1094 		wait_on_inode(old);
1095 		if (unlikely(inode_unhashed(old))) {
1096 			iput(old);
1097 			goto again;
1098 		}
1099 		return old;
1100 	}
1101 
1102 	if (set && unlikely(set(inode, data))) {
1103 		inode = NULL;
1104 		goto unlock;
1105 	}
1106 
1107 	/*
1108 	 * Return the locked inode with I_NEW set, the
1109 	 * caller is responsible for filling in the contents
1110 	 */
1111 	spin_lock(&inode->i_lock);
1112 	inode->i_state |= I_NEW;
1113 	hlist_add_head_rcu(&inode->i_hash, head);
1114 	spin_unlock(&inode->i_lock);
1115 	if (!creating)
1116 		inode_sb_list_add(inode);
1117 unlock:
1118 	spin_unlock(&inode_hash_lock);
1119 
1120 	return inode;
1121 }
1122 EXPORT_SYMBOL(inode_insert5);
1123 
1124 /**
1125  * iget5_locked - obtain an inode from a mounted file system
1126  * @sb:		super block of file system
1127  * @hashval:	hash value (usually inode number) to get
1128  * @test:	callback used for comparisons between inodes
1129  * @set:	callback used to initialize a new struct inode
1130  * @data:	opaque data pointer to pass to @test and @set
1131  *
1132  * Search for the inode specified by @hashval and @data in the inode cache,
1133  * and if present it is return it with an increased reference count. This is
1134  * a generalized version of iget_locked() for file systems where the inode
1135  * number is not sufficient for unique identification of an inode.
1136  *
1137  * If the inode is not in cache, allocate a new inode and return it locked,
1138  * hashed, and with the I_NEW flag set. The file system gets to fill it in
1139  * before unlocking it via unlock_new_inode().
1140  *
1141  * Note both @test and @set are called with the inode_hash_lock held, so can't
1142  * sleep.
1143  */
iget5_locked(struct super_block * sb,unsigned long hashval,int (* test)(struct inode *,void *),int (* set)(struct inode *,void *),void * data)1144 struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
1145 		int (*test)(struct inode *, void *),
1146 		int (*set)(struct inode *, void *), void *data)
1147 {
1148 	struct inode *inode = ilookup5(sb, hashval, test, data);
1149 
1150 	if (!inode) {
1151 		struct inode *new = alloc_inode(sb);
1152 
1153 		if (new) {
1154 			new->i_state = 0;
1155 			inode = inode_insert5(new, hashval, test, set, data);
1156 			if (unlikely(inode != new))
1157 				destroy_inode(new);
1158 		}
1159 	}
1160 	return inode;
1161 }
1162 EXPORT_SYMBOL(iget5_locked);
1163 
1164 /**
1165  * iget_locked - obtain an inode from a mounted file system
1166  * @sb:		super block of file system
1167  * @ino:	inode number to get
1168  *
1169  * Search for the inode specified by @ino in the inode cache and if present
1170  * return it with an increased reference count. This is for file systems
1171  * where the inode number is sufficient for unique identification of an inode.
1172  *
1173  * If the inode is not in cache, allocate a new inode and return it locked,
1174  * hashed, and with the I_NEW flag set.  The file system gets to fill it in
1175  * before unlocking it via unlock_new_inode().
1176  */
iget_locked(struct super_block * sb,unsigned long ino)1177 struct inode *iget_locked(struct super_block *sb, unsigned long ino)
1178 {
1179 	struct hlist_head *head = inode_hashtable + hash(sb, ino);
1180 	struct inode *inode;
1181 again:
1182 	spin_lock(&inode_hash_lock);
1183 	inode = find_inode_fast(sb, head, ino);
1184 	spin_unlock(&inode_hash_lock);
1185 	if (inode) {
1186 		if (IS_ERR(inode))
1187 			return NULL;
1188 		wait_on_inode(inode);
1189 		if (unlikely(inode_unhashed(inode))) {
1190 			iput(inode);
1191 			goto again;
1192 		}
1193 		return inode;
1194 	}
1195 
1196 	inode = alloc_inode(sb);
1197 	if (inode) {
1198 		struct inode *old;
1199 
1200 		spin_lock(&inode_hash_lock);
1201 		/* We released the lock, so.. */
1202 		old = find_inode_fast(sb, head, ino);
1203 		if (!old) {
1204 			inode->i_ino = ino;
1205 			spin_lock(&inode->i_lock);
1206 			inode->i_state = I_NEW;
1207 			hlist_add_head_rcu(&inode->i_hash, head);
1208 			spin_unlock(&inode->i_lock);
1209 			inode_sb_list_add(inode);
1210 			spin_unlock(&inode_hash_lock);
1211 
1212 			/* Return the locked inode with I_NEW set, the
1213 			 * caller is responsible for filling in the contents
1214 			 */
1215 			return inode;
1216 		}
1217 
1218 		/*
1219 		 * Uhhuh, somebody else created the same inode under
1220 		 * us. Use the old inode instead of the one we just
1221 		 * allocated.
1222 		 */
1223 		spin_unlock(&inode_hash_lock);
1224 		destroy_inode(inode);
1225 		if (IS_ERR(old))
1226 			return NULL;
1227 		inode = old;
1228 		wait_on_inode(inode);
1229 		if (unlikely(inode_unhashed(inode))) {
1230 			iput(inode);
1231 			goto again;
1232 		}
1233 	}
1234 	return inode;
1235 }
1236 EXPORT_SYMBOL(iget_locked);
1237 
1238 /*
1239  * search the inode cache for a matching inode number.
1240  * If we find one, then the inode number we are trying to
1241  * allocate is not unique and so we should not use it.
1242  *
1243  * Returns 1 if the inode number is unique, 0 if it is not.
1244  */
test_inode_iunique(struct super_block * sb,unsigned long ino)1245 static int test_inode_iunique(struct super_block *sb, unsigned long ino)
1246 {
1247 	struct hlist_head *b = inode_hashtable + hash(sb, ino);
1248 	struct inode *inode;
1249 
1250 	hlist_for_each_entry_rcu(inode, b, i_hash) {
1251 		if (inode->i_ino == ino && inode->i_sb == sb)
1252 			return 0;
1253 	}
1254 	return 1;
1255 }
1256 
1257 /**
1258  *	iunique - get a unique inode number
1259  *	@sb: superblock
1260  *	@max_reserved: highest reserved inode number
1261  *
1262  *	Obtain an inode number that is unique on the system for a given
1263  *	superblock. This is used by file systems that have no natural
1264  *	permanent inode numbering system. An inode number is returned that
1265  *	is higher than the reserved limit but unique.
1266  *
1267  *	BUGS:
1268  *	With a large number of inodes live on the file system this function
1269  *	currently becomes quite slow.
1270  */
iunique(struct super_block * sb,ino_t max_reserved)1271 ino_t iunique(struct super_block *sb, ino_t max_reserved)
1272 {
1273 	/*
1274 	 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW
1275 	 * error if st_ino won't fit in target struct field. Use 32bit counter
1276 	 * here to attempt to avoid that.
1277 	 */
1278 	static DEFINE_SPINLOCK(iunique_lock);
1279 	static unsigned int counter;
1280 	ino_t res;
1281 
1282 	rcu_read_lock();
1283 	spin_lock(&iunique_lock);
1284 	do {
1285 		if (counter <= max_reserved)
1286 			counter = max_reserved + 1;
1287 		res = counter++;
1288 	} while (!test_inode_iunique(sb, res));
1289 	spin_unlock(&iunique_lock);
1290 	rcu_read_unlock();
1291 
1292 	return res;
1293 }
1294 EXPORT_SYMBOL(iunique);
1295 
igrab(struct inode * inode)1296 struct inode *igrab(struct inode *inode)
1297 {
1298 	spin_lock(&inode->i_lock);
1299 	if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
1300 		__iget(inode);
1301 		spin_unlock(&inode->i_lock);
1302 	} else {
1303 		spin_unlock(&inode->i_lock);
1304 		/*
1305 		 * Handle the case where s_op->clear_inode is not been
1306 		 * called yet, and somebody is calling igrab
1307 		 * while the inode is getting freed.
1308 		 */
1309 		inode = NULL;
1310 	}
1311 	return inode;
1312 }
1313 EXPORT_SYMBOL(igrab);
1314 
1315 /**
1316  * ilookup5_nowait - search for an inode in the inode cache
1317  * @sb:		super block of file system to search
1318  * @hashval:	hash value (usually inode number) to search for
1319  * @test:	callback used for comparisons between inodes
1320  * @data:	opaque data pointer to pass to @test
1321  *
1322  * Search for the inode specified by @hashval and @data in the inode cache.
1323  * If the inode is in the cache, the inode is returned with an incremented
1324  * reference count.
1325  *
1326  * Note: I_NEW is not waited upon so you have to be very careful what you do
1327  * with the returned inode.  You probably should be using ilookup5() instead.
1328  *
1329  * Note2: @test is called with the inode_hash_lock held, so can't sleep.
1330  */
ilookup5_nowait(struct super_block * sb,unsigned long hashval,int (* test)(struct inode *,void *),void * data)1331 struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
1332 		int (*test)(struct inode *, void *), void *data)
1333 {
1334 	struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1335 	struct inode *inode;
1336 
1337 	spin_lock(&inode_hash_lock);
1338 	inode = find_inode(sb, head, test, data);
1339 	spin_unlock(&inode_hash_lock);
1340 
1341 	return IS_ERR(inode) ? NULL : inode;
1342 }
1343 EXPORT_SYMBOL(ilookup5_nowait);
1344 
1345 /**
1346  * ilookup5 - search for an inode in the inode cache
1347  * @sb:		super block of file system to search
1348  * @hashval:	hash value (usually inode number) to search for
1349  * @test:	callback used for comparisons between inodes
1350  * @data:	opaque data pointer to pass to @test
1351  *
1352  * Search for the inode specified by @hashval and @data in the inode cache,
1353  * and if the inode is in the cache, return the inode with an incremented
1354  * reference count.  Waits on I_NEW before returning the inode.
1355  * returned with an incremented reference count.
1356  *
1357  * This is a generalized version of ilookup() for file systems where the
1358  * inode number is not sufficient for unique identification of an inode.
1359  *
1360  * Note: @test is called with the inode_hash_lock held, so can't sleep.
1361  */
ilookup5(struct super_block * sb,unsigned long hashval,int (* test)(struct inode *,void *),void * data)1362 struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
1363 		int (*test)(struct inode *, void *), void *data)
1364 {
1365 	struct inode *inode;
1366 again:
1367 	inode = ilookup5_nowait(sb, hashval, test, data);
1368 	if (inode) {
1369 		wait_on_inode(inode);
1370 		if (unlikely(inode_unhashed(inode))) {
1371 			iput(inode);
1372 			goto again;
1373 		}
1374 	}
1375 	return inode;
1376 }
1377 EXPORT_SYMBOL(ilookup5);
1378 
1379 /**
1380  * ilookup - search for an inode in the inode cache
1381  * @sb:		super block of file system to search
1382  * @ino:	inode number to search for
1383  *
1384  * Search for the inode @ino in the inode cache, and if the inode is in the
1385  * cache, the inode is returned with an incremented reference count.
1386  */
ilookup(struct super_block * sb,unsigned long ino)1387 struct inode *ilookup(struct super_block *sb, unsigned long ino)
1388 {
1389 	struct hlist_head *head = inode_hashtable + hash(sb, ino);
1390 	struct inode *inode;
1391 again:
1392 	spin_lock(&inode_hash_lock);
1393 	inode = find_inode_fast(sb, head, ino);
1394 	spin_unlock(&inode_hash_lock);
1395 
1396 	if (inode) {
1397 		if (IS_ERR(inode))
1398 			return NULL;
1399 		wait_on_inode(inode);
1400 		if (unlikely(inode_unhashed(inode))) {
1401 			iput(inode);
1402 			goto again;
1403 		}
1404 	}
1405 	return inode;
1406 }
1407 EXPORT_SYMBOL(ilookup);
1408 
1409 /**
1410  * find_inode_nowait - find an inode in the inode cache
1411  * @sb:		super block of file system to search
1412  * @hashval:	hash value (usually inode number) to search for
1413  * @match:	callback used for comparisons between inodes
1414  * @data:	opaque data pointer to pass to @match
1415  *
1416  * Search for the inode specified by @hashval and @data in the inode
1417  * cache, where the helper function @match will return 0 if the inode
1418  * does not match, 1 if the inode does match, and -1 if the search
1419  * should be stopped.  The @match function must be responsible for
1420  * taking the i_lock spin_lock and checking i_state for an inode being
1421  * freed or being initialized, and incrementing the reference count
1422  * before returning 1.  It also must not sleep, since it is called with
1423  * the inode_hash_lock spinlock held.
1424  *
1425  * This is a even more generalized version of ilookup5() when the
1426  * function must never block --- find_inode() can block in
1427  * __wait_on_freeing_inode() --- or when the caller can not increment
1428  * the reference count because the resulting iput() might cause an
1429  * inode eviction.  The tradeoff is that the @match funtion must be
1430  * very carefully implemented.
1431  */
find_inode_nowait(struct super_block * sb,unsigned long hashval,int (* match)(struct inode *,unsigned long,void *),void * data)1432 struct inode *find_inode_nowait(struct super_block *sb,
1433 				unsigned long hashval,
1434 				int (*match)(struct inode *, unsigned long,
1435 					     void *),
1436 				void *data)
1437 {
1438 	struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1439 	struct inode *inode, *ret_inode = NULL;
1440 	int mval;
1441 
1442 	spin_lock(&inode_hash_lock);
1443 	hlist_for_each_entry(inode, head, i_hash) {
1444 		if (inode->i_sb != sb)
1445 			continue;
1446 		mval = match(inode, hashval, data);
1447 		if (mval == 0)
1448 			continue;
1449 		if (mval == 1)
1450 			ret_inode = inode;
1451 		goto out;
1452 	}
1453 out:
1454 	spin_unlock(&inode_hash_lock);
1455 	return ret_inode;
1456 }
1457 EXPORT_SYMBOL(find_inode_nowait);
1458 
1459 /**
1460  * find_inode_rcu - find an inode in the inode cache
1461  * @sb:		Super block of file system to search
1462  * @hashval:	Key to hash
1463  * @test:	Function to test match on an inode
1464  * @data:	Data for test function
1465  *
1466  * Search for the inode specified by @hashval and @data in the inode cache,
1467  * where the helper function @test will return 0 if the inode does not match
1468  * and 1 if it does.  The @test function must be responsible for taking the
1469  * i_lock spin_lock and checking i_state for an inode being freed or being
1470  * initialized.
1471  *
1472  * If successful, this will return the inode for which the @test function
1473  * returned 1 and NULL otherwise.
1474  *
1475  * The @test function is not permitted to take a ref on any inode presented.
1476  * It is also not permitted to sleep.
1477  *
1478  * The caller must hold the RCU read lock.
1479  */
find_inode_rcu(struct super_block * sb,unsigned long hashval,int (* test)(struct inode *,void *),void * data)1480 struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval,
1481 			     int (*test)(struct inode *, void *), void *data)
1482 {
1483 	struct hlist_head *head = inode_hashtable + hash(sb, hashval);
1484 	struct inode *inode;
1485 
1486 	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
1487 			 "suspicious find_inode_rcu() usage");
1488 
1489 	hlist_for_each_entry_rcu(inode, head, i_hash) {
1490 		if (inode->i_sb == sb &&
1491 		    !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)) &&
1492 		    test(inode, data))
1493 			return inode;
1494 	}
1495 	return NULL;
1496 }
1497 EXPORT_SYMBOL(find_inode_rcu);
1498 
1499 /**
1500  * find_inode_by_rcu - Find an inode in the inode cache
1501  * @sb:		Super block of file system to search
1502  * @ino:	The inode number to match
1503  *
1504  * Search for the inode specified by @hashval and @data in the inode cache,
1505  * where the helper function @test will return 0 if the inode does not match
1506  * and 1 if it does.  The @test function must be responsible for taking the
1507  * i_lock spin_lock and checking i_state for an inode being freed or being
1508  * initialized.
1509  *
1510  * If successful, this will return the inode for which the @test function
1511  * returned 1 and NULL otherwise.
1512  *
1513  * The @test function is not permitted to take a ref on any inode presented.
1514  * It is also not permitted to sleep.
1515  *
1516  * The caller must hold the RCU read lock.
1517  */
find_inode_by_ino_rcu(struct super_block * sb,unsigned long ino)1518 struct inode *find_inode_by_ino_rcu(struct super_block *sb,
1519 				    unsigned long ino)
1520 {
1521 	struct hlist_head *head = inode_hashtable + hash(sb, ino);
1522 	struct inode *inode;
1523 
1524 	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
1525 			 "suspicious find_inode_by_ino_rcu() usage");
1526 
1527 	hlist_for_each_entry_rcu(inode, head, i_hash) {
1528 		if (inode->i_ino == ino &&
1529 		    inode->i_sb == sb &&
1530 		    !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)))
1531 		    return inode;
1532 	}
1533 	return NULL;
1534 }
1535 EXPORT_SYMBOL(find_inode_by_ino_rcu);
1536 
insert_inode_locked(struct inode * inode)1537 int insert_inode_locked(struct inode *inode)
1538 {
1539 	struct super_block *sb = inode->i_sb;
1540 	ino_t ino = inode->i_ino;
1541 	struct hlist_head *head = inode_hashtable + hash(sb, ino);
1542 
1543 	while (1) {
1544 		struct inode *old = NULL;
1545 		spin_lock(&inode_hash_lock);
1546 		hlist_for_each_entry(old, head, i_hash) {
1547 			if (old->i_ino != ino)
1548 				continue;
1549 			if (old->i_sb != sb)
1550 				continue;
1551 			spin_lock(&old->i_lock);
1552 			if (old->i_state & (I_FREEING|I_WILL_FREE)) {
1553 				spin_unlock(&old->i_lock);
1554 				continue;
1555 			}
1556 			break;
1557 		}
1558 		if (likely(!old)) {
1559 			spin_lock(&inode->i_lock);
1560 			inode->i_state |= I_NEW | I_CREATING;
1561 			hlist_add_head_rcu(&inode->i_hash, head);
1562 			spin_unlock(&inode->i_lock);
1563 			spin_unlock(&inode_hash_lock);
1564 			return 0;
1565 		}
1566 		if (unlikely(old->i_state & I_CREATING)) {
1567 			spin_unlock(&old->i_lock);
1568 			spin_unlock(&inode_hash_lock);
1569 			return -EBUSY;
1570 		}
1571 		__iget(old);
1572 		spin_unlock(&old->i_lock);
1573 		spin_unlock(&inode_hash_lock);
1574 		wait_on_inode(old);
1575 		if (unlikely(!inode_unhashed(old))) {
1576 			iput(old);
1577 			return -EBUSY;
1578 		}
1579 		iput(old);
1580 	}
1581 }
1582 EXPORT_SYMBOL(insert_inode_locked);
1583 
insert_inode_locked4(struct inode * inode,unsigned long hashval,int (* test)(struct inode *,void *),void * data)1584 int insert_inode_locked4(struct inode *inode, unsigned long hashval,
1585 		int (*test)(struct inode *, void *), void *data)
1586 {
1587 	struct inode *old;
1588 
1589 	inode->i_state |= I_CREATING;
1590 	old = inode_insert5(inode, hashval, test, NULL, data);
1591 
1592 	if (old != inode) {
1593 		iput(old);
1594 		return -EBUSY;
1595 	}
1596 	return 0;
1597 }
1598 EXPORT_SYMBOL(insert_inode_locked4);
1599 
1600 
generic_delete_inode(struct inode * inode)1601 int generic_delete_inode(struct inode *inode)
1602 {
1603 	return 1;
1604 }
1605 EXPORT_SYMBOL(generic_delete_inode);
1606 
1607 /*
1608  * Called when we're dropping the last reference
1609  * to an inode.
1610  *
1611  * Call the FS "drop_inode()" function, defaulting to
1612  * the legacy UNIX filesystem behaviour.  If it tells
1613  * us to evict inode, do so.  Otherwise, retain inode
1614  * in cache if fs is alive, sync and evict if fs is
1615  * shutting down.
1616  */
iput_final(struct inode * inode)1617 static void iput_final(struct inode *inode)
1618 {
1619 	struct super_block *sb = inode->i_sb;
1620 	const struct super_operations *op = inode->i_sb->s_op;
1621 	unsigned long state;
1622 	int drop;
1623 
1624 	WARN_ON(inode->i_state & I_NEW);
1625 
1626 	if (op->drop_inode)
1627 		drop = op->drop_inode(inode);
1628 	else
1629 		drop = generic_drop_inode(inode);
1630 
1631 	if (!drop &&
1632 	    !(inode->i_state & I_DONTCACHE) &&
1633 	    (sb->s_flags & SB_ACTIVE)) {
1634 		inode_add_lru(inode);
1635 		spin_unlock(&inode->i_lock);
1636 		return;
1637 	}
1638 
1639 	state = inode->i_state;
1640 	if (!drop) {
1641 		WRITE_ONCE(inode->i_state, state | I_WILL_FREE);
1642 		spin_unlock(&inode->i_lock);
1643 
1644 		write_inode_now(inode, 1);
1645 
1646 		spin_lock(&inode->i_lock);
1647 		state = inode->i_state;
1648 		WARN_ON(state & I_NEW);
1649 		state &= ~I_WILL_FREE;
1650 	}
1651 
1652 	WRITE_ONCE(inode->i_state, state | I_FREEING);
1653 	if (!list_empty(&inode->i_lru))
1654 		inode_lru_list_del(inode);
1655 	spin_unlock(&inode->i_lock);
1656 
1657 	evict(inode);
1658 }
1659 
1660 /**
1661  *	iput	- put an inode
1662  *	@inode: inode to put
1663  *
1664  *	Puts an inode, dropping its usage count. If the inode use count hits
1665  *	zero, the inode is then freed and may also be destroyed.
1666  *
1667  *	Consequently, iput() can sleep.
1668  */
iput(struct inode * inode)1669 void iput(struct inode *inode)
1670 {
1671 	if (!inode)
1672 		return;
1673 	BUG_ON(inode->i_state & I_CLEAR);
1674 retry:
1675 	if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) {
1676 		if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) {
1677 			atomic_inc(&inode->i_count);
1678 			spin_unlock(&inode->i_lock);
1679 			trace_writeback_lazytime_iput(inode);
1680 			mark_inode_dirty_sync(inode);
1681 			goto retry;
1682 		}
1683 		iput_final(inode);
1684 	}
1685 }
1686 EXPORT_SYMBOL(iput);
1687 
1688 #ifdef CONFIG_BLOCK
1689 /**
1690  *	bmap	- find a block number in a file
1691  *	@inode:  inode owning the block number being requested
1692  *	@block: pointer containing the block to find
1693  *
1694  *	Replaces the value in ``*block`` with the block number on the device holding
1695  *	corresponding to the requested block number in the file.
1696  *	That is, asked for block 4 of inode 1 the function will replace the
1697  *	4 in ``*block``, with disk block relative to the disk start that holds that
1698  *	block of the file.
1699  *
1700  *	Returns -EINVAL in case of error, 0 otherwise. If mapping falls into a
1701  *	hole, returns 0 and ``*block`` is also set to 0.
1702  */
bmap(struct inode * inode,sector_t * block)1703 int bmap(struct inode *inode, sector_t *block)
1704 {
1705 	if (!inode->i_mapping->a_ops->bmap)
1706 		return -EINVAL;
1707 
1708 	*block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block);
1709 	return 0;
1710 }
1711 EXPORT_SYMBOL(bmap);
1712 #endif
1713 
1714 /*
1715  * With relative atime, only update atime if the previous atime is
1716  * earlier than either the ctime or mtime or if at least a day has
1717  * passed since the last atime update.
1718  */
relatime_need_update(struct vfsmount * mnt,struct inode * inode,struct timespec64 now)1719 static int relatime_need_update(struct vfsmount *mnt, struct inode *inode,
1720 			     struct timespec64 now)
1721 {
1722 
1723 	if (!(mnt->mnt_flags & MNT_RELATIME))
1724 		return 1;
1725 	/*
1726 	 * Is mtime younger than atime? If yes, update atime:
1727 	 */
1728 	if (timespec64_compare(&inode->i_mtime, &inode->i_atime) >= 0)
1729 		return 1;
1730 	/*
1731 	 * Is ctime younger than atime? If yes, update atime:
1732 	 */
1733 	if (timespec64_compare(&inode->i_ctime, &inode->i_atime) >= 0)
1734 		return 1;
1735 
1736 	/*
1737 	 * Is the previous atime value older than a day? If yes,
1738 	 * update atime:
1739 	 */
1740 	if ((long)(now.tv_sec - inode->i_atime.tv_sec) >= 24*60*60)
1741 		return 1;
1742 	/*
1743 	 * Good, we can skip the atime update:
1744 	 */
1745 	return 0;
1746 }
1747 
generic_update_time(struct inode * inode,struct timespec64 * time,int flags)1748 int generic_update_time(struct inode *inode, struct timespec64 *time, int flags)
1749 {
1750 	int iflags = I_DIRTY_TIME;
1751 	bool dirty = false;
1752 
1753 	if (flags & S_ATIME)
1754 		inode->i_atime = *time;
1755 	if (flags & S_VERSION)
1756 		dirty = inode_maybe_inc_iversion(inode, false);
1757 	if (flags & S_CTIME)
1758 		inode->i_ctime = *time;
1759 	if (flags & S_MTIME)
1760 		inode->i_mtime = *time;
1761 	if ((flags & (S_ATIME | S_CTIME | S_MTIME)) &&
1762 	    !(inode->i_sb->s_flags & SB_LAZYTIME))
1763 		dirty = true;
1764 
1765 	if (dirty)
1766 		iflags |= I_DIRTY_SYNC;
1767 	__mark_inode_dirty(inode, iflags);
1768 	return 0;
1769 }
1770 EXPORT_SYMBOL(generic_update_time);
1771 
1772 /*
1773  * This does the actual work of updating an inodes time or version.  Must have
1774  * had called mnt_want_write() before calling this.
1775  */
inode_update_time(struct inode * inode,struct timespec64 * time,int flags)1776 int inode_update_time(struct inode *inode, struct timespec64 *time, int flags)
1777 {
1778 	if (inode->i_op->update_time)
1779 		return inode->i_op->update_time(inode, time, flags);
1780 	return generic_update_time(inode, time, flags);
1781 }
1782 EXPORT_SYMBOL(inode_update_time);
1783 
1784 /**
1785  *	touch_atime	-	update the access time
1786  *	@path: the &struct path to update
1787  *	@inode: inode to update
1788  *
1789  *	Update the accessed time on an inode and mark it for writeback.
1790  *	This function automatically handles read only file systems and media,
1791  *	as well as the "noatime" flag and inode specific "noatime" markers.
1792  */
atime_needs_update(const struct path * path,struct inode * inode)1793 bool atime_needs_update(const struct path *path, struct inode *inode)
1794 {
1795 	struct vfsmount *mnt = path->mnt;
1796 	struct timespec64 now;
1797 
1798 	if (inode->i_flags & S_NOATIME)
1799 		return false;
1800 
1801 	/* Atime updates will likely cause i_uid and i_gid to be written
1802 	 * back improprely if their true value is unknown to the vfs.
1803 	 */
1804 	if (HAS_UNMAPPED_ID(inode))
1805 		return false;
1806 
1807 	if (IS_NOATIME(inode))
1808 		return false;
1809 	if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))
1810 		return false;
1811 
1812 	if (mnt->mnt_flags & MNT_NOATIME)
1813 		return false;
1814 	if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))
1815 		return false;
1816 
1817 	now = current_time(inode);
1818 
1819 	if (!relatime_need_update(mnt, inode, now))
1820 		return false;
1821 
1822 	if (timespec64_equal(&inode->i_atime, &now))
1823 		return false;
1824 
1825 	return true;
1826 }
1827 
touch_atime(const struct path * path)1828 void touch_atime(const struct path *path)
1829 {
1830 	struct vfsmount *mnt = path->mnt;
1831 	struct inode *inode = d_inode(path->dentry);
1832 	struct timespec64 now;
1833 
1834 	if (!atime_needs_update(path, inode))
1835 		return;
1836 
1837 	if (!sb_start_write_trylock(inode->i_sb))
1838 		return;
1839 
1840 	if (__mnt_want_write(mnt) != 0)
1841 		goto skip_update;
1842 	/*
1843 	 * File systems can error out when updating inodes if they need to
1844 	 * allocate new space to modify an inode (such is the case for
1845 	 * Btrfs), but since we touch atime while walking down the path we
1846 	 * really don't care if we failed to update the atime of the file,
1847 	 * so just ignore the return value.
1848 	 * We may also fail on filesystems that have the ability to make parts
1849 	 * of the fs read only, e.g. subvolumes in Btrfs.
1850 	 */
1851 	now = current_time(inode);
1852 	inode_update_time(inode, &now, S_ATIME);
1853 	__mnt_drop_write(mnt);
1854 skip_update:
1855 	sb_end_write(inode->i_sb);
1856 }
1857 EXPORT_SYMBOL(touch_atime);
1858 
1859 /*
1860  * The logic we want is
1861  *
1862  *	if suid or (sgid and xgrp)
1863  *		remove privs
1864  */
should_remove_suid(struct dentry * dentry)1865 int should_remove_suid(struct dentry *dentry)
1866 {
1867 	umode_t mode = d_inode(dentry)->i_mode;
1868 	int kill = 0;
1869 
1870 	/* suid always must be killed */
1871 	if (unlikely(mode & S_ISUID))
1872 		kill = ATTR_KILL_SUID;
1873 
1874 	/*
1875 	 * sgid without any exec bits is just a mandatory locking mark; leave
1876 	 * it alone.  If some exec bits are set, it's a real sgid; kill it.
1877 	 */
1878 	if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1879 		kill |= ATTR_KILL_SGID;
1880 
1881 	if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode)))
1882 		return kill;
1883 
1884 	return 0;
1885 }
1886 EXPORT_SYMBOL(should_remove_suid);
1887 
1888 /*
1889  * Return mask of changes for notify_change() that need to be done as a
1890  * response to write or truncate. Return 0 if nothing has to be changed.
1891  * Negative value on error (change should be denied).
1892  */
dentry_needs_remove_privs(struct dentry * dentry)1893 int dentry_needs_remove_privs(struct dentry *dentry)
1894 {
1895 	struct inode *inode = d_inode(dentry);
1896 	int mask = 0;
1897 	int ret;
1898 
1899 	if (IS_NOSEC(inode))
1900 		return 0;
1901 
1902 	mask = should_remove_suid(dentry);
1903 	ret = security_inode_need_killpriv(dentry);
1904 	if (ret < 0)
1905 		return ret;
1906 	if (ret)
1907 		mask |= ATTR_KILL_PRIV;
1908 	return mask;
1909 }
1910 
__remove_privs(struct dentry * dentry,int kill)1911 static int __remove_privs(struct dentry *dentry, int kill)
1912 {
1913 	struct iattr newattrs;
1914 
1915 	newattrs.ia_valid = ATTR_FORCE | kill;
1916 	/*
1917 	 * Note we call this on write, so notify_change will not
1918 	 * encounter any conflicting delegations:
1919 	 */
1920 	return notify_change(dentry, &newattrs, NULL);
1921 }
1922 
1923 /*
1924  * Remove special file priviledges (suid, capabilities) when file is written
1925  * to or truncated.
1926  */
file_remove_privs(struct file * file)1927 int file_remove_privs(struct file *file)
1928 {
1929 	struct dentry *dentry = file_dentry(file);
1930 	struct inode *inode = file_inode(file);
1931 	int kill;
1932 	int error = 0;
1933 
1934 	/*
1935 	 * Fast path for nothing security related.
1936 	 * As well for non-regular files, e.g. blkdev inodes.
1937 	 * For example, blkdev_write_iter() might get here
1938 	 * trying to remove privs which it is not allowed to.
1939 	 */
1940 	if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode))
1941 		return 0;
1942 
1943 	kill = dentry_needs_remove_privs(dentry);
1944 	if (kill < 0)
1945 		return kill;
1946 	if (kill)
1947 		error = __remove_privs(dentry, kill);
1948 	if (!error)
1949 		inode_has_no_xattr(inode);
1950 
1951 	return error;
1952 }
1953 EXPORT_SYMBOL(file_remove_privs);
1954 
1955 /**
1956  *	file_update_time	-	update mtime and ctime time
1957  *	@file: file accessed
1958  *
1959  *	Update the mtime and ctime members of an inode and mark the inode
1960  *	for writeback.  Note that this function is meant exclusively for
1961  *	usage in the file write path of filesystems, and filesystems may
1962  *	choose to explicitly ignore update via this function with the
1963  *	S_NOCMTIME inode flag, e.g. for network filesystem where these
1964  *	timestamps are handled by the server.  This can return an error for
1965  *	file systems who need to allocate space in order to update an inode.
1966  */
1967 
file_update_time(struct file * file)1968 int file_update_time(struct file *file)
1969 {
1970 	struct inode *inode = file_inode(file);
1971 	struct timespec64 now;
1972 	int sync_it = 0;
1973 	int ret;
1974 
1975 	/* First try to exhaust all avenues to not sync */
1976 	if (IS_NOCMTIME(inode))
1977 		return 0;
1978 
1979 	now = current_time(inode);
1980 	if (!timespec64_equal(&inode->i_mtime, &now))
1981 		sync_it = S_MTIME;
1982 
1983 	if (!timespec64_equal(&inode->i_ctime, &now))
1984 		sync_it |= S_CTIME;
1985 
1986 	if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode))
1987 		sync_it |= S_VERSION;
1988 
1989 	if (!sync_it)
1990 		return 0;
1991 
1992 	/* Finally allowed to write? Takes lock. */
1993 	if (__mnt_want_write_file(file))
1994 		return 0;
1995 
1996 	ret = inode_update_time(inode, &now, sync_it);
1997 	__mnt_drop_write_file(file);
1998 
1999 	return ret;
2000 }
2001 EXPORT_SYMBOL(file_update_time);
2002 
2003 /* Caller must hold the file's inode lock */
file_modified(struct file * file)2004 int file_modified(struct file *file)
2005 {
2006 	int err;
2007 
2008 	/*
2009 	 * Clear the security bits if the process is not being run by root.
2010 	 * This keeps people from modifying setuid and setgid binaries.
2011 	 */
2012 	err = file_remove_privs(file);
2013 	if (err)
2014 		return err;
2015 
2016 	if (unlikely(file->f_mode & FMODE_NOCMTIME))
2017 		return 0;
2018 
2019 	return file_update_time(file);
2020 }
2021 EXPORT_SYMBOL(file_modified);
2022 
inode_needs_sync(struct inode * inode)2023 int inode_needs_sync(struct inode *inode)
2024 {
2025 	if (IS_SYNC(inode))
2026 		return 1;
2027 	if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode))
2028 		return 1;
2029 	return 0;
2030 }
2031 EXPORT_SYMBOL(inode_needs_sync);
2032 
2033 /*
2034  * If we try to find an inode in the inode hash while it is being
2035  * deleted, we have to wait until the filesystem completes its
2036  * deletion before reporting that it isn't found.  This function waits
2037  * until the deletion _might_ have completed.  Callers are responsible
2038  * to recheck inode state.
2039  *
2040  * It doesn't matter if I_NEW is not set initially, a call to
2041  * wake_up_bit(&inode->i_state, __I_NEW) after removing from the hash list
2042  * will DTRT.
2043  */
__wait_on_freeing_inode(struct inode * inode)2044 static void __wait_on_freeing_inode(struct inode *inode)
2045 {
2046 	wait_queue_head_t *wq;
2047 	DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
2048 	wq = bit_waitqueue(&inode->i_state, __I_NEW);
2049 	prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
2050 	spin_unlock(&inode->i_lock);
2051 	spin_unlock(&inode_hash_lock);
2052 	schedule();
2053 	finish_wait(wq, &wait.wq_entry);
2054 	spin_lock(&inode_hash_lock);
2055 }
2056 
2057 static __initdata unsigned long ihash_entries;
set_ihash_entries(char * str)2058 static int __init set_ihash_entries(char *str)
2059 {
2060 	if (!str)
2061 		return 0;
2062 	ihash_entries = simple_strtoul(str, &str, 0);
2063 	return 1;
2064 }
2065 __setup("ihash_entries=", set_ihash_entries);
2066 
2067 /*
2068  * Initialize the waitqueues and inode hash table.
2069  */
inode_init_early(void)2070 void __init inode_init_early(void)
2071 {
2072 	/* If hashes are distributed across NUMA nodes, defer
2073 	 * hash allocation until vmalloc space is available.
2074 	 */
2075 	if (hashdist)
2076 		return;
2077 
2078 	inode_hashtable =
2079 		alloc_large_system_hash("Inode-cache",
2080 					sizeof(struct hlist_head),
2081 					ihash_entries,
2082 					14,
2083 					HASH_EARLY | HASH_ZERO,
2084 					&i_hash_shift,
2085 					&i_hash_mask,
2086 					0,
2087 					0);
2088 }
2089 
inode_init(void)2090 void __init inode_init(void)
2091 {
2092 	/* inode slab cache */
2093 	inode_cachep = kmem_cache_create("inode_cache",
2094 					 sizeof(struct inode),
2095 					 0,
2096 					 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
2097 					 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
2098 					 init_once);
2099 
2100 	/* Hash may have been set up in inode_init_early */
2101 	if (!hashdist)
2102 		return;
2103 
2104 	inode_hashtable =
2105 		alloc_large_system_hash("Inode-cache",
2106 					sizeof(struct hlist_head),
2107 					ihash_entries,
2108 					14,
2109 					HASH_ZERO,
2110 					&i_hash_shift,
2111 					&i_hash_mask,
2112 					0,
2113 					0);
2114 }
2115 
init_special_inode(struct inode * inode,umode_t mode,dev_t rdev)2116 void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev)
2117 {
2118 	inode->i_mode = mode;
2119 	if (S_ISCHR(mode)) {
2120 		inode->i_fop = &def_chr_fops;
2121 		inode->i_rdev = rdev;
2122 	} else if (S_ISBLK(mode)) {
2123 		inode->i_fop = &def_blk_fops;
2124 		inode->i_rdev = rdev;
2125 	} else if (S_ISFIFO(mode))
2126 		inode->i_fop = &pipefifo_fops;
2127 	else if (S_ISSOCK(mode))
2128 		;	/* leave it no_open_fops */
2129 	else
2130 		printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o) for"
2131 				  " inode %s:%lu\n", mode, inode->i_sb->s_id,
2132 				  inode->i_ino);
2133 }
2134 EXPORT_SYMBOL(init_special_inode);
2135 
2136 /**
2137  * inode_init_owner - Init uid,gid,mode for new inode according to posix standards
2138  * @inode: New inode
2139  * @dir: Directory inode
2140  * @mode: mode of the new inode
2141  */
inode_init_owner(struct inode * inode,const struct inode * dir,umode_t mode)2142 void inode_init_owner(struct inode *inode, const struct inode *dir,
2143 			umode_t mode)
2144 {
2145 	inode->i_uid = current_fsuid();
2146 	if (dir && dir->i_mode & S_ISGID) {
2147 		inode->i_gid = dir->i_gid;
2148 
2149 		/* Directories are special, and always inherit S_ISGID */
2150 		if (S_ISDIR(mode))
2151 			mode |= S_ISGID;
2152 		else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
2153 			 !in_group_p(inode->i_gid) &&
2154 			 !capable_wrt_inode_uidgid(dir, CAP_FSETID))
2155 			mode &= ~S_ISGID;
2156 	} else
2157 		inode->i_gid = current_fsgid();
2158 	inode->i_mode = mode;
2159 }
2160 EXPORT_SYMBOL(inode_init_owner);
2161 
2162 /**
2163  * inode_owner_or_capable - check current task permissions to inode
2164  * @inode: inode being checked
2165  *
2166  * Return true if current either has CAP_FOWNER in a namespace with the
2167  * inode owner uid mapped, or owns the file.
2168  */
inode_owner_or_capable(const struct inode * inode)2169 bool inode_owner_or_capable(const struct inode *inode)
2170 {
2171 	struct user_namespace *ns;
2172 
2173 	if (uid_eq(current_fsuid(), inode->i_uid))
2174 		return true;
2175 
2176 	ns = current_user_ns();
2177 	if (kuid_has_mapping(ns, inode->i_uid) && ns_capable(ns, CAP_FOWNER))
2178 		return true;
2179 	return false;
2180 }
2181 EXPORT_SYMBOL(inode_owner_or_capable);
2182 
2183 /*
2184  * Direct i/o helper functions
2185  */
__inode_dio_wait(struct inode * inode)2186 static void __inode_dio_wait(struct inode *inode)
2187 {
2188 	wait_queue_head_t *wq = bit_waitqueue(&inode->i_state, __I_DIO_WAKEUP);
2189 	DEFINE_WAIT_BIT(q, &inode->i_state, __I_DIO_WAKEUP);
2190 
2191 	do {
2192 		prepare_to_wait(wq, &q.wq_entry, TASK_UNINTERRUPTIBLE);
2193 		if (atomic_read(&inode->i_dio_count))
2194 			schedule();
2195 	} while (atomic_read(&inode->i_dio_count));
2196 	finish_wait(wq, &q.wq_entry);
2197 }
2198 
2199 /**
2200  * inode_dio_wait - wait for outstanding DIO requests to finish
2201  * @inode: inode to wait for
2202  *
2203  * Waits for all pending direct I/O requests to finish so that we can
2204  * proceed with a truncate or equivalent operation.
2205  *
2206  * Must be called under a lock that serializes taking new references
2207  * to i_dio_count, usually by inode->i_mutex.
2208  */
inode_dio_wait(struct inode * inode)2209 void inode_dio_wait(struct inode *inode)
2210 {
2211 	if (atomic_read(&inode->i_dio_count))
2212 		__inode_dio_wait(inode);
2213 }
2214 EXPORT_SYMBOL(inode_dio_wait);
2215 
2216 /*
2217  * inode_set_flags - atomically set some inode flags
2218  *
2219  * Note: the caller should be holding i_mutex, or else be sure that
2220  * they have exclusive access to the inode structure (i.e., while the
2221  * inode is being instantiated).  The reason for the cmpxchg() loop
2222  * --- which wouldn't be necessary if all code paths which modify
2223  * i_flags actually followed this rule, is that there is at least one
2224  * code path which doesn't today so we use cmpxchg() out of an abundance
2225  * of caution.
2226  *
2227  * In the long run, i_mutex is overkill, and we should probably look
2228  * at using the i_lock spinlock to protect i_flags, and then make sure
2229  * it is so documented in include/linux/fs.h and that all code follows
2230  * the locking convention!!
2231  */
inode_set_flags(struct inode * inode,unsigned int flags,unsigned int mask)2232 void inode_set_flags(struct inode *inode, unsigned int flags,
2233 		     unsigned int mask)
2234 {
2235 	WARN_ON_ONCE(flags & ~mask);
2236 	set_mask_bits(&inode->i_flags, mask, flags);
2237 }
2238 EXPORT_SYMBOL(inode_set_flags);
2239 
inode_nohighmem(struct inode * inode)2240 void inode_nohighmem(struct inode *inode)
2241 {
2242 	mapping_set_gfp_mask(inode->i_mapping, GFP_USER);
2243 }
2244 EXPORT_SYMBOL(inode_nohighmem);
2245 
2246 /**
2247  * timestamp_truncate - Truncate timespec to a granularity
2248  * @t: Timespec
2249  * @inode: inode being updated
2250  *
2251  * Truncate a timespec to the granularity supported by the fs
2252  * containing the inode. Always rounds down. gran must
2253  * not be 0 nor greater than a second (NSEC_PER_SEC, or 10^9 ns).
2254  */
timestamp_truncate(struct timespec64 t,struct inode * inode)2255 struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode)
2256 {
2257 	struct super_block *sb = inode->i_sb;
2258 	unsigned int gran = sb->s_time_gran;
2259 
2260 	t.tv_sec = clamp(t.tv_sec, sb->s_time_min, sb->s_time_max);
2261 	if (unlikely(t.tv_sec == sb->s_time_max || t.tv_sec == sb->s_time_min))
2262 		t.tv_nsec = 0;
2263 
2264 	/* Avoid division in the common cases 1 ns and 1 s. */
2265 	if (gran == 1)
2266 		; /* nothing */
2267 	else if (gran == NSEC_PER_SEC)
2268 		t.tv_nsec = 0;
2269 	else if (gran > 1 && gran < NSEC_PER_SEC)
2270 		t.tv_nsec -= t.tv_nsec % gran;
2271 	else
2272 		WARN(1, "invalid file time granularity: %u", gran);
2273 	return t;
2274 }
2275 EXPORT_SYMBOL(timestamp_truncate);
2276 
2277 /**
2278  * current_time - Return FS time
2279  * @inode: inode.
2280  *
2281  * Return the current time truncated to the time granularity supported by
2282  * the fs.
2283  *
2284  * Note that inode and inode->sb cannot be NULL.
2285  * Otherwise, the function warns and returns time without truncation.
2286  */
current_time(struct inode * inode)2287 struct timespec64 current_time(struct inode *inode)
2288 {
2289 	struct timespec64 now;
2290 
2291 	ktime_get_coarse_real_ts64(&now);
2292 
2293 	if (unlikely(!inode->i_sb)) {
2294 		WARN(1, "current_time() called with uninitialized super_block in the inode");
2295 		return now;
2296 	}
2297 
2298 	return timestamp_truncate(now, inode);
2299 }
2300 EXPORT_SYMBOL(current_time);
2301 
2302 /*
2303  * Generic function to check FS_IOC_SETFLAGS values and reject any invalid
2304  * configurations.
2305  *
2306  * Note: the caller should be holding i_mutex, or else be sure that they have
2307  * exclusive access to the inode structure.
2308  */
vfs_ioc_setflags_prepare(struct inode * inode,unsigned int oldflags,unsigned int flags)2309 int vfs_ioc_setflags_prepare(struct inode *inode, unsigned int oldflags,
2310 			     unsigned int flags)
2311 {
2312 	/*
2313 	 * The IMMUTABLE and APPEND_ONLY flags can only be changed by
2314 	 * the relevant capability.
2315 	 *
2316 	 * This test looks nicer. Thanks to Pauline Middelink
2317 	 */
2318 	if ((flags ^ oldflags) & (FS_APPEND_FL | FS_IMMUTABLE_FL) &&
2319 	    !capable(CAP_LINUX_IMMUTABLE))
2320 		return -EPERM;
2321 
2322 	return fscrypt_prepare_setflags(inode, oldflags, flags);
2323 }
2324 EXPORT_SYMBOL(vfs_ioc_setflags_prepare);
2325 
2326 /*
2327  * Generic function to check FS_IOC_FSSETXATTR values and reject any invalid
2328  * configurations.
2329  *
2330  * Note: the caller should be holding i_mutex, or else be sure that they have
2331  * exclusive access to the inode structure.
2332  */
vfs_ioc_fssetxattr_check(struct inode * inode,const struct fsxattr * old_fa,struct fsxattr * fa)2333 int vfs_ioc_fssetxattr_check(struct inode *inode, const struct fsxattr *old_fa,
2334 			     struct fsxattr *fa)
2335 {
2336 	/*
2337 	 * Can't modify an immutable/append-only file unless we have
2338 	 * appropriate permission.
2339 	 */
2340 	if ((old_fa->fsx_xflags ^ fa->fsx_xflags) &
2341 			(FS_XFLAG_IMMUTABLE | FS_XFLAG_APPEND) &&
2342 	    !capable(CAP_LINUX_IMMUTABLE))
2343 		return -EPERM;
2344 
2345 	/*
2346 	 * Project Quota ID state is only allowed to change from within the init
2347 	 * namespace. Enforce that restriction only if we are trying to change
2348 	 * the quota ID state. Everything else is allowed in user namespaces.
2349 	 */
2350 	if (current_user_ns() != &init_user_ns) {
2351 		if (old_fa->fsx_projid != fa->fsx_projid)
2352 			return -EINVAL;
2353 		if ((old_fa->fsx_xflags ^ fa->fsx_xflags) &
2354 				FS_XFLAG_PROJINHERIT)
2355 			return -EINVAL;
2356 	}
2357 
2358 	/* Check extent size hints. */
2359 	if ((fa->fsx_xflags & FS_XFLAG_EXTSIZE) && !S_ISREG(inode->i_mode))
2360 		return -EINVAL;
2361 
2362 	if ((fa->fsx_xflags & FS_XFLAG_EXTSZINHERIT) &&
2363 			!S_ISDIR(inode->i_mode))
2364 		return -EINVAL;
2365 
2366 	if ((fa->fsx_xflags & FS_XFLAG_COWEXTSIZE) &&
2367 	    !S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))
2368 		return -EINVAL;
2369 
2370 	/*
2371 	 * It is only valid to set the DAX flag on regular files and
2372 	 * directories on filesystems.
2373 	 */
2374 	if ((fa->fsx_xflags & FS_XFLAG_DAX) &&
2375 	    !(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode)))
2376 		return -EINVAL;
2377 
2378 	/* Extent size hints of zero turn off the flags. */
2379 	if (fa->fsx_extsize == 0)
2380 		fa->fsx_xflags &= ~(FS_XFLAG_EXTSIZE | FS_XFLAG_EXTSZINHERIT);
2381 	if (fa->fsx_cowextsize == 0)
2382 		fa->fsx_xflags &= ~FS_XFLAG_COWEXTSIZE;
2383 
2384 	return 0;
2385 }
2386 EXPORT_SYMBOL(vfs_ioc_fssetxattr_check);
2387