• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * fs/fs-writeback.c
3  *
4  * Copyright (C) 2002, Linus Torvalds.
5  *
6  * Contains all the functions related to writing back and waiting
7  * upon dirty inodes against superblocks, and writing back dirty
8  * pages against inodes.  ie: data writeback.  Writeout of the
9  * inode itself is not handled here.
10  *
11  * 10Apr2002	Andrew Morton
12  *		Split out of fs/inode.c
13  *		Additions for address_space-based writeback
14  */
15 
16 #include <linux/kernel.h>
17 #include <linux/export.h>
18 #include <linux/spinlock.h>
19 #include <linux/slab.h>
20 #include <linux/sched.h>
21 #include <linux/fs.h>
22 #include <linux/mm.h>
23 #include <linux/pagemap.h>
24 #include <linux/kthread.h>
25 #include <linux/writeback.h>
26 #include <linux/blkdev.h>
27 #include <linux/backing-dev.h>
28 #include <linux/tracepoint.h>
29 #include <linux/device.h>
30 #include <linux/memcontrol.h>
31 #include "internal.h"
32 
33 /*
34  * 4MB minimal write chunk size
35  */
36 #define MIN_WRITEBACK_PAGES	(4096UL >> (PAGE_SHIFT - 10))
37 
38 struct wb_completion {
39 	atomic_t		cnt;
40 };
41 
42 /*
43  * Passed into wb_writeback(), essentially a subset of writeback_control
44  */
45 struct wb_writeback_work {
46 	long nr_pages;
47 	struct super_block *sb;
48 	unsigned long *older_than_this;
49 	enum writeback_sync_modes sync_mode;
50 	unsigned int tagged_writepages:1;
51 	unsigned int for_kupdate:1;
52 	unsigned int range_cyclic:1;
53 	unsigned int for_background:1;
54 	unsigned int for_sync:1;	/* sync(2) WB_SYNC_ALL writeback */
55 	unsigned int auto_free:1;	/* free on completion */
56 	enum wb_reason reason;		/* why was writeback initiated? */
57 
58 	struct list_head list;		/* pending work list */
59 	struct wb_completion *done;	/* set if the caller waits */
60 };
61 
62 /*
63  * If one wants to wait for one or more wb_writeback_works, each work's
64  * ->done should be set to a wb_completion defined using the following
65  * macro.  Once all work items are issued with wb_queue_work(), the caller
66  * can wait for the completion of all using wb_wait_for_completion().  Work
67  * items which are waited upon aren't freed automatically on completion.
68  */
69 #define DEFINE_WB_COMPLETION_ONSTACK(cmpl)				\
70 	struct wb_completion cmpl = {					\
71 		.cnt		= ATOMIC_INIT(1),			\
72 	}
73 
74 
75 /*
76  * If an inode is constantly having its pages dirtied, but then the
77  * updates stop dirtytime_expire_interval seconds in the past, it's
78  * possible for the worst case time between when an inode has its
79  * timestamps updated and when they finally get written out to be two
80  * dirtytime_expire_intervals.  We set the default to 12 hours (in
81  * seconds), which means most of the time inodes will have their
82  * timestamps written to disk after 12 hours, but in the worst case a
83  * few inodes might not their timestamps updated for 24 hours.
84  */
85 unsigned int dirtytime_expire_interval = 12 * 60 * 60;
86 
wb_inode(struct list_head * head)87 static inline struct inode *wb_inode(struct list_head *head)
88 {
89 	return list_entry(head, struct inode, i_io_list);
90 }
91 
92 /*
93  * Include the creation of the trace points after defining the
94  * wb_writeback_work structure and inline functions so that the definition
95  * remains local to this file.
96  */
97 #define CREATE_TRACE_POINTS
98 #include <trace/events/writeback.h>
99 
100 EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage);
101 
wb_io_lists_populated(struct bdi_writeback * wb)102 static bool wb_io_lists_populated(struct bdi_writeback *wb)
103 {
104 	if (wb_has_dirty_io(wb)) {
105 		return false;
106 	} else {
107 		set_bit(WB_has_dirty_io, &wb->state);
108 		WARN_ON_ONCE(!wb->avg_write_bandwidth);
109 		atomic_long_add(wb->avg_write_bandwidth,
110 				&wb->bdi->tot_write_bandwidth);
111 		return true;
112 	}
113 }
114 
wb_io_lists_depopulated(struct bdi_writeback * wb)115 static void wb_io_lists_depopulated(struct bdi_writeback *wb)
116 {
117 	if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) &&
118 	    list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) {
119 		clear_bit(WB_has_dirty_io, &wb->state);
120 		WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth,
121 					&wb->bdi->tot_write_bandwidth) < 0);
122 	}
123 }
124 
125 /**
126  * inode_io_list_move_locked - move an inode onto a bdi_writeback IO list
127  * @inode: inode to be moved
128  * @wb: target bdi_writeback
129  * @head: one of @wb->b_{dirty|io|more_io}
130  *
131  * Move @inode->i_io_list to @list of @wb and set %WB_has_dirty_io.
132  * Returns %true if @inode is the first occupant of the !dirty_time IO
133  * lists; otherwise, %false.
134  */
inode_io_list_move_locked(struct inode * inode,struct bdi_writeback * wb,struct list_head * head)135 static bool inode_io_list_move_locked(struct inode *inode,
136 				      struct bdi_writeback *wb,
137 				      struct list_head *head)
138 {
139 	assert_spin_locked(&wb->list_lock);
140 
141 	list_move(&inode->i_io_list, head);
142 
143 	/* dirty_time doesn't count as dirty_io until expiration */
144 	if (head != &wb->b_dirty_time)
145 		return wb_io_lists_populated(wb);
146 
147 	wb_io_lists_depopulated(wb);
148 	return false;
149 }
150 
151 /**
152  * inode_io_list_del_locked - remove an inode from its bdi_writeback IO list
153  * @inode: inode to be removed
154  * @wb: bdi_writeback @inode is being removed from
155  *
156  * Remove @inode which may be on one of @wb->b_{dirty|io|more_io} lists and
157  * clear %WB_has_dirty_io if all are empty afterwards.
158  */
inode_io_list_del_locked(struct inode * inode,struct bdi_writeback * wb)159 static void inode_io_list_del_locked(struct inode *inode,
160 				     struct bdi_writeback *wb)
161 {
162 	assert_spin_locked(&wb->list_lock);
163 
164 	list_del_init(&inode->i_io_list);
165 	wb_io_lists_depopulated(wb);
166 }
167 
wb_wakeup(struct bdi_writeback * wb)168 static void wb_wakeup(struct bdi_writeback *wb)
169 {
170 	spin_lock_bh(&wb->work_lock);
171 	if (test_bit(WB_registered, &wb->state))
172 		mod_delayed_work(bdi_wq, &wb->dwork, 0);
173 	spin_unlock_bh(&wb->work_lock);
174 }
175 
finish_writeback_work(struct bdi_writeback * wb,struct wb_writeback_work * work)176 static void finish_writeback_work(struct bdi_writeback *wb,
177 				  struct wb_writeback_work *work)
178 {
179 	struct wb_completion *done = work->done;
180 
181 	if (work->auto_free)
182 		kfree(work);
183 	if (done && atomic_dec_and_test(&done->cnt))
184 		wake_up_all(&wb->bdi->wb_waitq);
185 }
186 
wb_queue_work(struct bdi_writeback * wb,struct wb_writeback_work * work)187 static void wb_queue_work(struct bdi_writeback *wb,
188 			  struct wb_writeback_work *work)
189 {
190 	trace_writeback_queue(wb, work);
191 
192 	if (work->done)
193 		atomic_inc(&work->done->cnt);
194 
195 	spin_lock_bh(&wb->work_lock);
196 
197 	if (test_bit(WB_registered, &wb->state)) {
198 		list_add_tail(&work->list, &wb->work_list);
199 		mod_delayed_work(bdi_wq, &wb->dwork, 0);
200 	} else
201 		finish_writeback_work(wb, work);
202 
203 	spin_unlock_bh(&wb->work_lock);
204 }
205 
206 /**
207  * wb_wait_for_completion - wait for completion of bdi_writeback_works
208  * @bdi: bdi work items were issued to
209  * @done: target wb_completion
210  *
211  * Wait for one or more work items issued to @bdi with their ->done field
212  * set to @done, which should have been defined with
213  * DEFINE_WB_COMPLETION_ONSTACK().  This function returns after all such
214  * work items are completed.  Work items which are waited upon aren't freed
215  * automatically on completion.
216  */
wb_wait_for_completion(struct backing_dev_info * bdi,struct wb_completion * done)217 static void wb_wait_for_completion(struct backing_dev_info *bdi,
218 				   struct wb_completion *done)
219 {
220 	atomic_dec(&done->cnt);		/* put down the initial count */
221 	wait_event(bdi->wb_waitq, !atomic_read(&done->cnt));
222 }
223 
224 #ifdef CONFIG_CGROUP_WRITEBACK
225 
226 /* parameters for foreign inode detection, see wb_detach_inode() */
227 #define WB_FRN_TIME_SHIFT	13	/* 1s = 2^13, upto 8 secs w/ 16bit */
228 #define WB_FRN_TIME_AVG_SHIFT	3	/* avg = avg * 7/8 + new * 1/8 */
229 #define WB_FRN_TIME_CUT_DIV	2	/* ignore rounds < avg / 2 */
230 #define WB_FRN_TIME_PERIOD	(2 * (1 << WB_FRN_TIME_SHIFT))	/* 2s */
231 
232 #define WB_FRN_HIST_SLOTS	16	/* inode->i_wb_frn_history is 16bit */
233 #define WB_FRN_HIST_UNIT	(WB_FRN_TIME_PERIOD / WB_FRN_HIST_SLOTS)
234 					/* each slot's duration is 2s / 16 */
235 #define WB_FRN_HIST_THR_SLOTS	(WB_FRN_HIST_SLOTS / 2)
236 					/* if foreign slots >= 8, switch */
237 #define WB_FRN_HIST_MAX_SLOTS	(WB_FRN_HIST_THR_SLOTS / 2 + 1)
238 					/* one round can affect upto 5 slots */
239 
240 static atomic_t isw_nr_in_flight = ATOMIC_INIT(0);
241 static struct workqueue_struct *isw_wq;
242 
__inode_attach_wb(struct inode * inode,struct page * page)243 void __inode_attach_wb(struct inode *inode, struct page *page)
244 {
245 	struct backing_dev_info *bdi = inode_to_bdi(inode);
246 	struct bdi_writeback *wb = NULL;
247 
248 	if (inode_cgwb_enabled(inode)) {
249 		struct cgroup_subsys_state *memcg_css;
250 
251 		if (page) {
252 			memcg_css = mem_cgroup_css_from_page(page);
253 			wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
254 		} else {
255 			/* must pin memcg_css, see wb_get_create() */
256 			memcg_css = task_get_css(current, memory_cgrp_id);
257 			wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
258 			css_put(memcg_css);
259 		}
260 	}
261 
262 	if (!wb)
263 		wb = &bdi->wb;
264 
265 	/*
266 	 * There may be multiple instances of this function racing to
267 	 * update the same inode.  Use cmpxchg() to tell the winner.
268 	 */
269 	if (unlikely(cmpxchg(&inode->i_wb, NULL, wb)))
270 		wb_put(wb);
271 }
272 
273 /**
274  * locked_inode_to_wb_and_lock_list - determine a locked inode's wb and lock it
275  * @inode: inode of interest with i_lock held
276  *
277  * Returns @inode's wb with its list_lock held.  @inode->i_lock must be
278  * held on entry and is released on return.  The returned wb is guaranteed
279  * to stay @inode's associated wb until its list_lock is released.
280  */
281 static struct bdi_writeback *
locked_inode_to_wb_and_lock_list(struct inode * inode)282 locked_inode_to_wb_and_lock_list(struct inode *inode)
283 	__releases(&inode->i_lock)
284 	__acquires(&wb->list_lock)
285 {
286 	while (true) {
287 		struct bdi_writeback *wb = inode_to_wb(inode);
288 
289 		/*
290 		 * inode_to_wb() association is protected by both
291 		 * @inode->i_lock and @wb->list_lock but list_lock nests
292 		 * outside i_lock.  Drop i_lock and verify that the
293 		 * association hasn't changed after acquiring list_lock.
294 		 */
295 		wb_get(wb);
296 		spin_unlock(&inode->i_lock);
297 		spin_lock(&wb->list_lock);
298 
299 		/* i_wb may have changed inbetween, can't use inode_to_wb() */
300 		if (likely(wb == inode->i_wb)) {
301 			wb_put(wb);	/* @inode already has ref */
302 			return wb;
303 		}
304 
305 		spin_unlock(&wb->list_lock);
306 		wb_put(wb);
307 		cpu_relax();
308 		spin_lock(&inode->i_lock);
309 	}
310 }
311 
312 /**
313  * inode_to_wb_and_lock_list - determine an inode's wb and lock it
314  * @inode: inode of interest
315  *
316  * Same as locked_inode_to_wb_and_lock_list() but @inode->i_lock isn't held
317  * on entry.
318  */
inode_to_wb_and_lock_list(struct inode * inode)319 static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)
320 	__acquires(&wb->list_lock)
321 {
322 	spin_lock(&inode->i_lock);
323 	return locked_inode_to_wb_and_lock_list(inode);
324 }
325 
326 struct inode_switch_wbs_context {
327 	struct inode		*inode;
328 	struct bdi_writeback	*new_wb;
329 
330 	struct rcu_head		rcu_head;
331 	struct work_struct	work;
332 };
333 
inode_switch_wbs_work_fn(struct work_struct * work)334 static void inode_switch_wbs_work_fn(struct work_struct *work)
335 {
336 	struct inode_switch_wbs_context *isw =
337 		container_of(work, struct inode_switch_wbs_context, work);
338 	struct inode *inode = isw->inode;
339 	struct address_space *mapping = inode->i_mapping;
340 	struct bdi_writeback *old_wb = inode->i_wb;
341 	struct bdi_writeback *new_wb = isw->new_wb;
342 	struct radix_tree_iter iter;
343 	bool switched = false;
344 	void **slot;
345 
346 	/*
347 	 * By the time control reaches here, RCU grace period has passed
348 	 * since I_WB_SWITCH assertion and all wb stat update transactions
349 	 * between unlocked_inode_to_wb_begin/end() are guaranteed to be
350 	 * synchronizing against mapping->tree_lock.
351 	 *
352 	 * Grabbing old_wb->list_lock, inode->i_lock and mapping->tree_lock
353 	 * gives us exclusion against all wb related operations on @inode
354 	 * including IO list manipulations and stat updates.
355 	 */
356 	if (old_wb < new_wb) {
357 		spin_lock(&old_wb->list_lock);
358 		spin_lock_nested(&new_wb->list_lock, SINGLE_DEPTH_NESTING);
359 	} else {
360 		spin_lock(&new_wb->list_lock);
361 		spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING);
362 	}
363 	spin_lock(&inode->i_lock);
364 	spin_lock_irq(&mapping->tree_lock);
365 
366 	/*
367 	 * Once I_FREEING is visible under i_lock, the eviction path owns
368 	 * the inode and we shouldn't modify ->i_io_list.
369 	 */
370 	if (unlikely(inode->i_state & I_FREEING))
371 		goto skip_switch;
372 
373 	/*
374 	 * Count and transfer stats.  Note that PAGECACHE_TAG_DIRTY points
375 	 * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to
376 	 * pages actually under underwriteback.
377 	 */
378 	radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0,
379 				   PAGECACHE_TAG_DIRTY) {
380 		struct page *page = radix_tree_deref_slot_protected(slot,
381 							&mapping->tree_lock);
382 		if (likely(page) && PageDirty(page)) {
383 			__dec_wb_stat(old_wb, WB_RECLAIMABLE);
384 			__inc_wb_stat(new_wb, WB_RECLAIMABLE);
385 		}
386 	}
387 
388 	radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0,
389 				   PAGECACHE_TAG_WRITEBACK) {
390 		struct page *page = radix_tree_deref_slot_protected(slot,
391 							&mapping->tree_lock);
392 		if (likely(page)) {
393 			WARN_ON_ONCE(!PageWriteback(page));
394 			__dec_wb_stat(old_wb, WB_WRITEBACK);
395 			__inc_wb_stat(new_wb, WB_WRITEBACK);
396 		}
397 	}
398 
399 	wb_get(new_wb);
400 
401 	/*
402 	 * Transfer to @new_wb's IO list if necessary.  The specific list
403 	 * @inode was on is ignored and the inode is put on ->b_dirty which
404 	 * is always correct including from ->b_dirty_time.  The transfer
405 	 * preserves @inode->dirtied_when ordering.
406 	 */
407 	if (!list_empty(&inode->i_io_list)) {
408 		struct inode *pos;
409 
410 		inode_io_list_del_locked(inode, old_wb);
411 		inode->i_wb = new_wb;
412 		list_for_each_entry(pos, &new_wb->b_dirty, i_io_list)
413 			if (time_after_eq(inode->dirtied_when,
414 					  pos->dirtied_when))
415 				break;
416 		inode_io_list_move_locked(inode, new_wb, pos->i_io_list.prev);
417 	} else {
418 		inode->i_wb = new_wb;
419 	}
420 
421 	/* ->i_wb_frn updates may race wbc_detach_inode() but doesn't matter */
422 	inode->i_wb_frn_winner = 0;
423 	inode->i_wb_frn_avg_time = 0;
424 	inode->i_wb_frn_history = 0;
425 	switched = true;
426 skip_switch:
427 	/*
428 	 * Paired with load_acquire in unlocked_inode_to_wb_begin() and
429 	 * ensures that the new wb is visible if they see !I_WB_SWITCH.
430 	 */
431 	smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH);
432 
433 	spin_unlock_irq(&mapping->tree_lock);
434 	spin_unlock(&inode->i_lock);
435 	spin_unlock(&new_wb->list_lock);
436 	spin_unlock(&old_wb->list_lock);
437 
438 	if (switched) {
439 		wb_wakeup(new_wb);
440 		wb_put(old_wb);
441 	}
442 	wb_put(new_wb);
443 
444 	iput(inode);
445 	kfree(isw);
446 
447 	atomic_dec(&isw_nr_in_flight);
448 }
449 
inode_switch_wbs_rcu_fn(struct rcu_head * rcu_head)450 static void inode_switch_wbs_rcu_fn(struct rcu_head *rcu_head)
451 {
452 	struct inode_switch_wbs_context *isw = container_of(rcu_head,
453 				struct inode_switch_wbs_context, rcu_head);
454 
455 	/* needs to grab bh-unsafe locks, bounce to work item */
456 	INIT_WORK(&isw->work, inode_switch_wbs_work_fn);
457 	queue_work(isw_wq, &isw->work);
458 }
459 
460 /**
461  * inode_switch_wbs - change the wb association of an inode
462  * @inode: target inode
463  * @new_wb_id: ID of the new wb
464  *
465  * Switch @inode's wb association to the wb identified by @new_wb_id.  The
466  * switching is performed asynchronously and may fail silently.
467  */
inode_switch_wbs(struct inode * inode,int new_wb_id)468 static void inode_switch_wbs(struct inode *inode, int new_wb_id)
469 {
470 	struct backing_dev_info *bdi = inode_to_bdi(inode);
471 	struct cgroup_subsys_state *memcg_css;
472 	struct inode_switch_wbs_context *isw;
473 
474 	/* noop if seems to be already in progress */
475 	if (inode->i_state & I_WB_SWITCH)
476 		return;
477 
478 	isw = kzalloc(sizeof(*isw), GFP_ATOMIC);
479 	if (!isw)
480 		return;
481 
482 	/* find and pin the new wb */
483 	rcu_read_lock();
484 	memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys);
485 	if (memcg_css)
486 		isw->new_wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
487 	rcu_read_unlock();
488 	if (!isw->new_wb)
489 		goto out_free;
490 
491 	/* while holding I_WB_SWITCH, no one else can update the association */
492 	spin_lock(&inode->i_lock);
493 	if (!(inode->i_sb->s_flags & MS_ACTIVE) ||
494 	    inode->i_state & (I_WB_SWITCH | I_FREEING) ||
495 	    inode_to_wb(inode) == isw->new_wb) {
496 		spin_unlock(&inode->i_lock);
497 		goto out_free;
498 	}
499 	inode->i_state |= I_WB_SWITCH;
500 	__iget(inode);
501 	spin_unlock(&inode->i_lock);
502 
503 	isw->inode = inode;
504 
505 	atomic_inc(&isw_nr_in_flight);
506 
507 	/*
508 	 * In addition to synchronizing among switchers, I_WB_SWITCH tells
509 	 * the RCU protected stat update paths to grab the mapping's
510 	 * tree_lock so that stat transfer can synchronize against them.
511 	 * Let's continue after I_WB_SWITCH is guaranteed to be visible.
512 	 */
513 	call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn);
514 	return;
515 
516 out_free:
517 	if (isw->new_wb)
518 		wb_put(isw->new_wb);
519 	kfree(isw);
520 }
521 
522 /**
523  * wbc_attach_and_unlock_inode - associate wbc with target inode and unlock it
524  * @wbc: writeback_control of interest
525  * @inode: target inode
526  *
527  * @inode is locked and about to be written back under the control of @wbc.
528  * Record @inode's writeback context into @wbc and unlock the i_lock.  On
529  * writeback completion, wbc_detach_inode() should be called.  This is used
530  * to track the cgroup writeback context.
531  */
wbc_attach_and_unlock_inode(struct writeback_control * wbc,struct inode * inode)532 void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
533 				 struct inode *inode)
534 {
535 	if (!inode_cgwb_enabled(inode)) {
536 		spin_unlock(&inode->i_lock);
537 		return;
538 	}
539 
540 	wbc->wb = inode_to_wb(inode);
541 	wbc->inode = inode;
542 
543 	wbc->wb_id = wbc->wb->memcg_css->id;
544 	wbc->wb_lcand_id = inode->i_wb_frn_winner;
545 	wbc->wb_tcand_id = 0;
546 	wbc->wb_bytes = 0;
547 	wbc->wb_lcand_bytes = 0;
548 	wbc->wb_tcand_bytes = 0;
549 
550 	wb_get(wbc->wb);
551 	spin_unlock(&inode->i_lock);
552 
553 	/*
554 	 * A dying wb indicates that the memcg-blkcg mapping has changed
555 	 * and a new wb is already serving the memcg.  Switch immediately.
556 	 */
557 	if (unlikely(wb_dying(wbc->wb)))
558 		inode_switch_wbs(inode, wbc->wb_id);
559 }
560 
561 /**
562  * wbc_detach_inode - disassociate wbc from inode and perform foreign detection
563  * @wbc: writeback_control of the just finished writeback
564  *
565  * To be called after a writeback attempt of an inode finishes and undoes
566  * wbc_attach_and_unlock_inode().  Can be called under any context.
567  *
568  * As concurrent write sharing of an inode is expected to be very rare and
569  * memcg only tracks page ownership on first-use basis severely confining
570  * the usefulness of such sharing, cgroup writeback tracks ownership
571  * per-inode.  While the support for concurrent write sharing of an inode
572  * is deemed unnecessary, an inode being written to by different cgroups at
573  * different points in time is a lot more common, and, more importantly,
574  * charging only by first-use can too readily lead to grossly incorrect
575  * behaviors (single foreign page can lead to gigabytes of writeback to be
576  * incorrectly attributed).
577  *
578  * To resolve this issue, cgroup writeback detects the majority dirtier of
579  * an inode and transfers the ownership to it.  To avoid unnnecessary
580  * oscillation, the detection mechanism keeps track of history and gives
581  * out the switch verdict only if the foreign usage pattern is stable over
582  * a certain amount of time and/or writeback attempts.
583  *
584  * On each writeback attempt, @wbc tries to detect the majority writer
585  * using Boyer-Moore majority vote algorithm.  In addition to the byte
586  * count from the majority voting, it also counts the bytes written for the
587  * current wb and the last round's winner wb (max of last round's current
588  * wb, the winner from two rounds ago, and the last round's majority
589  * candidate).  Keeping track of the historical winner helps the algorithm
590  * to semi-reliably detect the most active writer even when it's not the
591  * absolute majority.
592  *
593  * Once the winner of the round is determined, whether the winner is
594  * foreign or not and how much IO time the round consumed is recorded in
595  * inode->i_wb_frn_history.  If the amount of recorded foreign IO time is
596  * over a certain threshold, the switch verdict is given.
597  */
wbc_detach_inode(struct writeback_control * wbc)598 void wbc_detach_inode(struct writeback_control *wbc)
599 {
600 	struct bdi_writeback *wb = wbc->wb;
601 	struct inode *inode = wbc->inode;
602 	unsigned long avg_time, max_bytes, max_time;
603 	u16 history;
604 	int max_id;
605 
606 	if (!wb)
607 		return;
608 
609 	history = inode->i_wb_frn_history;
610 	avg_time = inode->i_wb_frn_avg_time;
611 
612 	/* pick the winner of this round */
613 	if (wbc->wb_bytes >= wbc->wb_lcand_bytes &&
614 	    wbc->wb_bytes >= wbc->wb_tcand_bytes) {
615 		max_id = wbc->wb_id;
616 		max_bytes = wbc->wb_bytes;
617 	} else if (wbc->wb_lcand_bytes >= wbc->wb_tcand_bytes) {
618 		max_id = wbc->wb_lcand_id;
619 		max_bytes = wbc->wb_lcand_bytes;
620 	} else {
621 		max_id = wbc->wb_tcand_id;
622 		max_bytes = wbc->wb_tcand_bytes;
623 	}
624 
625 	/*
626 	 * Calculate the amount of IO time the winner consumed and fold it
627 	 * into the running average kept per inode.  If the consumed IO
628 	 * time is lower than avag / WB_FRN_TIME_CUT_DIV, ignore it for
629 	 * deciding whether to switch or not.  This is to prevent one-off
630 	 * small dirtiers from skewing the verdict.
631 	 */
632 	max_time = DIV_ROUND_UP((max_bytes >> PAGE_SHIFT) << WB_FRN_TIME_SHIFT,
633 				wb->avg_write_bandwidth);
634 	if (avg_time)
635 		avg_time += (max_time >> WB_FRN_TIME_AVG_SHIFT) -
636 			    (avg_time >> WB_FRN_TIME_AVG_SHIFT);
637 	else
638 		avg_time = max_time;	/* immediate catch up on first run */
639 
640 	if (max_time >= avg_time / WB_FRN_TIME_CUT_DIV) {
641 		int slots;
642 
643 		/*
644 		 * The switch verdict is reached if foreign wb's consume
645 		 * more than a certain proportion of IO time in a
646 		 * WB_FRN_TIME_PERIOD.  This is loosely tracked by 16 slot
647 		 * history mask where each bit represents one sixteenth of
648 		 * the period.  Determine the number of slots to shift into
649 		 * history from @max_time.
650 		 */
651 		slots = min(DIV_ROUND_UP(max_time, WB_FRN_HIST_UNIT),
652 			    (unsigned long)WB_FRN_HIST_MAX_SLOTS);
653 		history <<= slots;
654 		if (wbc->wb_id != max_id)
655 			history |= (1U << slots) - 1;
656 
657 		/*
658 		 * Switch if the current wb isn't the consistent winner.
659 		 * If there are multiple closely competing dirtiers, the
660 		 * inode may switch across them repeatedly over time, which
661 		 * is okay.  The main goal is avoiding keeping an inode on
662 		 * the wrong wb for an extended period of time.
663 		 */
664 		if (hweight32(history) > WB_FRN_HIST_THR_SLOTS)
665 			inode_switch_wbs(inode, max_id);
666 	}
667 
668 	/*
669 	 * Multiple instances of this function may race to update the
670 	 * following fields but we don't mind occassional inaccuracies.
671 	 */
672 	inode->i_wb_frn_winner = max_id;
673 	inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX);
674 	inode->i_wb_frn_history = history;
675 
676 	wb_put(wbc->wb);
677 	wbc->wb = NULL;
678 }
679 
680 /**
681  * wbc_account_io - account IO issued during writeback
682  * @wbc: writeback_control of the writeback in progress
683  * @page: page being written out
684  * @bytes: number of bytes being written out
685  *
686  * @bytes from @page are about to written out during the writeback
687  * controlled by @wbc.  Keep the book for foreign inode detection.  See
688  * wbc_detach_inode().
689  */
wbc_account_io(struct writeback_control * wbc,struct page * page,size_t bytes)690 void wbc_account_io(struct writeback_control *wbc, struct page *page,
691 		    size_t bytes)
692 {
693 	int id;
694 
695 	/*
696 	 * pageout() path doesn't attach @wbc to the inode being written
697 	 * out.  This is intentional as we don't want the function to block
698 	 * behind a slow cgroup.  Ultimately, we want pageout() to kick off
699 	 * regular writeback instead of writing things out itself.
700 	 */
701 	if (!wbc->wb)
702 		return;
703 
704 	id = mem_cgroup_css_from_page(page)->id;
705 
706 	if (id == wbc->wb_id) {
707 		wbc->wb_bytes += bytes;
708 		return;
709 	}
710 
711 	if (id == wbc->wb_lcand_id)
712 		wbc->wb_lcand_bytes += bytes;
713 
714 	/* Boyer-Moore majority vote algorithm */
715 	if (!wbc->wb_tcand_bytes)
716 		wbc->wb_tcand_id = id;
717 	if (id == wbc->wb_tcand_id)
718 		wbc->wb_tcand_bytes += bytes;
719 	else
720 		wbc->wb_tcand_bytes -= min(bytes, wbc->wb_tcand_bytes);
721 }
722 EXPORT_SYMBOL_GPL(wbc_account_io);
723 
724 /**
725  * inode_congested - test whether an inode is congested
726  * @inode: inode to test for congestion (may be NULL)
727  * @cong_bits: mask of WB_[a]sync_congested bits to test
728  *
729  * Tests whether @inode is congested.  @cong_bits is the mask of congestion
730  * bits to test and the return value is the mask of set bits.
731  *
732  * If cgroup writeback is enabled for @inode, the congestion state is
733  * determined by whether the cgwb (cgroup bdi_writeback) for the blkcg
734  * associated with @inode is congested; otherwise, the root wb's congestion
735  * state is used.
736  *
737  * @inode is allowed to be NULL as this function is often called on
738  * mapping->host which is NULL for the swapper space.
739  */
inode_congested(struct inode * inode,int cong_bits)740 int inode_congested(struct inode *inode, int cong_bits)
741 {
742 	/*
743 	 * Once set, ->i_wb never becomes NULL while the inode is alive.
744 	 * Start transaction iff ->i_wb is visible.
745 	 */
746 	if (inode && inode_to_wb_is_valid(inode)) {
747 		struct bdi_writeback *wb;
748 		struct wb_lock_cookie lock_cookie = {};
749 		bool congested;
750 
751 		wb = unlocked_inode_to_wb_begin(inode, &lock_cookie);
752 		congested = wb_congested(wb, cong_bits);
753 		unlocked_inode_to_wb_end(inode, &lock_cookie);
754 		return congested;
755 	}
756 
757 	return wb_congested(&inode_to_bdi(inode)->wb, cong_bits);
758 }
759 EXPORT_SYMBOL_GPL(inode_congested);
760 
761 /**
762  * wb_split_bdi_pages - split nr_pages to write according to bandwidth
763  * @wb: target bdi_writeback to split @nr_pages to
764  * @nr_pages: number of pages to write for the whole bdi
765  *
766  * Split @wb's portion of @nr_pages according to @wb's write bandwidth in
767  * relation to the total write bandwidth of all wb's w/ dirty inodes on
768  * @wb->bdi.
769  */
wb_split_bdi_pages(struct bdi_writeback * wb,long nr_pages)770 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)
771 {
772 	unsigned long this_bw = wb->avg_write_bandwidth;
773 	unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
774 
775 	if (nr_pages == LONG_MAX)
776 		return LONG_MAX;
777 
778 	/*
779 	 * This may be called on clean wb's and proportional distribution
780 	 * may not make sense, just use the original @nr_pages in those
781 	 * cases.  In general, we wanna err on the side of writing more.
782 	 */
783 	if (!tot_bw || this_bw >= tot_bw)
784 		return nr_pages;
785 	else
786 		return DIV_ROUND_UP_ULL((u64)nr_pages * this_bw, tot_bw);
787 }
788 
789 /**
790  * bdi_split_work_to_wbs - split a wb_writeback_work to all wb's of a bdi
791  * @bdi: target backing_dev_info
792  * @base_work: wb_writeback_work to issue
793  * @skip_if_busy: skip wb's which already have writeback in progress
794  *
795  * Split and issue @base_work to all wb's (bdi_writeback's) of @bdi which
796  * have dirty inodes.  If @base_work->nr_page isn't %LONG_MAX, it's
797  * distributed to the busy wbs according to each wb's proportion in the
798  * total active write bandwidth of @bdi.
799  */
bdi_split_work_to_wbs(struct backing_dev_info * bdi,struct wb_writeback_work * base_work,bool skip_if_busy)800 static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
801 				  struct wb_writeback_work *base_work,
802 				  bool skip_if_busy)
803 {
804 	struct bdi_writeback *last_wb = NULL;
805 	struct bdi_writeback *wb = list_entry(&bdi->wb_list,
806 					      struct bdi_writeback, bdi_node);
807 
808 	might_sleep();
809 restart:
810 	rcu_read_lock();
811 	list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) {
812 		DEFINE_WB_COMPLETION_ONSTACK(fallback_work_done);
813 		struct wb_writeback_work fallback_work;
814 		struct wb_writeback_work *work;
815 		long nr_pages;
816 
817 		if (last_wb) {
818 			wb_put(last_wb);
819 			last_wb = NULL;
820 		}
821 
822 		/* SYNC_ALL writes out I_DIRTY_TIME too */
823 		if (!wb_has_dirty_io(wb) &&
824 		    (base_work->sync_mode == WB_SYNC_NONE ||
825 		     list_empty(&wb->b_dirty_time)))
826 			continue;
827 		if (skip_if_busy && writeback_in_progress(wb))
828 			continue;
829 
830 		nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages);
831 
832 		work = kmalloc(sizeof(*work), GFP_ATOMIC);
833 		if (work) {
834 			*work = *base_work;
835 			work->nr_pages = nr_pages;
836 			work->auto_free = 1;
837 			wb_queue_work(wb, work);
838 			continue;
839 		}
840 
841 		/* alloc failed, execute synchronously using on-stack fallback */
842 		work = &fallback_work;
843 		*work = *base_work;
844 		work->nr_pages = nr_pages;
845 		work->auto_free = 0;
846 		work->done = &fallback_work_done;
847 
848 		wb_queue_work(wb, work);
849 
850 		/*
851 		 * Pin @wb so that it stays on @bdi->wb_list.  This allows
852 		 * continuing iteration from @wb after dropping and
853 		 * regrabbing rcu read lock.
854 		 */
855 		wb_get(wb);
856 		last_wb = wb;
857 
858 		rcu_read_unlock();
859 		wb_wait_for_completion(bdi, &fallback_work_done);
860 		goto restart;
861 	}
862 	rcu_read_unlock();
863 
864 	if (last_wb)
865 		wb_put(last_wb);
866 }
867 
868 /**
869  * cgroup_writeback_umount - flush inode wb switches for umount
870  *
871  * This function is called when a super_block is about to be destroyed and
872  * flushes in-flight inode wb switches.  An inode wb switch goes through
873  * RCU and then workqueue, so the two need to be flushed in order to ensure
874  * that all previously scheduled switches are finished.  As wb switches are
875  * rare occurrences and synchronize_rcu() can take a while, perform
876  * flushing iff wb switches are in flight.
877  */
cgroup_writeback_umount(void)878 void cgroup_writeback_umount(void)
879 {
880 	if (atomic_read(&isw_nr_in_flight)) {
881 		synchronize_rcu();
882 		flush_workqueue(isw_wq);
883 	}
884 }
885 
cgroup_writeback_init(void)886 static int __init cgroup_writeback_init(void)
887 {
888 	isw_wq = alloc_workqueue("inode_switch_wbs", 0, 0);
889 	if (!isw_wq)
890 		return -ENOMEM;
891 	return 0;
892 }
893 fs_initcall(cgroup_writeback_init);
894 
895 #else	/* CONFIG_CGROUP_WRITEBACK */
896 
897 static struct bdi_writeback *
locked_inode_to_wb_and_lock_list(struct inode * inode)898 locked_inode_to_wb_and_lock_list(struct inode *inode)
899 	__releases(&inode->i_lock)
900 	__acquires(&wb->list_lock)
901 {
902 	struct bdi_writeback *wb = inode_to_wb(inode);
903 
904 	spin_unlock(&inode->i_lock);
905 	spin_lock(&wb->list_lock);
906 	return wb;
907 }
908 
inode_to_wb_and_lock_list(struct inode * inode)909 static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)
910 	__acquires(&wb->list_lock)
911 {
912 	struct bdi_writeback *wb = inode_to_wb(inode);
913 
914 	spin_lock(&wb->list_lock);
915 	return wb;
916 }
917 
wb_split_bdi_pages(struct bdi_writeback * wb,long nr_pages)918 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)
919 {
920 	return nr_pages;
921 }
922 
bdi_split_work_to_wbs(struct backing_dev_info * bdi,struct wb_writeback_work * base_work,bool skip_if_busy)923 static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
924 				  struct wb_writeback_work *base_work,
925 				  bool skip_if_busy)
926 {
927 	might_sleep();
928 
929 	if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) {
930 		base_work->auto_free = 0;
931 		wb_queue_work(&bdi->wb, base_work);
932 	}
933 }
934 
935 #endif	/* CONFIG_CGROUP_WRITEBACK */
936 
wb_start_writeback(struct bdi_writeback * wb,long nr_pages,bool range_cyclic,enum wb_reason reason)937 void wb_start_writeback(struct bdi_writeback *wb, long nr_pages,
938 			bool range_cyclic, enum wb_reason reason)
939 {
940 	struct wb_writeback_work *work;
941 
942 	if (!wb_has_dirty_io(wb))
943 		return;
944 
945 	/*
946 	 * This is WB_SYNC_NONE writeback, so if allocation fails just
947 	 * wakeup the thread for old dirty data writeback
948 	 */
949 	work = kzalloc(sizeof(*work),
950 		       GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
951 	if (!work) {
952 		trace_writeback_nowork(wb);
953 		wb_wakeup(wb);
954 		return;
955 	}
956 
957 	work->sync_mode	= WB_SYNC_NONE;
958 	work->nr_pages	= nr_pages;
959 	work->range_cyclic = range_cyclic;
960 	work->reason	= reason;
961 	work->auto_free	= 1;
962 
963 	wb_queue_work(wb, work);
964 }
965 
966 /**
967  * wb_start_background_writeback - start background writeback
968  * @wb: bdi_writback to write from
969  *
970  * Description:
971  *   This makes sure WB_SYNC_NONE background writeback happens. When
972  *   this function returns, it is only guaranteed that for given wb
973  *   some IO is happening if we are over background dirty threshold.
974  *   Caller need not hold sb s_umount semaphore.
975  */
wb_start_background_writeback(struct bdi_writeback * wb)976 void wb_start_background_writeback(struct bdi_writeback *wb)
977 {
978 	/*
979 	 * We just wake up the flusher thread. It will perform background
980 	 * writeback as soon as there is no other work to do.
981 	 */
982 	trace_writeback_wake_background(wb);
983 	wb_wakeup(wb);
984 }
985 
986 /*
987  * Remove the inode from the writeback list it is on.
988  */
inode_io_list_del(struct inode * inode)989 void inode_io_list_del(struct inode *inode)
990 {
991 	struct bdi_writeback *wb;
992 
993 	wb = inode_to_wb_and_lock_list(inode);
994 	inode_io_list_del_locked(inode, wb);
995 	spin_unlock(&wb->list_lock);
996 }
997 
998 /*
999  * mark an inode as under writeback on the sb
1000  */
sb_mark_inode_writeback(struct inode * inode)1001 void sb_mark_inode_writeback(struct inode *inode)
1002 {
1003 	struct super_block *sb = inode->i_sb;
1004 	unsigned long flags;
1005 
1006 	if (list_empty(&inode->i_wb_list)) {
1007 		spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);
1008 		if (list_empty(&inode->i_wb_list)) {
1009 			list_add_tail(&inode->i_wb_list, &sb->s_inodes_wb);
1010 			trace_sb_mark_inode_writeback(inode);
1011 		}
1012 		spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);
1013 	}
1014 }
1015 
1016 /*
1017  * clear an inode as under writeback on the sb
1018  */
sb_clear_inode_writeback(struct inode * inode)1019 void sb_clear_inode_writeback(struct inode *inode)
1020 {
1021 	struct super_block *sb = inode->i_sb;
1022 	unsigned long flags;
1023 
1024 	if (!list_empty(&inode->i_wb_list)) {
1025 		spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);
1026 		if (!list_empty(&inode->i_wb_list)) {
1027 			list_del_init(&inode->i_wb_list);
1028 			trace_sb_clear_inode_writeback(inode);
1029 		}
1030 		spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);
1031 	}
1032 }
1033 
1034 /*
1035  * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
1036  * furthest end of its superblock's dirty-inode list.
1037  *
1038  * Before stamping the inode's ->dirtied_when, we check to see whether it is
1039  * already the most-recently-dirtied inode on the b_dirty list.  If that is
1040  * the case then the inode must have been redirtied while it was being written
1041  * out and we don't reset its dirtied_when.
1042  */
redirty_tail(struct inode * inode,struct bdi_writeback * wb)1043 static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)
1044 {
1045 	if (!list_empty(&wb->b_dirty)) {
1046 		struct inode *tail;
1047 
1048 		tail = wb_inode(wb->b_dirty.next);
1049 		if (time_before(inode->dirtied_when, tail->dirtied_when))
1050 			inode->dirtied_when = jiffies;
1051 	}
1052 	inode_io_list_move_locked(inode, wb, &wb->b_dirty);
1053 }
1054 
1055 /*
1056  * requeue inode for re-scanning after bdi->b_io list is exhausted.
1057  */
requeue_io(struct inode * inode,struct bdi_writeback * wb)1058 static void requeue_io(struct inode *inode, struct bdi_writeback *wb)
1059 {
1060 	inode_io_list_move_locked(inode, wb, &wb->b_more_io);
1061 }
1062 
inode_sync_complete(struct inode * inode)1063 static void inode_sync_complete(struct inode *inode)
1064 {
1065 	inode->i_state &= ~I_SYNC;
1066 	/* If inode is clean an unused, put it into LRU now... */
1067 	inode_add_lru(inode);
1068 	/* Waiters must see I_SYNC cleared before being woken up */
1069 	smp_mb();
1070 	wake_up_bit(&inode->i_state, __I_SYNC);
1071 }
1072 
inode_dirtied_after(struct inode * inode,unsigned long t)1073 static bool inode_dirtied_after(struct inode *inode, unsigned long t)
1074 {
1075 	bool ret = time_after(inode->dirtied_when, t);
1076 #ifndef CONFIG_64BIT
1077 	/*
1078 	 * For inodes being constantly redirtied, dirtied_when can get stuck.
1079 	 * It _appears_ to be in the future, but is actually in distant past.
1080 	 * This test is necessary to prevent such wrapped-around relative times
1081 	 * from permanently stopping the whole bdi writeback.
1082 	 */
1083 	ret = ret && time_before_eq(inode->dirtied_when, jiffies);
1084 #endif
1085 	return ret;
1086 }
1087 
1088 #define EXPIRE_DIRTY_ATIME 0x0001
1089 
1090 /*
1091  * Move expired (dirtied before work->older_than_this) dirty inodes from
1092  * @delaying_queue to @dispatch_queue.
1093  */
move_expired_inodes(struct list_head * delaying_queue,struct list_head * dispatch_queue,int flags,struct wb_writeback_work * work)1094 static int move_expired_inodes(struct list_head *delaying_queue,
1095 			       struct list_head *dispatch_queue,
1096 			       int flags,
1097 			       struct wb_writeback_work *work)
1098 {
1099 	unsigned long *older_than_this = NULL;
1100 	unsigned long expire_time;
1101 	LIST_HEAD(tmp);
1102 	struct list_head *pos, *node;
1103 	struct super_block *sb = NULL;
1104 	struct inode *inode;
1105 	int do_sb_sort = 0;
1106 	int moved = 0;
1107 
1108 	if ((flags & EXPIRE_DIRTY_ATIME) == 0)
1109 		older_than_this = work->older_than_this;
1110 	else if (!work->for_sync) {
1111 		expire_time = jiffies - (dirtytime_expire_interval * HZ);
1112 		older_than_this = &expire_time;
1113 	}
1114 	while (!list_empty(delaying_queue)) {
1115 		inode = wb_inode(delaying_queue->prev);
1116 		if (older_than_this &&
1117 		    inode_dirtied_after(inode, *older_than_this))
1118 			break;
1119 		list_move(&inode->i_io_list, &tmp);
1120 		moved++;
1121 		if (flags & EXPIRE_DIRTY_ATIME)
1122 			set_bit(__I_DIRTY_TIME_EXPIRED, &inode->i_state);
1123 		if (sb_is_blkdev_sb(inode->i_sb))
1124 			continue;
1125 		if (sb && sb != inode->i_sb)
1126 			do_sb_sort = 1;
1127 		sb = inode->i_sb;
1128 	}
1129 
1130 	/* just one sb in list, splice to dispatch_queue and we're done */
1131 	if (!do_sb_sort) {
1132 		list_splice(&tmp, dispatch_queue);
1133 		goto out;
1134 	}
1135 
1136 	/* Move inodes from one superblock together */
1137 	while (!list_empty(&tmp)) {
1138 		sb = wb_inode(tmp.prev)->i_sb;
1139 		list_for_each_prev_safe(pos, node, &tmp) {
1140 			inode = wb_inode(pos);
1141 			if (inode->i_sb == sb)
1142 				list_move(&inode->i_io_list, dispatch_queue);
1143 		}
1144 	}
1145 out:
1146 	return moved;
1147 }
1148 
1149 /*
1150  * Queue all expired dirty inodes for io, eldest first.
1151  * Before
1152  *         newly dirtied     b_dirty    b_io    b_more_io
1153  *         =============>    gf         edc     BA
1154  * After
1155  *         newly dirtied     b_dirty    b_io    b_more_io
1156  *         =============>    g          fBAedc
1157  *                                           |
1158  *                                           +--> dequeue for IO
1159  */
queue_io(struct bdi_writeback * wb,struct wb_writeback_work * work)1160 static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work)
1161 {
1162 	int moved;
1163 
1164 	assert_spin_locked(&wb->list_lock);
1165 	list_splice_init(&wb->b_more_io, &wb->b_io);
1166 	moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, 0, work);
1167 	moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io,
1168 				     EXPIRE_DIRTY_ATIME, work);
1169 	if (moved)
1170 		wb_io_lists_populated(wb);
1171 	trace_writeback_queue_io(wb, work, moved);
1172 }
1173 
write_inode(struct inode * inode,struct writeback_control * wbc)1174 static int write_inode(struct inode *inode, struct writeback_control *wbc)
1175 {
1176 	int ret;
1177 
1178 	if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {
1179 		trace_writeback_write_inode_start(inode, wbc);
1180 		ret = inode->i_sb->s_op->write_inode(inode, wbc);
1181 		trace_writeback_write_inode(inode, wbc);
1182 		return ret;
1183 	}
1184 	return 0;
1185 }
1186 
1187 /*
1188  * Wait for writeback on an inode to complete. Called with i_lock held.
1189  * Caller must make sure inode cannot go away when we drop i_lock.
1190  */
__inode_wait_for_writeback(struct inode * inode)1191 static void __inode_wait_for_writeback(struct inode *inode)
1192 	__releases(inode->i_lock)
1193 	__acquires(inode->i_lock)
1194 {
1195 	DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
1196 	wait_queue_head_t *wqh;
1197 
1198 	wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
1199 	while (inode->i_state & I_SYNC) {
1200 		spin_unlock(&inode->i_lock);
1201 		__wait_on_bit(wqh, &wq, bit_wait,
1202 			      TASK_UNINTERRUPTIBLE);
1203 		spin_lock(&inode->i_lock);
1204 	}
1205 }
1206 
1207 /*
1208  * Wait for writeback on an inode to complete. Caller must have inode pinned.
1209  */
inode_wait_for_writeback(struct inode * inode)1210 void inode_wait_for_writeback(struct inode *inode)
1211 {
1212 	spin_lock(&inode->i_lock);
1213 	__inode_wait_for_writeback(inode);
1214 	spin_unlock(&inode->i_lock);
1215 }
1216 
1217 /*
1218  * Sleep until I_SYNC is cleared. This function must be called with i_lock
1219  * held and drops it. It is aimed for callers not holding any inode reference
1220  * so once i_lock is dropped, inode can go away.
1221  */
inode_sleep_on_writeback(struct inode * inode)1222 static void inode_sleep_on_writeback(struct inode *inode)
1223 	__releases(inode->i_lock)
1224 {
1225 	DEFINE_WAIT(wait);
1226 	wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
1227 	int sleep;
1228 
1229 	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
1230 	sleep = inode->i_state & I_SYNC;
1231 	spin_unlock(&inode->i_lock);
1232 	if (sleep)
1233 		schedule();
1234 	finish_wait(wqh, &wait);
1235 }
1236 
1237 /*
1238  * Find proper writeback list for the inode depending on its current state and
1239  * possibly also change of its state while we were doing writeback.  Here we
1240  * handle things such as livelock prevention or fairness of writeback among
1241  * inodes. This function can be called only by flusher thread - noone else
1242  * processes all inodes in writeback lists and requeueing inodes behind flusher
1243  * thread's back can have unexpected consequences.
1244  */
requeue_inode(struct inode * inode,struct bdi_writeback * wb,struct writeback_control * wbc)1245 static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,
1246 			  struct writeback_control *wbc)
1247 {
1248 	if (inode->i_state & I_FREEING)
1249 		return;
1250 
1251 	/*
1252 	 * Sync livelock prevention. Each inode is tagged and synced in one
1253 	 * shot. If still dirty, it will be redirty_tail()'ed below.  Update
1254 	 * the dirty time to prevent enqueue and sync it again.
1255 	 */
1256 	if ((inode->i_state & I_DIRTY) &&
1257 	    (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages))
1258 		inode->dirtied_when = jiffies;
1259 
1260 	if (wbc->pages_skipped) {
1261 		/*
1262 		 * writeback is not making progress due to locked
1263 		 * buffers. Skip this inode for now.
1264 		 */
1265 		redirty_tail(inode, wb);
1266 		return;
1267 	}
1268 
1269 	if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {
1270 		/*
1271 		 * We didn't write back all the pages.  nfs_writepages()
1272 		 * sometimes bales out without doing anything.
1273 		 */
1274 		if (wbc->nr_to_write <= 0) {
1275 			/* Slice used up. Queue for next turn. */
1276 			requeue_io(inode, wb);
1277 		} else {
1278 			/*
1279 			 * Writeback blocked by something other than
1280 			 * congestion. Delay the inode for some time to
1281 			 * avoid spinning on the CPU (100% iowait)
1282 			 * retrying writeback of the dirty page/inode
1283 			 * that cannot be performed immediately.
1284 			 */
1285 			redirty_tail(inode, wb);
1286 		}
1287 	} else if (inode->i_state & I_DIRTY) {
1288 		/*
1289 		 * Filesystems can dirty the inode during writeback operations,
1290 		 * such as delayed allocation during submission or metadata
1291 		 * updates after data IO completion.
1292 		 */
1293 		redirty_tail(inode, wb);
1294 	} else if (inode->i_state & I_DIRTY_TIME) {
1295 		inode->dirtied_when = jiffies;
1296 		inode_io_list_move_locked(inode, wb, &wb->b_dirty_time);
1297 	} else {
1298 		/* The inode is clean. Remove from writeback lists. */
1299 		inode_io_list_del_locked(inode, wb);
1300 	}
1301 }
1302 
1303 /*
1304  * Write out an inode and its dirty pages. Do not update the writeback list
1305  * linkage. That is left to the caller. The caller is also responsible for
1306  * setting I_SYNC flag and calling inode_sync_complete() to clear it.
1307  */
1308 static int
__writeback_single_inode(struct inode * inode,struct writeback_control * wbc)1309 __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
1310 {
1311 	struct address_space *mapping = inode->i_mapping;
1312 	long nr_to_write = wbc->nr_to_write;
1313 	unsigned dirty;
1314 	int ret;
1315 
1316 	WARN_ON(!(inode->i_state & I_SYNC));
1317 
1318 	trace_writeback_single_inode_start(inode, wbc, nr_to_write);
1319 
1320 	ret = do_writepages(mapping, wbc);
1321 
1322 	/*
1323 	 * Make sure to wait on the data before writing out the metadata.
1324 	 * This is important for filesystems that modify metadata on data
1325 	 * I/O completion. We don't do it for sync(2) writeback because it has a
1326 	 * separate, external IO completion path and ->sync_fs for guaranteeing
1327 	 * inode metadata is written back correctly.
1328 	 */
1329 	if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) {
1330 		int err = filemap_fdatawait(mapping);
1331 		if (ret == 0)
1332 			ret = err;
1333 	}
1334 
1335 	/*
1336 	 * Some filesystems may redirty the inode during the writeback
1337 	 * due to delalloc, clear dirty metadata flags right before
1338 	 * write_inode()
1339 	 */
1340 	spin_lock(&inode->i_lock);
1341 
1342 	dirty = inode->i_state & I_DIRTY;
1343 	if (inode->i_state & I_DIRTY_TIME) {
1344 		if ((dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) ||
1345 		    wbc->sync_mode == WB_SYNC_ALL ||
1346 		    unlikely(inode->i_state & I_DIRTY_TIME_EXPIRED) ||
1347 		    unlikely(time_after(jiffies,
1348 					(inode->dirtied_time_when +
1349 					 dirtytime_expire_interval * HZ)))) {
1350 			dirty |= I_DIRTY_TIME | I_DIRTY_TIME_EXPIRED;
1351 			trace_writeback_lazytime(inode);
1352 		}
1353 	} else
1354 		inode->i_state &= ~I_DIRTY_TIME_EXPIRED;
1355 	inode->i_state &= ~dirty;
1356 
1357 	/*
1358 	 * Paired with smp_mb() in __mark_inode_dirty().  This allows
1359 	 * __mark_inode_dirty() to test i_state without grabbing i_lock -
1360 	 * either they see the I_DIRTY bits cleared or we see the dirtied
1361 	 * inode.
1362 	 *
1363 	 * I_DIRTY_PAGES is always cleared together above even if @mapping
1364 	 * still has dirty pages.  The flag is reinstated after smp_mb() if
1365 	 * necessary.  This guarantees that either __mark_inode_dirty()
1366 	 * sees clear I_DIRTY_PAGES or we see PAGECACHE_TAG_DIRTY.
1367 	 */
1368 	smp_mb();
1369 
1370 	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
1371 		inode->i_state |= I_DIRTY_PAGES;
1372 
1373 	spin_unlock(&inode->i_lock);
1374 
1375 	if (dirty & I_DIRTY_TIME)
1376 		mark_inode_dirty_sync(inode);
1377 	/* Don't write the inode if only I_DIRTY_PAGES was set */
1378 	if (dirty & ~I_DIRTY_PAGES) {
1379 		int err = write_inode(inode, wbc);
1380 		if (ret == 0)
1381 			ret = err;
1382 	}
1383 	trace_writeback_single_inode(inode, wbc, nr_to_write);
1384 	return ret;
1385 }
1386 
1387 /*
1388  * Write out an inode's dirty pages. Either the caller has an active reference
1389  * on the inode or the inode has I_WILL_FREE set.
1390  *
1391  * This function is designed to be called for writing back one inode which
1392  * we go e.g. from filesystem. Flusher thread uses __writeback_single_inode()
1393  * and does more profound writeback list handling in writeback_sb_inodes().
1394  */
writeback_single_inode(struct inode * inode,struct writeback_control * wbc)1395 static int writeback_single_inode(struct inode *inode,
1396 				  struct writeback_control *wbc)
1397 {
1398 	struct bdi_writeback *wb;
1399 	int ret = 0;
1400 
1401 	spin_lock(&inode->i_lock);
1402 	if (!atomic_read(&inode->i_count))
1403 		WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
1404 	else
1405 		WARN_ON(inode->i_state & I_WILL_FREE);
1406 
1407 	if (inode->i_state & I_SYNC) {
1408 		if (wbc->sync_mode != WB_SYNC_ALL)
1409 			goto out;
1410 		/*
1411 		 * It's a data-integrity sync. We must wait. Since callers hold
1412 		 * inode reference or inode has I_WILL_FREE set, it cannot go
1413 		 * away under us.
1414 		 */
1415 		__inode_wait_for_writeback(inode);
1416 	}
1417 	WARN_ON(inode->i_state & I_SYNC);
1418 	/*
1419 	 * Skip inode if it is clean and we have no outstanding writeback in
1420 	 * WB_SYNC_ALL mode. We don't want to mess with writeback lists in this
1421 	 * function since flusher thread may be doing for example sync in
1422 	 * parallel and if we move the inode, it could get skipped. So here we
1423 	 * make sure inode is on some writeback list and leave it there unless
1424 	 * we have completely cleaned the inode.
1425 	 */
1426 	if (!(inode->i_state & I_DIRTY_ALL) &&
1427 	    (wbc->sync_mode != WB_SYNC_ALL ||
1428 	     !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)))
1429 		goto out;
1430 	inode->i_state |= I_SYNC;
1431 	wbc_attach_and_unlock_inode(wbc, inode);
1432 
1433 	ret = __writeback_single_inode(inode, wbc);
1434 
1435 	wbc_detach_inode(wbc);
1436 
1437 	wb = inode_to_wb_and_lock_list(inode);
1438 	spin_lock(&inode->i_lock);
1439 	/*
1440 	 * If inode is clean, remove it from writeback lists. Otherwise don't
1441 	 * touch it. See comment above for explanation.
1442 	 */
1443 	if (!(inode->i_state & I_DIRTY_ALL))
1444 		inode_io_list_del_locked(inode, wb);
1445 	spin_unlock(&wb->list_lock);
1446 	inode_sync_complete(inode);
1447 out:
1448 	spin_unlock(&inode->i_lock);
1449 	return ret;
1450 }
1451 
writeback_chunk_size(struct bdi_writeback * wb,struct wb_writeback_work * work)1452 static long writeback_chunk_size(struct bdi_writeback *wb,
1453 				 struct wb_writeback_work *work)
1454 {
1455 	long pages;
1456 
1457 	/*
1458 	 * WB_SYNC_ALL mode does livelock avoidance by syncing dirty
1459 	 * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX
1460 	 * here avoids calling into writeback_inodes_wb() more than once.
1461 	 *
1462 	 * The intended call sequence for WB_SYNC_ALL writeback is:
1463 	 *
1464 	 *      wb_writeback()
1465 	 *          writeback_sb_inodes()       <== called only once
1466 	 *              write_cache_pages()     <== called once for each inode
1467 	 *                   (quickly) tag currently dirty pages
1468 	 *                   (maybe slowly) sync all tagged pages
1469 	 */
1470 	if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages)
1471 		pages = LONG_MAX;
1472 	else {
1473 		pages = min(wb->avg_write_bandwidth / 2,
1474 			    global_wb_domain.dirty_limit / DIRTY_SCOPE);
1475 		pages = min(pages, work->nr_pages);
1476 		pages = round_down(pages + MIN_WRITEBACK_PAGES,
1477 				   MIN_WRITEBACK_PAGES);
1478 	}
1479 
1480 	return pages;
1481 }
1482 
1483 /*
1484  * Write a portion of b_io inodes which belong to @sb.
1485  *
1486  * Return the number of pages and/or inodes written.
1487  *
1488  * NOTE! This is called with wb->list_lock held, and will
1489  * unlock and relock that for each inode it ends up doing
1490  * IO for.
1491  */
writeback_sb_inodes(struct super_block * sb,struct bdi_writeback * wb,struct wb_writeback_work * work)1492 static long writeback_sb_inodes(struct super_block *sb,
1493 				struct bdi_writeback *wb,
1494 				struct wb_writeback_work *work)
1495 {
1496 	struct writeback_control wbc = {
1497 		.sync_mode		= work->sync_mode,
1498 		.tagged_writepages	= work->tagged_writepages,
1499 		.for_kupdate		= work->for_kupdate,
1500 		.for_background		= work->for_background,
1501 		.for_sync		= work->for_sync,
1502 		.range_cyclic		= work->range_cyclic,
1503 		.range_start		= 0,
1504 		.range_end		= LLONG_MAX,
1505 	};
1506 	unsigned long start_time = jiffies;
1507 	long write_chunk;
1508 	long wrote = 0;  /* count both pages and inodes */
1509 
1510 	while (!list_empty(&wb->b_io)) {
1511 		struct inode *inode = wb_inode(wb->b_io.prev);
1512 		struct bdi_writeback *tmp_wb;
1513 
1514 		if (inode->i_sb != sb) {
1515 			if (work->sb) {
1516 				/*
1517 				 * We only want to write back data for this
1518 				 * superblock, move all inodes not belonging
1519 				 * to it back onto the dirty list.
1520 				 */
1521 				redirty_tail(inode, wb);
1522 				continue;
1523 			}
1524 
1525 			/*
1526 			 * The inode belongs to a different superblock.
1527 			 * Bounce back to the caller to unpin this and
1528 			 * pin the next superblock.
1529 			 */
1530 			break;
1531 		}
1532 
1533 		/*
1534 		 * Don't bother with new inodes or inodes being freed, first
1535 		 * kind does not need periodic writeout yet, and for the latter
1536 		 * kind writeout is handled by the freer.
1537 		 */
1538 		spin_lock(&inode->i_lock);
1539 		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
1540 			spin_unlock(&inode->i_lock);
1541 			redirty_tail(inode, wb);
1542 			continue;
1543 		}
1544 		if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {
1545 			/*
1546 			 * If this inode is locked for writeback and we are not
1547 			 * doing writeback-for-data-integrity, move it to
1548 			 * b_more_io so that writeback can proceed with the
1549 			 * other inodes on s_io.
1550 			 *
1551 			 * We'll have another go at writing back this inode
1552 			 * when we completed a full scan of b_io.
1553 			 */
1554 			spin_unlock(&inode->i_lock);
1555 			requeue_io(inode, wb);
1556 			trace_writeback_sb_inodes_requeue(inode);
1557 			continue;
1558 		}
1559 		spin_unlock(&wb->list_lock);
1560 
1561 		/*
1562 		 * We already requeued the inode if it had I_SYNC set and we
1563 		 * are doing WB_SYNC_NONE writeback. So this catches only the
1564 		 * WB_SYNC_ALL case.
1565 		 */
1566 		if (inode->i_state & I_SYNC) {
1567 			/* Wait for I_SYNC. This function drops i_lock... */
1568 			inode_sleep_on_writeback(inode);
1569 			/* Inode may be gone, start again */
1570 			spin_lock(&wb->list_lock);
1571 			continue;
1572 		}
1573 		inode->i_state |= I_SYNC;
1574 		wbc_attach_and_unlock_inode(&wbc, inode);
1575 
1576 		write_chunk = writeback_chunk_size(wb, work);
1577 		wbc.nr_to_write = write_chunk;
1578 		wbc.pages_skipped = 0;
1579 
1580 		/*
1581 		 * We use I_SYNC to pin the inode in memory. While it is set
1582 		 * evict_inode() will wait so the inode cannot be freed.
1583 		 */
1584 		__writeback_single_inode(inode, &wbc);
1585 
1586 		wbc_detach_inode(&wbc);
1587 		work->nr_pages -= write_chunk - wbc.nr_to_write;
1588 		wrote += write_chunk - wbc.nr_to_write;
1589 
1590 		if (need_resched()) {
1591 			/*
1592 			 * We're trying to balance between building up a nice
1593 			 * long list of IOs to improve our merge rate, and
1594 			 * getting those IOs out quickly for anyone throttling
1595 			 * in balance_dirty_pages().  cond_resched() doesn't
1596 			 * unplug, so get our IOs out the door before we
1597 			 * give up the CPU.
1598 			 */
1599 			blk_flush_plug(current);
1600 			cond_resched();
1601 		}
1602 
1603 		/*
1604 		 * Requeue @inode if still dirty.  Be careful as @inode may
1605 		 * have been switched to another wb in the meantime.
1606 		 */
1607 		tmp_wb = inode_to_wb_and_lock_list(inode);
1608 		spin_lock(&inode->i_lock);
1609 		if (!(inode->i_state & I_DIRTY_ALL))
1610 			wrote++;
1611 		requeue_inode(inode, tmp_wb, &wbc);
1612 		inode_sync_complete(inode);
1613 		spin_unlock(&inode->i_lock);
1614 
1615 		if (unlikely(tmp_wb != wb)) {
1616 			spin_unlock(&tmp_wb->list_lock);
1617 			spin_lock(&wb->list_lock);
1618 		}
1619 
1620 		/*
1621 		 * bail out to wb_writeback() often enough to check
1622 		 * background threshold and other termination conditions.
1623 		 */
1624 		if (wrote) {
1625 			if (time_is_before_jiffies(start_time + HZ / 10UL))
1626 				break;
1627 			if (work->nr_pages <= 0)
1628 				break;
1629 		}
1630 	}
1631 	return wrote;
1632 }
1633 
__writeback_inodes_wb(struct bdi_writeback * wb,struct wb_writeback_work * work)1634 static long __writeback_inodes_wb(struct bdi_writeback *wb,
1635 				  struct wb_writeback_work *work)
1636 {
1637 	unsigned long start_time = jiffies;
1638 	long wrote = 0;
1639 
1640 	while (!list_empty(&wb->b_io)) {
1641 		struct inode *inode = wb_inode(wb->b_io.prev);
1642 		struct super_block *sb = inode->i_sb;
1643 
1644 		if (!trylock_super(sb)) {
1645 			/*
1646 			 * trylock_super() may fail consistently due to
1647 			 * s_umount being grabbed by someone else. Don't use
1648 			 * requeue_io() to avoid busy retrying the inode/sb.
1649 			 */
1650 			redirty_tail(inode, wb);
1651 			continue;
1652 		}
1653 		wrote += writeback_sb_inodes(sb, wb, work);
1654 		up_read(&sb->s_umount);
1655 
1656 		/* refer to the same tests at the end of writeback_sb_inodes */
1657 		if (wrote) {
1658 			if (time_is_before_jiffies(start_time + HZ / 10UL))
1659 				break;
1660 			if (work->nr_pages <= 0)
1661 				break;
1662 		}
1663 	}
1664 	/* Leave any unwritten inodes on b_io */
1665 	return wrote;
1666 }
1667 
writeback_inodes_wb(struct bdi_writeback * wb,long nr_pages,enum wb_reason reason)1668 static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,
1669 				enum wb_reason reason)
1670 {
1671 	struct wb_writeback_work work = {
1672 		.nr_pages	= nr_pages,
1673 		.sync_mode	= WB_SYNC_NONE,
1674 		.range_cyclic	= 1,
1675 		.reason		= reason,
1676 	};
1677 	struct blk_plug plug;
1678 
1679 	blk_start_plug(&plug);
1680 	spin_lock(&wb->list_lock);
1681 	if (list_empty(&wb->b_io))
1682 		queue_io(wb, &work);
1683 	__writeback_inodes_wb(wb, &work);
1684 	spin_unlock(&wb->list_lock);
1685 	blk_finish_plug(&plug);
1686 
1687 	return nr_pages - work.nr_pages;
1688 }
1689 
1690 /*
1691  * Explicit flushing or periodic writeback of "old" data.
1692  *
1693  * Define "old": the first time one of an inode's pages is dirtied, we mark the
1694  * dirtying-time in the inode's address_space.  So this periodic writeback code
1695  * just walks the superblock inode list, writing back any inodes which are
1696  * older than a specific point in time.
1697  *
1698  * Try to run once per dirty_writeback_interval.  But if a writeback event
1699  * takes longer than a dirty_writeback_interval interval, then leave a
1700  * one-second gap.
1701  *
1702  * older_than_this takes precedence over nr_to_write.  So we'll only write back
1703  * all dirty pages if they are all attached to "old" mappings.
1704  */
wb_writeback(struct bdi_writeback * wb,struct wb_writeback_work * work)1705 static long wb_writeback(struct bdi_writeback *wb,
1706 			 struct wb_writeback_work *work)
1707 {
1708 	unsigned long wb_start = jiffies;
1709 	long nr_pages = work->nr_pages;
1710 	unsigned long oldest_jif;
1711 	struct inode *inode;
1712 	long progress;
1713 	struct blk_plug plug;
1714 
1715 	oldest_jif = jiffies;
1716 	work->older_than_this = &oldest_jif;
1717 
1718 	blk_start_plug(&plug);
1719 	spin_lock(&wb->list_lock);
1720 	for (;;) {
1721 		/*
1722 		 * Stop writeback when nr_pages has been consumed
1723 		 */
1724 		if (work->nr_pages <= 0)
1725 			break;
1726 
1727 		/*
1728 		 * Background writeout and kupdate-style writeback may
1729 		 * run forever. Stop them if there is other work to do
1730 		 * so that e.g. sync can proceed. They'll be restarted
1731 		 * after the other works are all done.
1732 		 */
1733 		if ((work->for_background || work->for_kupdate) &&
1734 		    !list_empty(&wb->work_list))
1735 			break;
1736 
1737 		/*
1738 		 * For background writeout, stop when we are below the
1739 		 * background dirty threshold
1740 		 */
1741 		if (work->for_background && !wb_over_bg_thresh(wb))
1742 			break;
1743 
1744 		/*
1745 		 * Kupdate and background works are special and we want to
1746 		 * include all inodes that need writing. Livelock avoidance is
1747 		 * handled by these works yielding to any other work so we are
1748 		 * safe.
1749 		 */
1750 		if (work->for_kupdate) {
1751 			oldest_jif = jiffies -
1752 				msecs_to_jiffies(dirty_expire_interval * 10);
1753 		} else if (work->for_background)
1754 			oldest_jif = jiffies;
1755 
1756 		trace_writeback_start(wb, work);
1757 		if (list_empty(&wb->b_io))
1758 			queue_io(wb, work);
1759 		if (work->sb)
1760 			progress = writeback_sb_inodes(work->sb, wb, work);
1761 		else
1762 			progress = __writeback_inodes_wb(wb, work);
1763 		trace_writeback_written(wb, work);
1764 
1765 		wb_update_bandwidth(wb, wb_start);
1766 
1767 		/*
1768 		 * Did we write something? Try for more
1769 		 *
1770 		 * Dirty inodes are moved to b_io for writeback in batches.
1771 		 * The completion of the current batch does not necessarily
1772 		 * mean the overall work is done. So we keep looping as long
1773 		 * as made some progress on cleaning pages or inodes.
1774 		 */
1775 		if (progress)
1776 			continue;
1777 		/*
1778 		 * No more inodes for IO, bail
1779 		 */
1780 		if (list_empty(&wb->b_more_io))
1781 			break;
1782 		/*
1783 		 * Nothing written. Wait for some inode to
1784 		 * become available for writeback. Otherwise
1785 		 * we'll just busyloop.
1786 		 */
1787 		if (!list_empty(&wb->b_more_io))  {
1788 			trace_writeback_wait(wb, work);
1789 			inode = wb_inode(wb->b_more_io.prev);
1790 			spin_lock(&inode->i_lock);
1791 			spin_unlock(&wb->list_lock);
1792 			/* This function drops i_lock... */
1793 			inode_sleep_on_writeback(inode);
1794 			spin_lock(&wb->list_lock);
1795 		}
1796 	}
1797 	spin_unlock(&wb->list_lock);
1798 	blk_finish_plug(&plug);
1799 
1800 	return nr_pages - work->nr_pages;
1801 }
1802 
1803 /*
1804  * Return the next wb_writeback_work struct that hasn't been processed yet.
1805  */
get_next_work_item(struct bdi_writeback * wb)1806 static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb)
1807 {
1808 	struct wb_writeback_work *work = NULL;
1809 
1810 	spin_lock_bh(&wb->work_lock);
1811 	if (!list_empty(&wb->work_list)) {
1812 		work = list_entry(wb->work_list.next,
1813 				  struct wb_writeback_work, list);
1814 		list_del_init(&work->list);
1815 	}
1816 	spin_unlock_bh(&wb->work_lock);
1817 	return work;
1818 }
1819 
1820 /*
1821  * Add in the number of potentially dirty inodes, because each inode
1822  * write can dirty pagecache in the underlying blockdev.
1823  */
get_nr_dirty_pages(void)1824 static unsigned long get_nr_dirty_pages(void)
1825 {
1826 	return global_node_page_state(NR_FILE_DIRTY) +
1827 		global_node_page_state(NR_UNSTABLE_NFS) +
1828 		get_nr_dirty_inodes();
1829 }
1830 
wb_check_background_flush(struct bdi_writeback * wb)1831 static long wb_check_background_flush(struct bdi_writeback *wb)
1832 {
1833 	if (wb_over_bg_thresh(wb)) {
1834 
1835 		struct wb_writeback_work work = {
1836 			.nr_pages	= LONG_MAX,
1837 			.sync_mode	= WB_SYNC_NONE,
1838 			.for_background	= 1,
1839 			.range_cyclic	= 1,
1840 			.reason		= WB_REASON_BACKGROUND,
1841 		};
1842 
1843 		return wb_writeback(wb, &work);
1844 	}
1845 
1846 	return 0;
1847 }
1848 
wb_check_old_data_flush(struct bdi_writeback * wb)1849 static long wb_check_old_data_flush(struct bdi_writeback *wb)
1850 {
1851 	unsigned long expired;
1852 	long nr_pages;
1853 
1854 	/*
1855 	 * When set to zero, disable periodic writeback
1856 	 */
1857 	if (!dirty_writeback_interval)
1858 		return 0;
1859 
1860 	expired = wb->last_old_flush +
1861 			msecs_to_jiffies(dirty_writeback_interval * 10);
1862 	if (time_before(jiffies, expired))
1863 		return 0;
1864 
1865 	wb->last_old_flush = jiffies;
1866 	nr_pages = get_nr_dirty_pages();
1867 
1868 	if (nr_pages) {
1869 		struct wb_writeback_work work = {
1870 			.nr_pages	= nr_pages,
1871 			.sync_mode	= WB_SYNC_NONE,
1872 			.for_kupdate	= 1,
1873 			.range_cyclic	= 1,
1874 			.reason		= WB_REASON_PERIODIC,
1875 		};
1876 
1877 		return wb_writeback(wb, &work);
1878 	}
1879 
1880 	return 0;
1881 }
1882 
1883 /*
1884  * Retrieve work items and do the writeback they describe
1885  */
wb_do_writeback(struct bdi_writeback * wb)1886 static long wb_do_writeback(struct bdi_writeback *wb)
1887 {
1888 	struct wb_writeback_work *work;
1889 	long wrote = 0;
1890 
1891 	set_bit(WB_writeback_running, &wb->state);
1892 	while ((work = get_next_work_item(wb)) != NULL) {
1893 		trace_writeback_exec(wb, work);
1894 		wrote += wb_writeback(wb, work);
1895 		finish_writeback_work(wb, work);
1896 	}
1897 
1898 	/*
1899 	 * Check for periodic writeback, kupdated() style
1900 	 */
1901 	wrote += wb_check_old_data_flush(wb);
1902 	wrote += wb_check_background_flush(wb);
1903 	clear_bit(WB_writeback_running, &wb->state);
1904 
1905 	return wrote;
1906 }
1907 
1908 /*
1909  * Handle writeback of dirty data for the device backed by this bdi. Also
1910  * reschedules periodically and does kupdated style flushing.
1911  */
wb_workfn(struct work_struct * work)1912 void wb_workfn(struct work_struct *work)
1913 {
1914 	struct bdi_writeback *wb = container_of(to_delayed_work(work),
1915 						struct bdi_writeback, dwork);
1916 	long pages_written;
1917 
1918 	set_worker_desc("flush-%s", dev_name(wb->bdi->dev));
1919 	current->flags |= PF_SWAPWRITE;
1920 
1921 	if (likely(!current_is_workqueue_rescuer() ||
1922 		   !test_bit(WB_registered, &wb->state))) {
1923 		/*
1924 		 * The normal path.  Keep writing back @wb until its
1925 		 * work_list is empty.  Note that this path is also taken
1926 		 * if @wb is shutting down even when we're running off the
1927 		 * rescuer as work_list needs to be drained.
1928 		 */
1929 		do {
1930 			pages_written = wb_do_writeback(wb);
1931 			trace_writeback_pages_written(pages_written);
1932 		} while (!list_empty(&wb->work_list));
1933 	} else {
1934 		/*
1935 		 * bdi_wq can't get enough workers and we're running off
1936 		 * the emergency worker.  Don't hog it.  Hopefully, 1024 is
1937 		 * enough for efficient IO.
1938 		 */
1939 		pages_written = writeback_inodes_wb(wb, 1024,
1940 						    WB_REASON_FORKER_THREAD);
1941 		trace_writeback_pages_written(pages_written);
1942 	}
1943 
1944 	if (!list_empty(&wb->work_list))
1945 		mod_delayed_work(bdi_wq, &wb->dwork, 0);
1946 	else if (wb_has_dirty_io(wb) && dirty_writeback_interval)
1947 		wb_wakeup_delayed(wb);
1948 
1949 	current->flags &= ~PF_SWAPWRITE;
1950 }
1951 
1952 /*
1953  * Start writeback of `nr_pages' pages.  If `nr_pages' is zero, write back
1954  * the whole world.
1955  */
wakeup_flusher_threads(long nr_pages,enum wb_reason reason)1956 void wakeup_flusher_threads(long nr_pages, enum wb_reason reason)
1957 {
1958 	struct backing_dev_info *bdi;
1959 
1960 	/*
1961 	 * If we are expecting writeback progress we must submit plugged IO.
1962 	 */
1963 	if (blk_needs_flush_plug(current))
1964 		blk_schedule_flush_plug(current);
1965 
1966 	if (!nr_pages)
1967 		nr_pages = get_nr_dirty_pages();
1968 
1969 	rcu_read_lock();
1970 	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
1971 		struct bdi_writeback *wb;
1972 
1973 		if (!bdi_has_dirty_io(bdi))
1974 			continue;
1975 
1976 		list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)
1977 			wb_start_writeback(wb, wb_split_bdi_pages(wb, nr_pages),
1978 					   false, reason);
1979 	}
1980 	rcu_read_unlock();
1981 }
1982 
1983 /*
1984  * Wake up bdi's periodically to make sure dirtytime inodes gets
1985  * written back periodically.  We deliberately do *not* check the
1986  * b_dirtytime list in wb_has_dirty_io(), since this would cause the
1987  * kernel to be constantly waking up once there are any dirtytime
1988  * inodes on the system.  So instead we define a separate delayed work
1989  * function which gets called much more rarely.  (By default, only
1990  * once every 12 hours.)
1991  *
1992  * If there is any other write activity going on in the file system,
1993  * this function won't be necessary.  But if the only thing that has
1994  * happened on the file system is a dirtytime inode caused by an atime
1995  * update, we need this infrastructure below to make sure that inode
1996  * eventually gets pushed out to disk.
1997  */
1998 static void wakeup_dirtytime_writeback(struct work_struct *w);
1999 static DECLARE_DELAYED_WORK(dirtytime_work, wakeup_dirtytime_writeback);
2000 
wakeup_dirtytime_writeback(struct work_struct * w)2001 static void wakeup_dirtytime_writeback(struct work_struct *w)
2002 {
2003 	struct backing_dev_info *bdi;
2004 
2005 	rcu_read_lock();
2006 	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
2007 		struct bdi_writeback *wb;
2008 
2009 		list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)
2010 			if (!list_empty(&wb->b_dirty_time))
2011 				wb_wakeup(wb);
2012 	}
2013 	rcu_read_unlock();
2014 	schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
2015 }
2016 
start_dirtytime_writeback(void)2017 static int __init start_dirtytime_writeback(void)
2018 {
2019 	schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
2020 	return 0;
2021 }
2022 __initcall(start_dirtytime_writeback);
2023 
dirtytime_interval_handler(struct ctl_table * table,int write,void __user * buffer,size_t * lenp,loff_t * ppos)2024 int dirtytime_interval_handler(struct ctl_table *table, int write,
2025 			       void __user *buffer, size_t *lenp, loff_t *ppos)
2026 {
2027 	int ret;
2028 
2029 	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2030 	if (ret == 0 && write)
2031 		mod_delayed_work(system_wq, &dirtytime_work, 0);
2032 	return ret;
2033 }
2034 
block_dump___mark_inode_dirty(struct inode * inode)2035 static noinline void block_dump___mark_inode_dirty(struct inode *inode)
2036 {
2037 	if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
2038 		struct dentry *dentry;
2039 		const char *name = "?";
2040 
2041 		dentry = d_find_alias(inode);
2042 		if (dentry) {
2043 			spin_lock(&dentry->d_lock);
2044 			name = (const char *) dentry->d_name.name;
2045 		}
2046 		printk(KERN_DEBUG
2047 		       "%s(%d): dirtied inode %lu (%s) on %s\n",
2048 		       current->comm, task_pid_nr(current), inode->i_ino,
2049 		       name, inode->i_sb->s_id);
2050 		if (dentry) {
2051 			spin_unlock(&dentry->d_lock);
2052 			dput(dentry);
2053 		}
2054 	}
2055 }
2056 
2057 /**
2058  *	__mark_inode_dirty -	internal function
2059  *	@inode: inode to mark
2060  *	@flags: what kind of dirty (i.e. I_DIRTY_SYNC)
2061  *	Mark an inode as dirty. Callers should use mark_inode_dirty or
2062  *  	mark_inode_dirty_sync.
2063  *
2064  * Put the inode on the super block's dirty list.
2065  *
2066  * CAREFUL! We mark it dirty unconditionally, but move it onto the
2067  * dirty list only if it is hashed or if it refers to a blockdev.
2068  * If it was not hashed, it will never be added to the dirty list
2069  * even if it is later hashed, as it will have been marked dirty already.
2070  *
2071  * In short, make sure you hash any inodes _before_ you start marking
2072  * them dirty.
2073  *
2074  * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
2075  * the block-special inode (/dev/hda1) itself.  And the ->dirtied_when field of
2076  * the kernel-internal blockdev inode represents the dirtying time of the
2077  * blockdev's pages.  This is why for I_DIRTY_PAGES we always use
2078  * page->mapping->host, so the page-dirtying time is recorded in the internal
2079  * blockdev inode.
2080  */
__mark_inode_dirty(struct inode * inode,int flags)2081 void __mark_inode_dirty(struct inode *inode, int flags)
2082 {
2083 #define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC)
2084 	struct super_block *sb = inode->i_sb;
2085 	int dirtytime;
2086 
2087 	trace_writeback_mark_inode_dirty(inode, flags);
2088 
2089 	/*
2090 	 * Don't do this for I_DIRTY_PAGES - that doesn't actually
2091 	 * dirty the inode itself
2092 	 */
2093 	if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC | I_DIRTY_TIME)) {
2094 		trace_writeback_dirty_inode_start(inode, flags);
2095 
2096 		if (sb->s_op->dirty_inode)
2097 			sb->s_op->dirty_inode(inode, flags);
2098 
2099 		trace_writeback_dirty_inode(inode, flags);
2100 	}
2101 	if (flags & I_DIRTY_INODE)
2102 		flags &= ~I_DIRTY_TIME;
2103 	dirtytime = flags & I_DIRTY_TIME;
2104 
2105 	/*
2106 	 * Paired with smp_mb() in __writeback_single_inode() for the
2107 	 * following lockless i_state test.  See there for details.
2108 	 */
2109 	smp_mb();
2110 
2111 	if (((inode->i_state & flags) == flags) ||
2112 	    (dirtytime && (inode->i_state & I_DIRTY_INODE)))
2113 		return;
2114 
2115 	if (unlikely(block_dump > 1))
2116 		block_dump___mark_inode_dirty(inode);
2117 
2118 	spin_lock(&inode->i_lock);
2119 	if (dirtytime && (inode->i_state & I_DIRTY_INODE))
2120 		goto out_unlock_inode;
2121 	if ((inode->i_state & flags) != flags) {
2122 		const int was_dirty = inode->i_state & I_DIRTY;
2123 
2124 		inode_attach_wb(inode, NULL);
2125 
2126 		if (flags & I_DIRTY_INODE)
2127 			inode->i_state &= ~I_DIRTY_TIME;
2128 		inode->i_state |= flags;
2129 
2130 		/*
2131 		 * If the inode is being synced, just update its dirty state.
2132 		 * The unlocker will place the inode on the appropriate
2133 		 * superblock list, based upon its state.
2134 		 */
2135 		if (inode->i_state & I_SYNC)
2136 			goto out_unlock_inode;
2137 
2138 		/*
2139 		 * Only add valid (hashed) inodes to the superblock's
2140 		 * dirty list.  Add blockdev inodes as well.
2141 		 */
2142 		if (!S_ISBLK(inode->i_mode)) {
2143 			if (inode_unhashed(inode))
2144 				goto out_unlock_inode;
2145 		}
2146 		if (inode->i_state & I_FREEING)
2147 			goto out_unlock_inode;
2148 
2149 		/*
2150 		 * If the inode was already on b_dirty/b_io/b_more_io, don't
2151 		 * reposition it (that would break b_dirty time-ordering).
2152 		 */
2153 		if (!was_dirty) {
2154 			struct bdi_writeback *wb;
2155 			struct list_head *dirty_list;
2156 			bool wakeup_bdi = false;
2157 
2158 			wb = locked_inode_to_wb_and_lock_list(inode);
2159 
2160 			WARN(bdi_cap_writeback_dirty(wb->bdi) &&
2161 			     !test_bit(WB_registered, &wb->state),
2162 			     "bdi-%s not registered\n", wb->bdi->name);
2163 
2164 			inode->dirtied_when = jiffies;
2165 			if (dirtytime)
2166 				inode->dirtied_time_when = jiffies;
2167 
2168 			if (inode->i_state & (I_DIRTY_INODE | I_DIRTY_PAGES))
2169 				dirty_list = &wb->b_dirty;
2170 			else
2171 				dirty_list = &wb->b_dirty_time;
2172 
2173 			wakeup_bdi = inode_io_list_move_locked(inode, wb,
2174 							       dirty_list);
2175 
2176 			spin_unlock(&wb->list_lock);
2177 			trace_writeback_dirty_inode_enqueue(inode);
2178 
2179 			/*
2180 			 * If this is the first dirty inode for this bdi,
2181 			 * we have to wake-up the corresponding bdi thread
2182 			 * to make sure background write-back happens
2183 			 * later.
2184 			 */
2185 			if (bdi_cap_writeback_dirty(wb->bdi) && wakeup_bdi)
2186 				wb_wakeup_delayed(wb);
2187 			return;
2188 		}
2189 	}
2190 out_unlock_inode:
2191 	spin_unlock(&inode->i_lock);
2192 
2193 #undef I_DIRTY_INODE
2194 }
2195 EXPORT_SYMBOL(__mark_inode_dirty);
2196 
2197 /*
2198  * The @s_sync_lock is used to serialise concurrent sync operations
2199  * to avoid lock contention problems with concurrent wait_sb_inodes() calls.
2200  * Concurrent callers will block on the s_sync_lock rather than doing contending
2201  * walks. The queueing maintains sync(2) required behaviour as all the IO that
2202  * has been issued up to the time this function is enter is guaranteed to be
2203  * completed by the time we have gained the lock and waited for all IO that is
2204  * in progress regardless of the order callers are granted the lock.
2205  */
wait_sb_inodes(struct super_block * sb)2206 static void wait_sb_inodes(struct super_block *sb)
2207 {
2208 	LIST_HEAD(sync_list);
2209 
2210 	/*
2211 	 * We need to be protected against the filesystem going from
2212 	 * r/o to r/w or vice versa.
2213 	 */
2214 	WARN_ON(!rwsem_is_locked(&sb->s_umount));
2215 
2216 	mutex_lock(&sb->s_sync_lock);
2217 
2218 	/*
2219 	 * Splice the writeback list onto a temporary list to avoid waiting on
2220 	 * inodes that have started writeback after this point.
2221 	 *
2222 	 * Use rcu_read_lock() to keep the inodes around until we have a
2223 	 * reference. s_inode_wblist_lock protects sb->s_inodes_wb as well as
2224 	 * the local list because inodes can be dropped from either by writeback
2225 	 * completion.
2226 	 */
2227 	rcu_read_lock();
2228 	spin_lock_irq(&sb->s_inode_wblist_lock);
2229 	list_splice_init(&sb->s_inodes_wb, &sync_list);
2230 
2231 	/*
2232 	 * Data integrity sync. Must wait for all pages under writeback, because
2233 	 * there may have been pages dirtied before our sync call, but which had
2234 	 * writeout started before we write it out.  In which case, the inode
2235 	 * may not be on the dirty list, but we still have to wait for that
2236 	 * writeout.
2237 	 */
2238 	while (!list_empty(&sync_list)) {
2239 		struct inode *inode = list_first_entry(&sync_list, struct inode,
2240 						       i_wb_list);
2241 		struct address_space *mapping = inode->i_mapping;
2242 
2243 		/*
2244 		 * Move each inode back to the wb list before we drop the lock
2245 		 * to preserve consistency between i_wb_list and the mapping
2246 		 * writeback tag. Writeback completion is responsible to remove
2247 		 * the inode from either list once the writeback tag is cleared.
2248 		 */
2249 		list_move_tail(&inode->i_wb_list, &sb->s_inodes_wb);
2250 
2251 		/*
2252 		 * The mapping can appear untagged while still on-list since we
2253 		 * do not have the mapping lock. Skip it here, wb completion
2254 		 * will remove it.
2255 		 */
2256 		if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
2257 			continue;
2258 
2259 		spin_unlock_irq(&sb->s_inode_wblist_lock);
2260 
2261 		spin_lock(&inode->i_lock);
2262 		if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) {
2263 			spin_unlock(&inode->i_lock);
2264 
2265 			spin_lock_irq(&sb->s_inode_wblist_lock);
2266 			continue;
2267 		}
2268 		__iget(inode);
2269 		spin_unlock(&inode->i_lock);
2270 		rcu_read_unlock();
2271 
2272 		/*
2273 		 * We keep the error status of individual mapping so that
2274 		 * applications can catch the writeback error using fsync(2).
2275 		 * See filemap_fdatawait_keep_errors() for details.
2276 		 */
2277 		filemap_fdatawait_keep_errors(mapping);
2278 
2279 		cond_resched();
2280 
2281 		iput(inode);
2282 
2283 		rcu_read_lock();
2284 		spin_lock_irq(&sb->s_inode_wblist_lock);
2285 	}
2286 	spin_unlock_irq(&sb->s_inode_wblist_lock);
2287 	rcu_read_unlock();
2288 	mutex_unlock(&sb->s_sync_lock);
2289 }
2290 
__writeback_inodes_sb_nr(struct super_block * sb,unsigned long nr,enum wb_reason reason,bool skip_if_busy)2291 static void __writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr,
2292 				     enum wb_reason reason, bool skip_if_busy)
2293 {
2294 	DEFINE_WB_COMPLETION_ONSTACK(done);
2295 	struct wb_writeback_work work = {
2296 		.sb			= sb,
2297 		.sync_mode		= WB_SYNC_NONE,
2298 		.tagged_writepages	= 1,
2299 		.done			= &done,
2300 		.nr_pages		= nr,
2301 		.reason			= reason,
2302 	};
2303 	struct backing_dev_info *bdi = sb->s_bdi;
2304 
2305 	if (!bdi_has_dirty_io(bdi) || bdi == &noop_backing_dev_info)
2306 		return;
2307 	WARN_ON(!rwsem_is_locked(&sb->s_umount));
2308 
2309 	bdi_split_work_to_wbs(sb->s_bdi, &work, skip_if_busy);
2310 	wb_wait_for_completion(bdi, &done);
2311 }
2312 
2313 /**
2314  * writeback_inodes_sb_nr -	writeback dirty inodes from given super_block
2315  * @sb: the superblock
2316  * @nr: the number of pages to write
2317  * @reason: reason why some writeback work initiated
2318  *
2319  * Start writeback on some inodes on this super_block. No guarantees are made
2320  * on how many (if any) will be written, and this function does not wait
2321  * for IO completion of submitted IO.
2322  */
writeback_inodes_sb_nr(struct super_block * sb,unsigned long nr,enum wb_reason reason)2323 void writeback_inodes_sb_nr(struct super_block *sb,
2324 			    unsigned long nr,
2325 			    enum wb_reason reason)
2326 {
2327 	__writeback_inodes_sb_nr(sb, nr, reason, false);
2328 }
2329 EXPORT_SYMBOL(writeback_inodes_sb_nr);
2330 
2331 /**
2332  * writeback_inodes_sb	-	writeback dirty inodes from given super_block
2333  * @sb: the superblock
2334  * @reason: reason why some writeback work was initiated
2335  *
2336  * Start writeback on some inodes on this super_block. No guarantees are made
2337  * on how many (if any) will be written, and this function does not wait
2338  * for IO completion of submitted IO.
2339  */
writeback_inodes_sb(struct super_block * sb,enum wb_reason reason)2340 void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
2341 {
2342 	return writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
2343 }
2344 EXPORT_SYMBOL(writeback_inodes_sb);
2345 
2346 /**
2347  * try_to_writeback_inodes_sb_nr - try to start writeback if none underway
2348  * @sb: the superblock
2349  * @nr: the number of pages to write
2350  * @reason: the reason of writeback
2351  *
2352  * Invoke writeback_inodes_sb_nr if no writeback is currently underway.
2353  * Returns 1 if writeback was started, 0 if not.
2354  */
try_to_writeback_inodes_sb_nr(struct super_block * sb,unsigned long nr,enum wb_reason reason)2355 bool try_to_writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr,
2356 				   enum wb_reason reason)
2357 {
2358 	if (!down_read_trylock(&sb->s_umount))
2359 		return false;
2360 
2361 	__writeback_inodes_sb_nr(sb, nr, reason, true);
2362 	up_read(&sb->s_umount);
2363 	return true;
2364 }
2365 EXPORT_SYMBOL(try_to_writeback_inodes_sb_nr);
2366 
2367 /**
2368  * try_to_writeback_inodes_sb - try to start writeback if none underway
2369  * @sb: the superblock
2370  * @reason: reason why some writeback work was initiated
2371  *
2372  * Implement by try_to_writeback_inodes_sb_nr()
2373  * Returns 1 if writeback was started, 0 if not.
2374  */
try_to_writeback_inodes_sb(struct super_block * sb,enum wb_reason reason)2375 bool try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
2376 {
2377 	return try_to_writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
2378 }
2379 EXPORT_SYMBOL(try_to_writeback_inodes_sb);
2380 
2381 /**
2382  * sync_inodes_sb	-	sync sb inode pages
2383  * @sb: the superblock
2384  *
2385  * This function writes and waits on any dirty inode belonging to this
2386  * super_block.
2387  */
sync_inodes_sb(struct super_block * sb)2388 void sync_inodes_sb(struct super_block *sb)
2389 {
2390 	DEFINE_WB_COMPLETION_ONSTACK(done);
2391 	struct wb_writeback_work work = {
2392 		.sb		= sb,
2393 		.sync_mode	= WB_SYNC_ALL,
2394 		.nr_pages	= LONG_MAX,
2395 		.range_cyclic	= 0,
2396 		.done		= &done,
2397 		.reason		= WB_REASON_SYNC,
2398 		.for_sync	= 1,
2399 	};
2400 	struct backing_dev_info *bdi = sb->s_bdi;
2401 
2402 	/*
2403 	 * Can't skip on !bdi_has_dirty() because we should wait for !dirty
2404 	 * inodes under writeback and I_DIRTY_TIME inodes ignored by
2405 	 * bdi_has_dirty() need to be written out too.
2406 	 */
2407 	if (bdi == &noop_backing_dev_info)
2408 		return;
2409 	WARN_ON(!rwsem_is_locked(&sb->s_umount));
2410 
2411 	bdi_split_work_to_wbs(bdi, &work, false);
2412 	wb_wait_for_completion(bdi, &done);
2413 
2414 	wait_sb_inodes(sb);
2415 }
2416 EXPORT_SYMBOL(sync_inodes_sb);
2417 
2418 /**
2419  * write_inode_now	-	write an inode to disk
2420  * @inode: inode to write to disk
2421  * @sync: whether the write should be synchronous or not
2422  *
2423  * This function commits an inode to disk immediately if it is dirty. This is
2424  * primarily needed by knfsd.
2425  *
2426  * The caller must either have a ref on the inode or must have set I_WILL_FREE.
2427  */
write_inode_now(struct inode * inode,int sync)2428 int write_inode_now(struct inode *inode, int sync)
2429 {
2430 	struct writeback_control wbc = {
2431 		.nr_to_write = LONG_MAX,
2432 		.sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
2433 		.range_start = 0,
2434 		.range_end = LLONG_MAX,
2435 	};
2436 
2437 	if (!mapping_cap_writeback_dirty(inode->i_mapping))
2438 		wbc.nr_to_write = 0;
2439 
2440 	might_sleep();
2441 	return writeback_single_inode(inode, &wbc);
2442 }
2443 EXPORT_SYMBOL(write_inode_now);
2444 
2445 /**
2446  * sync_inode - write an inode and its pages to disk.
2447  * @inode: the inode to sync
2448  * @wbc: controls the writeback mode
2449  *
2450  * sync_inode() will write an inode and its pages to disk.  It will also
2451  * correctly update the inode on its superblock's dirty inode lists and will
2452  * update inode->i_state.
2453  *
2454  * The caller must have a ref on the inode.
2455  */
sync_inode(struct inode * inode,struct writeback_control * wbc)2456 int sync_inode(struct inode *inode, struct writeback_control *wbc)
2457 {
2458 	return writeback_single_inode(inode, wbc);
2459 }
2460 EXPORT_SYMBOL(sync_inode);
2461 
2462 /**
2463  * sync_inode_metadata - write an inode to disk
2464  * @inode: the inode to sync
2465  * @wait: wait for I/O to complete.
2466  *
2467  * Write an inode to disk and adjust its dirty state after completion.
2468  *
2469  * Note: only writes the actual inode, no associated data or other metadata.
2470  */
sync_inode_metadata(struct inode * inode,int wait)2471 int sync_inode_metadata(struct inode *inode, int wait)
2472 {
2473 	struct writeback_control wbc = {
2474 		.sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
2475 		.nr_to_write = 0, /* metadata-only */
2476 	};
2477 
2478 	return sync_inode(inode, &wbc);
2479 }
2480 EXPORT_SYMBOL(sync_inode_metadata);
2481