Lines Matching +full:charge +full:- +full:current +full:- +full:limit +full:- +full:mapping
2 * mm/page-writeback.c
24 #include <linux/backing-dev.h>
53 #define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10))
99 * The interval between `kupdate'-style writebacks
123 /* End of sysctl-exported parameters */
141 unsigned long wb_dirty; /* per-wb counterparts */
151 * reflect changes in current writeout rate.
159 .wb_completions = &(__wb)->completions
165 .wb_completions = &(__wb)->memcg_completions, \
170 return dtc->dom; in mdtc_valid()
175 return dtc->dom; in dtc_dom()
180 return mdtc->gdtc; in mdtc_gdtc()
185 return &wb->memcg_completions; in wb_memcg_completions()
191 unsigned long this_bw = wb->avg_write_bandwidth; in wb_min_max_ratio()
192 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); in wb_min_max_ratio()
193 unsigned long long min = wb->bdi->min_ratio; in wb_min_max_ratio()
194 unsigned long long max = wb->bdi->max_ratio; in wb_min_max_ratio()
218 .wb_completions = &(__wb)->completions
245 *minp = wb->bdi->min_ratio; in wb_min_max_ratio()
246 *maxp = wb->bdi->max_ratio; in wb_min_max_ratio()
259 * user-configurable dirty ratio is the effictive number of pages that
263 * Because the user is allowed to specify the dirty limit globally as
264 * absolute number of bytes, calculating the per-zone dirty limit can
265 * require translating the configured limit into a percentage of
270 * node_dirtyable_memory - number of dirtyable pages in a node
274 * page cache. This is the base value for the per-node dirty limits.
282 struct zone *zone = pgdat->node_zones + z; in node_dirtyable_memory()
295 nr_pages -= min(nr_pages, pgdat->totalreserve_pages); in node_dirtyable_memory()
318 z = &NODE_DATA(node)->node_zones[i]; in highmem_dirtyable_memory()
324 nr_pages -= min(nr_pages, high_wmark_pages(z)); in highmem_dirtyable_memory()
356 * global_dirtyable_memory - number of globally dirtyable pages
371 x -= min(x, totalreserve_pages); in global_dirtyable_memory()
377 x -= highmem_dirtyable_memory(x); in global_dirtyable_memory()
383 * domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain
386 * Calculate @dtc->thresh and ->bg_thresh considering
388 * must ensure that @dtc->avail is set before calling this function. The
390 * real-time tasks.
394 const unsigned long available_memory = dtc->avail; in domain_dirty_limits()
398 /* convert ratios to per-PAGE_SIZE for higher precision */ in domain_dirty_limits()
407 unsigned long global_avail = gdtc->avail; in domain_dirty_limits()
413 * per-PAGE_SIZE, they can be obtained by dividing bytes by in domain_dirty_limits()
437 tsk = current; in domain_dirty_limits()
438 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) { in domain_dirty_limits()
442 dtc->thresh = thresh; in domain_dirty_limits()
443 dtc->bg_thresh = bg_thresh; in domain_dirty_limits()
451 * global_dirty_limits - background-writeback and dirty-throttling thresholds
470 * node_dirty_limit - maximum number of dirty pages allowed in a node
479 struct task_struct *tsk = current; in node_dirty_limit()
488 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) in node_dirty_limit()
495 * node_dirty_ok - tells whether a node is within its dirty limits
499 * dirty limit, %false if the limit is exceeded.
503 unsigned long limit = node_dirty_limit(pgdat); in node_dirty_ok() local
510 return nr_pages <= limit; in node_dirty_ok()
580 __fprop_inc_percpu_max(&dom->completions, completions, in wb_domain_writeout_inc()
583 if (unlikely(!dom->period_time)) { in wb_domain_writeout_inc()
590 dom->period_time = wp_next_time(jiffies); in wb_domain_writeout_inc()
591 mod_timer(&dom->period_timer, dom->period_time); in wb_domain_writeout_inc()
604 wb_domain_writeout_inc(&global_wb_domain, &wb->completions, in __wb_writeout_inc()
605 wb->bdi->max_prop_frac); in __wb_writeout_inc()
610 wb->bdi->max_prop_frac); in __wb_writeout_inc()
630 int miss_periods = (jiffies - dom->period_time) / in writeout_period()
633 if (fprop_new_period(&dom->completions, miss_periods + 1)) { in writeout_period()
634 dom->period_time = wp_next_time(dom->period_time + in writeout_period()
636 mod_timer(&dom->period_timer, dom->period_time); in writeout_period()
642 dom->period_time = 0; in writeout_period()
650 spin_lock_init(&dom->lock); in wb_domain_init()
652 timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE); in wb_domain_init()
654 dom->dirty_limit_tstamp = jiffies; in wb_domain_init()
656 return fprop_global_init(&dom->completions, gfp); in wb_domain_init()
662 del_timer_sync(&dom->period_timer); in wb_domain_exit()
663 fprop_global_destroy(&dom->completions); in wb_domain_exit()
679 if (min_ratio > bdi->max_ratio) { in bdi_set_min_ratio()
680 ret = -EINVAL; in bdi_set_min_ratio()
682 min_ratio -= bdi->min_ratio; in bdi_set_min_ratio()
685 bdi->min_ratio += min_ratio; in bdi_set_min_ratio()
687 ret = -EINVAL; in bdi_set_min_ratio()
700 return -EINVAL; in bdi_set_max_ratio()
703 if (bdi->min_ratio > max_ratio) { in bdi_set_max_ratio()
704 ret = -EINVAL; in bdi_set_max_ratio()
706 bdi->max_ratio = max_ratio; in bdi_set_max_ratio()
707 bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100; in bdi_set_max_ratio()
724 return max(thresh, dom->dirty_limit); in hard_dirty_limit()
729 * system-wide clean memory excluding the amount being used in the domain.
735 unsigned long clean = filepages - min(filepages, mdtc->dirty); in mdtc_calc_avail()
736 unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty); in mdtc_calc_avail()
737 unsigned long other_clean = global_clean - min(global_clean, clean); in mdtc_calc_avail()
739 mdtc->avail = filepages + min(headroom, other_clean); in mdtc_calc_avail()
743 * __wb_calc_thresh - @wb's share of dirty throttling threshold
746 * Returns @wb's dirty limit in pages. The term "dirty" in the context of
749 * Note that balance_dirty_pages() will only seriously take it as a hard limit
757 * - starving fast devices
758 * - piling up dirty pages (that will take long time to sync) on slow devices
760 * The wb's share of dirty limit will be adapting to its throughput and
761 * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
766 unsigned long thresh = dtc->thresh; in __wb_calc_thresh()
774 fprop_fraction_percpu(&dom->completions, dtc->wb_completions, in __wb_calc_thresh()
777 wb_thresh = (thresh * (100 - bdi_min_ratio)) / 100; in __wb_calc_thresh()
781 wb_min_max_ratio(dtc->wb, &wb_min_ratio, &wb_max_ratio); in __wb_calc_thresh()
798 * setpoint - dirty 3
799 * f(dirty) := 1.0 + (----------------)
800 * limit - setpoint
806 * (3) f(limit) = 0 => the hard limit
813 unsigned long limit) in pos_ratio_polynom() argument
818 x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT, in pos_ratio_polynom()
819 (limit - setpoint) | 1); in pos_ratio_polynom()
866 * 0 +------------.------------------.----------------------*------------->
867 * freerun^ setpoint^ limit^ dirty pages
894 * 0 +----------------------.-------------------------------.------------->
899 * - start writing to a slow SD card and a fast disk at the same time. The SD
901 * - the wb dirty thresh drops quickly due to change of JBOD workload
905 struct bdi_writeback *wb = dtc->wb; in wb_position_ratio()
906 unsigned long write_bw = wb->avg_write_bandwidth; in wb_position_ratio()
907 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); in wb_position_ratio()
908 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); in wb_position_ratio() local
909 unsigned long wb_thresh = dtc->wb_thresh; in wb_position_ratio()
914 long long pos_ratio; /* for scaling up/down the rate limit */ in wb_position_ratio()
917 dtc->pos_ratio = 0; in wb_position_ratio()
919 if (unlikely(dtc->dirty >= limit)) in wb_position_ratio()
927 setpoint = (freerun + limit) / 2; in wb_position_ratio()
928 pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit); in wb_position_ratio()
935 * This is especially important for fuse which sets bdi->max_ratio to in wb_position_ratio()
942 * total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global in wb_position_ratio()
955 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { in wb_position_ratio()
958 if (dtc->wb_dirty < 8) { in wb_position_ratio()
959 dtc->pos_ratio = min_t(long long, pos_ratio * 2, in wb_position_ratio()
964 if (dtc->wb_dirty >= wb_thresh) in wb_position_ratio()
968 dtc->wb_bg_thresh); in wb_position_ratio()
973 wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty, in wb_position_ratio()
983 * wb's) while given strictlimit wb is below limit. in wb_position_ratio()
986 * but it would look too non-natural for the case of all in wb_position_ratio()
988 * with bdi->max_ratio == 100%. in wb_position_ratio()
997 dtc->pos_ratio = min(pos_ratio, wb_pos_ratio); in wb_position_ratio()
1010 * f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint) in wb_position_ratio()
1012 * x_intercept - wb_dirty in wb_position_ratio()
1013 * := -------------------------- in wb_position_ratio()
1014 * x_intercept - wb_setpoint in wb_position_ratio()
1019 * (2) k = - 1 / (8 * write_bw) (in single wb case) in wb_position_ratio()
1024 * [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2] in wb_position_ratio()
1032 if (unlikely(wb_thresh > dtc->thresh)) in wb_position_ratio()
1033 wb_thresh = dtc->thresh; in wb_position_ratio()
1041 wb_thresh = max(wb_thresh, (limit - dtc->dirty) / 8); in wb_position_ratio()
1046 x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1); in wb_position_ratio()
1050 * (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case. in wb_position_ratio()
1052 * wb_thresh thresh - wb_thresh in wb_position_ratio()
1053 * span = --------- * (8 * write_bw) + ------------------ * wb_thresh in wb_position_ratio()
1056 span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16; in wb_position_ratio()
1059 if (dtc->wb_dirty < x_intercept - span / 4) { in wb_position_ratio()
1060 pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty), in wb_position_ratio()
1061 (x_intercept - wb_setpoint) | 1); in wb_position_ratio()
1071 if (dtc->wb_dirty < x_intercept) { in wb_position_ratio()
1072 if (dtc->wb_dirty > x_intercept / 8) in wb_position_ratio()
1074 dtc->wb_dirty); in wb_position_ratio()
1079 dtc->pos_ratio = pos_ratio; in wb_position_ratio()
1087 unsigned long avg = wb->avg_write_bandwidth; in wb_update_write_bandwidth()
1088 unsigned long old = wb->write_bandwidth; in wb_update_write_bandwidth()
1094 * bw * elapsed + write_bandwidth * (period - elapsed) in wb_update_write_bandwidth()
1095 * write_bandwidth = --------------------------------------------------- in wb_update_write_bandwidth()
1101 bw = written - min(written, wb->written_stamp); in wb_update_write_bandwidth()
1108 bw += (u64)wb->write_bandwidth * (period - elapsed); in wb_update_write_bandwidth()
1115 avg -= (avg - old) >> 3; in wb_update_write_bandwidth()
1118 avg += (old - avg) >> 3; in wb_update_write_bandwidth()
1124 long delta = avg - wb->avg_write_bandwidth; in wb_update_write_bandwidth()
1126 &wb->bdi->tot_write_bandwidth) <= 0); in wb_update_write_bandwidth()
1128 wb->write_bandwidth = bw; in wb_update_write_bandwidth()
1129 wb->avg_write_bandwidth = avg; in wb_update_write_bandwidth()
1135 unsigned long thresh = dtc->thresh; in update_dirty_limit()
1136 unsigned long limit = dom->dirty_limit; in update_dirty_limit() local
1141 if (limit < thresh) { in update_dirty_limit()
1142 limit = thresh; in update_dirty_limit()
1149 * dom->dirty_limit which is guaranteed to lie above the dirty pages. in update_dirty_limit()
1151 thresh = max(thresh, dtc->dirty); in update_dirty_limit()
1152 if (limit > thresh) { in update_dirty_limit()
1153 limit -= (limit - thresh) >> 5; in update_dirty_limit()
1158 dom->dirty_limit = limit; in update_dirty_limit()
1169 if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) in domain_update_bandwidth()
1172 spin_lock(&dom->lock); in domain_update_bandwidth()
1173 if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) { in domain_update_bandwidth()
1175 dom->dirty_limit_tstamp = now; in domain_update_bandwidth()
1177 spin_unlock(&dom->lock); in domain_update_bandwidth()
1181 * Maintain wb->dirty_ratelimit, the base dirty throttle rate.
1190 struct bdi_writeback *wb = dtc->wb; in wb_update_dirty_ratelimit()
1191 unsigned long dirty = dtc->dirty; in wb_update_dirty_ratelimit()
1192 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); in wb_update_dirty_ratelimit()
1193 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); in wb_update_dirty_ratelimit() local
1194 unsigned long setpoint = (freerun + limit) / 2; in wb_update_dirty_ratelimit()
1195 unsigned long write_bw = wb->avg_write_bandwidth; in wb_update_dirty_ratelimit()
1196 unsigned long dirty_ratelimit = wb->dirty_ratelimit; in wb_update_dirty_ratelimit()
1206 * when dirty pages are truncated by userspace or re-dirtied by FS. in wb_update_dirty_ratelimit()
1208 dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed; in wb_update_dirty_ratelimit()
1214 dtc->pos_ratio >> RATELIMIT_CALC_SHIFT; in wb_update_dirty_ratelimit()
1221 * formula will yield the balanced rate limit (write_bw / N). in wb_update_dirty_ratelimit()
1258 * wb->dirty_ratelimit = balanced_dirty_ratelimit; in wb_update_dirty_ratelimit()
1262 * limit the step size. in wb_update_dirty_ratelimit()
1266 * task_ratelimit - dirty_ratelimit in wb_update_dirty_ratelimit()
1267 * = (pos_ratio - 1) * dirty_ratelimit in wb_update_dirty_ratelimit()
1276 * - dirty_ratelimit > balanced_dirty_ratelimit in wb_update_dirty_ratelimit()
1277 * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint) in wb_update_dirty_ratelimit()
1283 * |task_ratelimit - dirty_ratelimit| is used to limit the step size in wb_update_dirty_ratelimit()
1302 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { in wb_update_dirty_ratelimit()
1303 dirty = dtc->wb_dirty; in wb_update_dirty_ratelimit()
1304 if (dtc->wb_dirty < 8) in wb_update_dirty_ratelimit()
1305 setpoint = dtc->wb_dirty + 1; in wb_update_dirty_ratelimit()
1307 setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2; in wb_update_dirty_ratelimit()
1311 x = min3(wb->balanced_dirty_ratelimit, in wb_update_dirty_ratelimit()
1314 step = x - dirty_ratelimit; in wb_update_dirty_ratelimit()
1316 x = max3(wb->balanced_dirty_ratelimit, in wb_update_dirty_ratelimit()
1319 step = dirty_ratelimit - x; in wb_update_dirty_ratelimit()
1336 dirty_ratelimit -= step; in wb_update_dirty_ratelimit()
1338 wb->dirty_ratelimit = max(dirty_ratelimit, 1UL); in wb_update_dirty_ratelimit()
1339 wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit; in wb_update_dirty_ratelimit()
1349 struct bdi_writeback *wb = gdtc->wb; in __wb_update_bandwidth()
1351 unsigned long elapsed = now - wb->bw_time_stamp; in __wb_update_bandwidth()
1355 lockdep_assert_held(&wb->list_lock); in __wb_update_bandwidth()
1358 * rate-limit, only update once every 200ms. in __wb_update_bandwidth()
1363 dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]); in __wb_update_bandwidth()
1364 written = percpu_counter_read(&wb->stat[WB_WRITTEN]); in __wb_update_bandwidth()
1367 * Skip quiet periods when disk bandwidth is under-utilized. in __wb_update_bandwidth()
1370 if (elapsed > HZ && time_before(wb->bw_time_stamp, start_time)) in __wb_update_bandwidth()
1389 wb->dirtied_stamp = dirtied; in __wb_update_bandwidth()
1390 wb->written_stamp = written; in __wb_update_bandwidth()
1391 wb->bw_time_stamp = now; in __wb_update_bandwidth()
1406 * global_zone_page_state() too often. So scale it near-sqrt to the safety margin
1413 return 1UL << (ilog2(thresh - dirty) >> 1); in dirty_poll_interval()
1421 unsigned long bw = wb->avg_write_bandwidth; in wb_max_pause()
1425 * Limit pause time for small memory systems. If sleeping for too long in wb_max_pause()
1443 long hi = ilog2(wb->avg_write_bandwidth); in wb_min_pause()
1444 long lo = ilog2(wb->dirty_ratelimit); in wb_min_pause()
1449 /* target for 10ms pause on 1-dd case */ in wb_min_pause()
1459 t += (hi - lo) * (10 * HZ) / 1024; in wb_min_pause()
1475 * 2) limit the target pause time to max_pause/2, so that the normal in wb_min_pause()
1484 * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}. in wb_min_pause()
1514 struct bdi_writeback *wb = dtc->wb; in wb_dirty_limits()
1520 * - in JBOD setup, wb_thresh can fluctuate a lot in wb_dirty_limits()
1521 * - in a system with HDD and USB key, the USB key may somehow in wb_dirty_limits()
1530 dtc->wb_thresh = __wb_calc_thresh(dtc); in wb_dirty_limits()
1531 dtc->wb_bg_thresh = dtc->thresh ? in wb_dirty_limits()
1532 div_u64((u64)dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0; in wb_dirty_limits()
1540 * reported dirty, even though there are thresh-m pages in wb_dirty_limits()
1544 if (dtc->wb_thresh < 2 * wb_stat_error()) { in wb_dirty_limits()
1546 dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK); in wb_dirty_limits()
1549 dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK); in wb_dirty_limits()
1578 struct backing_dev_info *bdi = wb->bdi; in balance_dirty_pages()
1579 bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT; in balance_dirty_pages()
1597 gdtc->avail = global_dirtyable_memory(); in balance_dirty_pages()
1598 gdtc->dirty = nr_reclaimable + global_node_page_state(NR_WRITEBACK); in balance_dirty_pages()
1605 dirty = gdtc->wb_dirty; in balance_dirty_pages()
1606 thresh = gdtc->wb_thresh; in balance_dirty_pages()
1607 bg_thresh = gdtc->wb_bg_thresh; in balance_dirty_pages()
1609 dirty = gdtc->dirty; in balance_dirty_pages()
1610 thresh = gdtc->thresh; in balance_dirty_pages()
1611 bg_thresh = gdtc->bg_thresh; in balance_dirty_pages()
1622 &mdtc->dirty, &writeback); in balance_dirty_pages()
1623 mdtc->dirty += writeback; in balance_dirty_pages()
1630 m_dirty = mdtc->wb_dirty; in balance_dirty_pages()
1631 m_thresh = mdtc->wb_thresh; in balance_dirty_pages()
1632 m_bg_thresh = mdtc->wb_bg_thresh; in balance_dirty_pages()
1634 m_dirty = mdtc->dirty; in balance_dirty_pages()
1635 m_thresh = mdtc->thresh; in balance_dirty_pages()
1636 m_bg_thresh = mdtc->bg_thresh; in balance_dirty_pages()
1642 * catch-up. This avoids (excessively) small writeouts in balance_dirty_pages()
1647 * up are the price we consciously pay for strictlimit-ing. in balance_dirty_pages()
1658 current->dirty_paused_when = now; in balance_dirty_pages()
1659 current->nr_dirtied = 0; in balance_dirty_pages()
1662 current->nr_dirtied_pause = min(intv, m_intv); in balance_dirty_pages()
1676 dirty_exceeded = (gdtc->wb_dirty > gdtc->wb_thresh) && in balance_dirty_pages()
1677 ((gdtc->dirty > gdtc->thresh) || strictlimit); in balance_dirty_pages()
1692 dirty_exceeded |= (mdtc->wb_dirty > mdtc->wb_thresh) && in balance_dirty_pages()
1693 ((mdtc->dirty > mdtc->thresh) || strictlimit); in balance_dirty_pages()
1696 if (mdtc->pos_ratio < gdtc->pos_ratio) in balance_dirty_pages()
1700 if (dirty_exceeded && !wb->dirty_exceeded) in balance_dirty_pages()
1701 wb->dirty_exceeded = 1; in balance_dirty_pages()
1703 if (time_is_before_jiffies(wb->bw_time_stamp + in balance_dirty_pages()
1705 spin_lock(&wb->list_lock); in balance_dirty_pages()
1707 spin_unlock(&wb->list_lock); in balance_dirty_pages()
1711 dirty_ratelimit = wb->dirty_ratelimit; in balance_dirty_pages()
1712 task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >> in balance_dirty_pages()
1714 max_pause = wb_max_pause(wb, sdtc->wb_dirty); in balance_dirty_pages()
1726 if (current->dirty_paused_when) in balance_dirty_pages()
1727 pause -= now - current->dirty_paused_when; in balance_dirty_pages()
1730 * for up to 800ms from time to time on 1-HDD; so does xfs, in balance_dirty_pages()
1737 sdtc->thresh, in balance_dirty_pages()
1738 sdtc->bg_thresh, in balance_dirty_pages()
1739 sdtc->dirty, in balance_dirty_pages()
1740 sdtc->wb_thresh, in balance_dirty_pages()
1741 sdtc->wb_dirty, in balance_dirty_pages()
1748 if (pause < -HZ) { in balance_dirty_pages()
1749 current->dirty_paused_when = now; in balance_dirty_pages()
1750 current->nr_dirtied = 0; in balance_dirty_pages()
1752 current->dirty_paused_when += period; in balance_dirty_pages()
1753 current->nr_dirtied = 0; in balance_dirty_pages()
1754 } else if (current->nr_dirtied_pause <= pages_dirtied) in balance_dirty_pages()
1755 current->nr_dirtied_pause += pages_dirtied; in balance_dirty_pages()
1760 now += min(pause - max_pause, max_pause); in balance_dirty_pages()
1766 sdtc->thresh, in balance_dirty_pages()
1767 sdtc->bg_thresh, in balance_dirty_pages()
1768 sdtc->dirty, in balance_dirty_pages()
1769 sdtc->wb_thresh, in balance_dirty_pages()
1770 sdtc->wb_dirty, in balance_dirty_pages()
1778 wb->dirty_sleep = now; in balance_dirty_pages()
1781 current->dirty_paused_when = now + pause; in balance_dirty_pages()
1782 current->nr_dirtied = 0; in balance_dirty_pages()
1783 current->nr_dirtied_pause = nr_dirtied_pause; in balance_dirty_pages()
1797 * In theory 1 page is enough to keep the consumer-producer in balance_dirty_pages()
1802 if (sdtc->wb_dirty <= wb_stat_error()) in balance_dirty_pages()
1805 if (fatal_signal_pending(current)) in balance_dirty_pages()
1809 if (!dirty_exceeded && wb->dirty_exceeded) in balance_dirty_pages()
1810 wb->dirty_exceeded = 0; in balance_dirty_pages()
1826 if (nr_reclaimable > gdtc->bg_thresh) in balance_dirty_pages()
1835 * dirty tsk->nr_dirtied_pause pages;
1839 * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be
1841 * throttled page dirties in dirty_throttle_leaks on task exit and charge them
1849 * balance_dirty_pages_ratelimited - balance dirty memory state
1850 * @mapping: address_space which was dirtied
1858 * limit we decrease the ratelimiting by a lot, to prevent individual processes
1859 * from overshooting the limit by (ratelimit_pages) each.
1861 void balance_dirty_pages_ratelimited(struct address_space *mapping) in balance_dirty_pages_ratelimited() argument
1863 struct inode *inode = mapping->host; in balance_dirty_pages_ratelimited()
1875 wb = &bdi->wb; in balance_dirty_pages_ratelimited()
1877 ratelimit = current->nr_dirtied_pause; in balance_dirty_pages_ratelimited()
1878 if (wb->dirty_exceeded) in balance_dirty_pages_ratelimited()
1879 ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10)); in balance_dirty_pages_ratelimited()
1886 * time, hence all honoured too large initial task->nr_dirtied_pause. in balance_dirty_pages_ratelimited()
1889 if (unlikely(current->nr_dirtied >= ratelimit)) in balance_dirty_pages_ratelimited()
1897 * short-lived tasks (eg. gcc invocations in a kernel build) escaping in balance_dirty_pages_ratelimited()
1898 * the dirty throttling and livelock other long-run dirtiers. in balance_dirty_pages_ratelimited()
1901 if (*p > 0 && current->nr_dirtied < ratelimit) { in balance_dirty_pages_ratelimited()
1903 nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied); in balance_dirty_pages_ratelimited()
1904 *p -= nr_pages_dirtied; in balance_dirty_pages_ratelimited()
1905 current->nr_dirtied += nr_pages_dirtied; in balance_dirty_pages_ratelimited()
1909 if (unlikely(current->nr_dirtied >= ratelimit)) in balance_dirty_pages_ratelimited()
1910 balance_dirty_pages(wb, current->nr_dirtied); in balance_dirty_pages_ratelimited()
1917 * wb_over_bg_thresh - does @wb need to be written back?
1935 gdtc->avail = global_dirtyable_memory(); in wb_over_bg_thresh()
1936 gdtc->dirty = global_node_page_state(NR_FILE_DIRTY) + in wb_over_bg_thresh()
1940 if (gdtc->dirty > gdtc->bg_thresh) in wb_over_bg_thresh()
1944 wb_calc_thresh(gdtc->wb, gdtc->bg_thresh)) in wb_over_bg_thresh()
1950 mem_cgroup_wb_stats(wb, &filepages, &headroom, &mdtc->dirty, in wb_over_bg_thresh()
1955 if (mdtc->dirty > mdtc->bg_thresh) in wb_over_bg_thresh()
1959 wb_calc_thresh(mdtc->wb, mdtc->bg_thresh)) in wb_over_bg_thresh()
1979 * and a different non-zero value will wakeup the writeback threads. in dirty_writeback_centisecs_handler()
2003 * then push it back - the user is still using the disk.
2007 mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode); in laptop_io_completion()
2022 del_timer(&bdi->laptop_mode_wb_timer); in laptop_sync_completion()
2029 * If ratelimit_pages is too high then we can get into dirty-data overload
2046 dom->dirty_limit = dirty_thresh; in writeback_set_ratelimit()
2067 * is now applied to total non-HIGHPAGE memory (by subtracting
2071 * non-HIGHMEM memory.
2087 * tag_pages_for_writeback - tag pages to be written by write_cache_pages
2088 * @mapping: address space structure to write
2104 void tag_pages_for_writeback(struct address_space *mapping, in tag_pages_for_writeback() argument
2112 xa_lock_irq(&mapping->i_pages); in tag_pages_for_writeback()
2113 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, in tag_pages_for_writeback()
2117 radix_tree_iter_tag_set(&mapping->i_pages, &iter, in tag_pages_for_writeback()
2123 xa_unlock_irq(&mapping->i_pages); in tag_pages_for_writeback()
2125 xa_lock_irq(&mapping->i_pages); in tag_pages_for_writeback()
2127 xa_unlock_irq(&mapping->i_pages); in tag_pages_for_writeback()
2132 …* write_cache_pages - walk the list of dirty pages of the given address space and write all of the…
2133 * @mapping: address space structure to write
2134 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
2139 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
2140 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
2142 * the call was made get new I/O started against them. If wbc->sync_mode is
2148 * writing them. For data-integrity sync we have to be careful so that we do
2156 * lock/page writeback access order inversion - we should only ever lock
2157 * multiple pages in ascending page->index order, and looping back to the start
2160 int write_cache_pages(struct address_space *mapping, in write_cache_pages() argument
2177 if (wbc->range_cyclic) { in write_cache_pages()
2178 writeback_index = mapping->writeback_index; /* prev offset */ in write_cache_pages()
2180 end = -1; in write_cache_pages()
2182 index = wbc->range_start >> PAGE_SHIFT; in write_cache_pages()
2183 end = wbc->range_end >> PAGE_SHIFT; in write_cache_pages()
2184 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) in write_cache_pages()
2187 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) in write_cache_pages()
2191 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) in write_cache_pages()
2192 tag_pages_for_writeback(mapping, index, end); in write_cache_pages()
2197 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, in write_cache_pages()
2205 done_index = page->index; in write_cache_pages()
2217 if (unlikely(page->mapping != mapping)) { in write_cache_pages()
2229 if (wbc->sync_mode != WB_SYNC_NONE) in write_cache_pages()
2239 trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); in write_cache_pages()
2257 } else if (wbc->sync_mode != WB_SYNC_ALL) { in write_cache_pages()
2259 done_index = page->index + 1; in write_cache_pages()
2273 if (--wbc->nr_to_write <= 0 && in write_cache_pages()
2274 wbc->sync_mode == WB_SYNC_NONE) { in write_cache_pages()
2288 if (wbc->range_cyclic && !done) in write_cache_pages()
2290 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) in write_cache_pages()
2291 mapping->writeback_index = done_index; in write_cache_pages()
2299 * function and set the mapping flags on error
2304 struct address_space *mapping = data; in __writepage() local
2305 int ret = mapping->a_ops->writepage(page, wbc); in __writepage()
2306 mapping_set_error(mapping, ret); in __writepage()
2311 …* generic_writepages - walk the list of dirty pages of the given address space and writepage() all…
2312 * @mapping: address space structure to write
2313 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
2318 int generic_writepages(struct address_space *mapping, in generic_writepages() argument
2325 if (!mapping->a_ops->writepage) in generic_writepages()
2329 ret = write_cache_pages(mapping, wbc, __writepage, mapping); in generic_writepages()
2336 int do_writepages(struct address_space *mapping, struct writeback_control *wbc) in do_writepages() argument
2340 if (wbc->nr_to_write <= 0) in do_writepages()
2343 if (mapping->a_ops->writepages) in do_writepages()
2344 ret = mapping->a_ops->writepages(mapping, wbc); in do_writepages()
2346 ret = generic_writepages(mapping, wbc); in do_writepages()
2347 if ((ret != -ENOMEM) || (wbc->sync_mode != WB_SYNC_ALL)) in do_writepages()
2356 * write_one_page - write out a single page and wait on I/O
2361 * Note that the mapping's AS_EIO/AS_ENOSPC flags will be cleared when this
2366 struct address_space *mapping = page->mapping; in write_one_page() local
2379 ret = mapping->a_ops->writepage(page, &wbc); in write_one_page()
2388 ret = filemap_check_errors(mapping); in write_one_page()
2410 void account_page_dirtied(struct page *page, struct address_space *mapping) in account_page_dirtied() argument
2412 struct inode *inode = mapping->host; in account_page_dirtied()
2414 trace_writeback_dirty_page(page, mapping); in account_page_dirtied()
2416 if (mapping_cap_account_dirty(mapping)) { in account_page_dirtied()
2428 current->nr_dirtied++; in account_page_dirtied()
2439 void account_page_cleaned(struct page *page, struct address_space *mapping, in account_page_cleaned() argument
2442 if (mapping_cap_account_dirty(mapping)) { in account_page_cleaned()
2455 * page dirty in that case, but not all the buffers. This is a "bottom-up"
2456 * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying.
2466 struct address_space *mapping = page_mapping(page); in __set_page_dirty_nobuffers() local
2469 if (!mapping) { in __set_page_dirty_nobuffers()
2474 xa_lock_irqsave(&mapping->i_pages, flags); in __set_page_dirty_nobuffers()
2475 BUG_ON(page_mapping(page) != mapping); in __set_page_dirty_nobuffers()
2477 account_page_dirtied(page, mapping); in __set_page_dirty_nobuffers()
2478 radix_tree_tag_set(&mapping->i_pages, page_index(page), in __set_page_dirty_nobuffers()
2480 xa_unlock_irqrestore(&mapping->i_pages, flags); in __set_page_dirty_nobuffers()
2483 if (mapping->host) { in __set_page_dirty_nobuffers()
2485 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); in __set_page_dirty_nobuffers()
2495 * Call this whenever redirtying a page, to de-account the dirty counters
2496 * (NR_DIRTIED, WB_DIRTIED, tsk->nr_dirtied), so that they match the written
2503 struct address_space *mapping = page->mapping; in account_page_redirty() local
2505 if (mapping && mapping_cap_account_dirty(mapping)) { in account_page_redirty()
2506 struct inode *inode = mapping->host; in account_page_redirty()
2511 current->nr_dirtied--; in account_page_redirty()
2528 wbc->pages_skipped++; in redirty_page_for_writepage()
2538 * For pages with a mapping this should be done under the page lock
2543 * If the mapping doesn't provide a set_page_dirty a_op, then
2548 struct address_space *mapping = page_mapping(page); in set_page_dirty() local
2551 if (likely(mapping)) { in set_page_dirty()
2552 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty; in set_page_dirty()
2581 * page->mapping->host, and if the page is unlocked. This is because another
2582 * CPU could truncate the page off the mapping and then free the mapping.
2584 * Usually, the page _is_ locked, or the caller is a user-space process which
2615 struct address_space *mapping = page_mapping(page); in __cancel_dirty_page() local
2617 if (mapping_cap_account_dirty(mapping)) { in __cancel_dirty_page()
2618 struct inode *inode = mapping->host; in __cancel_dirty_page()
2626 account_page_cleaned(page, mapping, wb); in __cancel_dirty_page()
2641 * tagged as dirty in the radix tree so that a concurrent write-for-sync
2642 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage
2644 * at which stage we bring the page's dirty flag and radix-tree dirty tag
2647 * This incoherency between the page's dirty flag and radix-tree tag is
2652 struct address_space *mapping = page_mapping(page); in clear_page_dirty_for_io() local
2657 if (mapping && mapping_cap_account_dirty(mapping)) { in clear_page_dirty_for_io()
2658 struct inode *inode = mapping->host; in clear_page_dirty_for_io()
2667 * (b) we tell the low-level filesystem to in clear_page_dirty_for_io()
2678 * has no effect on the actual dirty bit - since in clear_page_dirty_for_io()
2713 struct address_space *mapping = page_mapping(page); in test_clear_page_writeback() local
2720 if (mapping && mapping_use_writeback_tags(mapping)) { in test_clear_page_writeback()
2721 struct inode *inode = mapping->host; in test_clear_page_writeback()
2725 xa_lock_irqsave(&mapping->i_pages, flags); in test_clear_page_writeback()
2728 radix_tree_tag_clear(&mapping->i_pages, page_index(page), in test_clear_page_writeback()
2738 if (mapping->host && !mapping_tagged(mapping, in test_clear_page_writeback()
2740 sb_clear_inode_writeback(mapping->host); in test_clear_page_writeback()
2742 xa_unlock_irqrestore(&mapping->i_pages, flags); in test_clear_page_writeback()
2763 struct address_space *mapping = page_mapping(page); in __test_set_page_writeback() local
2767 if (mapping && mapping_use_writeback_tags(mapping)) { in __test_set_page_writeback()
2768 struct inode *inode = mapping->host; in __test_set_page_writeback()
2772 xa_lock_irqsave(&mapping->i_pages, flags); in __test_set_page_writeback()
2777 on_wblist = mapping_tagged(mapping, in __test_set_page_writeback()
2780 radix_tree_tag_set(&mapping->i_pages, page_index(page), in __test_set_page_writeback()
2790 if (mapping->host && !on_wblist) in __test_set_page_writeback()
2791 sb_mark_inode_writeback(mapping->host); in __test_set_page_writeback()
2794 radix_tree_tag_clear(&mapping->i_pages, page_index(page), in __test_set_page_writeback()
2797 radix_tree_tag_clear(&mapping->i_pages, page_index(page), in __test_set_page_writeback()
2799 xa_unlock_irqrestore(&mapping->i_pages, flags); in __test_set_page_writeback()
2814 * Return true if any of the pages in the mapping are marked with the
2817 int mapping_tagged(struct address_space *mapping, int tag) in mapping_tagged() argument
2819 return radix_tree_tagged(&mapping->i_pages, tag); in mapping_tagged()
2824 * wait_for_stable_page() - wait for writeback to finish, if necessary.
2833 if (bdi_cap_stable_pages_required(inode_to_bdi(page->mapping->host))) in wait_for_stable_page()