1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Common Primitives for Data Access Monitoring
4 *
5 * Author: SeongJae Park <sj@kernel.org>
6 */
7
8 #include <linux/mmu_notifier.h>
9 #include <linux/page_idle.h>
10 #include <linux/pagemap.h>
11 #include <linux/rmap.h>
12
13 #include "prmtv-common.h"
14
15 /*
16 * Get an online page for a pfn if it's in the LRU list. Otherwise, returns
17 * NULL.
18 *
19 * The body of this function is stolen from the 'page_idle_get_page()'. We
20 * steal rather than reuse it because the code is quite simple.
21 */
damon_get_page(unsigned long pfn)22 struct page *damon_get_page(unsigned long pfn)
23 {
24 struct page *page = pfn_to_online_page(pfn);
25
26 if (!page || !PageLRU(page) || !get_page_unless_zero(page))
27 return NULL;
28
29 if (unlikely(!PageLRU(page))) {
30 put_page(page);
31 page = NULL;
32 }
33 return page;
34 }
35
damon_ptep_mkold(pte_t * pte,struct vm_area_struct * vma,unsigned long addr)36 void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr)
37 {
38 bool referenced = false;
39 struct page *page = damon_get_page(pte_pfn(*pte));
40
41 if (!page)
42 return;
43
44 if (ptep_test_and_clear_young(vma, addr, pte))
45 referenced = true;
46
47 #ifdef CONFIG_MMU_NOTIFIER
48 if (mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE))
49 referenced = true;
50 #endif /* CONFIG_MMU_NOTIFIER */
51
52 if (referenced)
53 set_page_young(page);
54
55 set_page_idle(page);
56 put_page(page);
57 }
58
damon_pmdp_mkold(pmd_t * pmd,struct vm_area_struct * vma,unsigned long addr)59 void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr)
60 {
61 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
62 bool referenced = false;
63 struct page *page = damon_get_page(pmd_pfn(*pmd));
64
65 if (!page)
66 return;
67
68 if (pmdp_test_and_clear_young(vma, addr, pmd))
69 referenced = true;
70
71 #ifdef CONFIG_MMU_NOTIFIER
72 if (mmu_notifier_clear_young(vma->vm_mm, addr,
73 addr + ((1UL) << HPAGE_PMD_SHIFT)))
74 referenced = true;
75 #endif /* CONFIG_MMU_NOTIFIER */
76
77 if (referenced)
78 set_page_young(page);
79
80 set_page_idle(page);
81 put_page(page);
82 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
83 }
84
85 #define DAMON_MAX_SUBSCORE (100)
86 #define DAMON_MAX_AGE_IN_LOG (32)
87
damon_pageout_score(struct damon_ctx * c,struct damon_region * r,struct damos * s)88 int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
89 struct damos *s)
90 {
91 unsigned int max_nr_accesses;
92 int freq_subscore;
93 unsigned int age_in_sec;
94 int age_in_log, age_subscore;
95 unsigned int freq_weight = s->quota.weight_nr_accesses;
96 unsigned int age_weight = s->quota.weight_age;
97 int hotness;
98
99 max_nr_accesses = c->aggr_interval / c->sample_interval;
100 freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
101
102 age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
103 for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
104 age_in_log++, age_in_sec >>= 1)
105 ;
106
107 /* If frequency is 0, higher age means it's colder */
108 if (freq_subscore == 0)
109 age_in_log *= -1;
110
111 /*
112 * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
113 * Scale it to be in [0, 100] and set it as age subscore.
114 */
115 age_in_log += DAMON_MAX_AGE_IN_LOG;
116 age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
117 DAMON_MAX_AGE_IN_LOG / 2;
118
119 hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
120 if (freq_weight + age_weight)
121 hotness /= freq_weight + age_weight;
122 /*
123 * Transform it to fit in [0, DAMOS_MAX_SCORE]
124 */
125 hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
126
127 /* Return coldness of the region */
128 return DAMOS_MAX_SCORE - hotness;
129 }
130