1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * mm/reclaim_acct.c
4 *
5 * Copyright (c) 2022 Huawei Technologies Co., Ltd.
6 */
7
8 #include <linux/slab.h>
9 #include <linux/types.h>
10
11 #include "internal.h"
12
13
14 const char *stub_name[NR_RA_STUBS] = {
15 "direct_reclaim",
16 "drain_all_pages",
17 "shrink_file_list",
18 "shrink_anon_list",
19 "shrink_slab",
20 };
21
22 /* Once initialized, the variable should never be changed */
23 static bool reclaimacct_is_off = true;
24 static int reclaimacct_disable = 1;
25
reclaimacct_free(struct reclaim_acct * ra,enum reclaim_type type)26 static void reclaimacct_free(struct reclaim_acct *ra, enum reclaim_type type)
27 {
28 memset(ra, 0, sizeof(struct reclaim_acct));
29 }
30
__reclaimacct_end(struct reclaim_acct * ra,u64 freed,enum reclaimacct_stubs stub,const struct shrinker * shrinker)31 static void __reclaimacct_end(struct reclaim_acct *ra, u64 freed,
32 enum reclaimacct_stubs stub, const struct shrinker *shrinker)
33 {
34 u64 now, delay, start;
35
36 start = ra->start[stub];
37 now = ktime_get_ns();
38 if (now < start)
39 return;
40
41 delay = now - start;
42 if (delay < DELAY_LV5 || is_system_reclaim(ra->reclaim_type)) {
43 ra->delay[stub] += delay;
44 ra->count[stub]++;
45 ra->freed[stub] += freed;
46 }
47
48 if (delay > DELAY_LV4 && delay < DELAY_LV5) {
49 pr_warn_ratelimited("%s timeout:%llu\n", stub_name[stub], delay);
50 if (shrinker)
51 pr_warn_ratelimited("shrinker = %pF\n", shrinker);
52 }
53 }
54
reclaimacct_tsk_init(struct task_struct * tsk)55 void reclaimacct_tsk_init(struct task_struct *tsk)
56 {
57 if (tsk)
58 tsk->reclaim_acct = NULL;
59 }
60
61 /* Reinitialize in case parent's non-null pointer was duped */
reclaimacct_init(void)62 void reclaimacct_init(void)
63 {
64 reclaimacct_tsk_init(&init_task);
65 }
66
reclaimacct_substage_start(enum reclaimacct_stubs stub)67 void reclaimacct_substage_start(enum reclaimacct_stubs stub)
68 {
69 if (!current->reclaim_acct)
70 return;
71
72 current->reclaim_acct->start[stub] = ktime_get_ns();
73 }
74
reclaimacct_substage_end(enum reclaimacct_stubs stub,unsigned long freed,const struct shrinker * shrinker)75 void reclaimacct_substage_end(enum reclaimacct_stubs stub, unsigned long freed,
76 const struct shrinker *shrinker)
77 {
78 if (!current->reclaim_acct)
79 return;
80
81 __reclaimacct_end(current->reclaim_acct, freed, stub, shrinker);
82 }
83
reclaimacct_directreclaim_end(struct reclaim_acct * ra)84 static void reclaimacct_directreclaim_end(struct reclaim_acct *ra)
85 {
86 int i;
87
88 if (ra->delay[RA_RECLAIM] > DELAY_LV4) {
89 pr_warn_ratelimited("Summary");
90 for (i = 0; i < NR_RA_STUBS; i++)
91 pr_warn_ratelimited(" %s=%llu %llu", stub_name[i],
92 ra->delay[i], ra->count[i]);
93 pr_warn_ratelimited("\n");
94 }
95
96 reclaimacct_collect_data();
97 reclaimacct_free(ra, ra->reclaim_type);
98 }
99
reclaimacct_system_reclaim_end(struct reclaim_acct * ra)100 static void reclaimacct_system_reclaim_end(struct reclaim_acct *ra)
101 {
102 reclaimacct_free(ra, ra->reclaim_type);
103 }
104
reclaimacct_start(enum reclaim_type type,struct reclaim_acct * ra)105 void reclaimacct_start(enum reclaim_type type, struct reclaim_acct *ra)
106 {
107 if (reclaimacct_disable || reclaimacct_is_off)
108 return;
109
110 if (!current->reclaim_acct)
111 current->reclaim_acct = ra;
112
113 current->reclaim_acct->reclaim_type = type;
114 current->reclaim_acct->start[RA_RECLAIM] = ktime_get_ns();
115 }
116
reclaimacct_end(enum reclaim_type type)117 void reclaimacct_end(enum reclaim_type type)
118 {
119 if (!current->reclaim_acct)
120 return;
121
122 __reclaimacct_end(current->reclaim_acct, 0, RA_RECLAIM, NULL);
123
124 reclaimacct_collect_reclaim_efficiency();
125
126 if (is_system_reclaim(type))
127 reclaimacct_system_reclaim_end(current->reclaim_acct);
128 else
129 reclaimacct_directreclaim_end(current->reclaim_acct);
130
131 current->reclaim_acct = NULL;
132 }
133
134 /* Reclaim accounting module initialize */
reclaimacct_init_handle(void * p)135 static int reclaimacct_init_handle(void *p)
136 {
137 if (!reclaimacct_initialize_show_data())
138 goto alloc_show_failed;
139
140 reclaimacct_is_off = false;
141 pr_info("enabled\n");
142 return 0;
143
144 alloc_show_failed:
145 reclaimacct_is_off = true;
146 pr_err("disabled\n");
147 return 0;
148 }
149
reclaimacct_module_init(void)150 static int __init reclaimacct_module_init(void)
151 {
152 struct task_struct *task = NULL;
153
154 task = kthread_run(reclaimacct_init_handle, NULL, "reclaimacct_init");
155 if (IS_ERR(task))
156 pr_err("run reclaimacct_init failed\n");
157 else
158 pr_info("run reclaimacct_init successfully\n");
159 return 0;
160 }
161
162 late_initcall(reclaimacct_module_init);
163
reclaimacct_disable_set(const char * val,const struct kernel_param * kp)164 static int reclaimacct_disable_set(const char *val, const struct kernel_param *kp)
165 {
166 int ret;
167
168 ret = param_set_int(val, kp);
169 if (ret)
170 return ret;
171
172 if (!reclaimacct_disable)
173 reclaimacct_reinitialize_show_data();
174 return 0;
175 }
176
177 static const struct kernel_param_ops reclaimacct_disable_ops = {
178 .set = reclaimacct_disable_set,
179 .get = param_get_int,
180 };
181
182 module_param_cb(disable, &reclaimacct_disable_ops, &reclaimacct_disable, 0644);
183