1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * mm/reclaim_acct.c
4 *
5 * Copyright (c) 2022 Huawei Technologies Co., Ltd.
6 */
7
8 #include <linux/slab.h>
9 #include <linux/types.h>
10
11 #include "internal.h"
12
13 /* Once initialized, the variable should never be changed */
14 static bool reclaimacct_is_off = true;
15 static int reclaimacct_disable = 1;
16
reclaimacct_free(struct reclaim_acct * ra,enum reclaim_type type)17 static void reclaimacct_free(struct reclaim_acct *ra, enum reclaim_type type)
18 {
19 memset(ra, 0, sizeof(struct reclaim_acct));
20 }
21
__reclaimacct_end(struct reclaim_acct * ra,u64 freed,enum reclaimacct_stubs stub,const struct shrinker * shrinker)22 static void __reclaimacct_end(struct reclaim_acct *ra, u64 freed,
23 enum reclaimacct_stubs stub, const struct shrinker *shrinker)
24 {
25 u64 now, delay, start;
26
27 start = ra->start[stub];
28 now = ktime_get_ns();
29 if (now < start)
30 return;
31
32 delay = now - start;
33 if (delay < DELAY_LV5 || is_system_reclaim(ra->reclaim_type)) {
34 ra->delay[stub] += delay;
35 ra->count[stub]++;
36 ra->freed[stub] += freed;
37 }
38
39 if (delay > DELAY_LV4 && delay < DELAY_LV5) {
40 pr_warn_ratelimited("%s timeout:%llu\n", stub_name[stub], delay);
41 if (shrinker)
42 pr_warn_ratelimited("shrinker = %pF\n", shrinker);
43 }
44 }
45
reclaimacct_tsk_init(struct task_struct * tsk)46 void reclaimacct_tsk_init(struct task_struct *tsk)
47 {
48 if (tsk)
49 tsk->reclaim_acct = NULL;
50 }
51
52 /* Reinitialize in case parent's non-null pointer was duped */
reclaimacct_init(void)53 void reclaimacct_init(void)
54 {
55 reclaimacct_tsk_init(&init_task);
56 }
57
reclaimacct_substage_start(enum reclaimacct_stubs stub)58 void reclaimacct_substage_start(enum reclaimacct_stubs stub)
59 {
60 if (!current->reclaim_acct)
61 return;
62
63 current->reclaim_acct->start[stub] = ktime_get_ns();
64 }
65
reclaimacct_substage_end(enum reclaimacct_stubs stub,unsigned long freed,const struct shrinker * shrinker)66 void reclaimacct_substage_end(enum reclaimacct_stubs stub, unsigned long freed,
67 const struct shrinker *shrinker)
68 {
69 if (!current->reclaim_acct)
70 return;
71
72 __reclaimacct_end(current->reclaim_acct, freed, stub, shrinker);
73 }
74
reclaimacct_directreclaim_end(struct reclaim_acct * ra)75 static void reclaimacct_directreclaim_end(struct reclaim_acct *ra)
76 {
77 int i;
78
79 if (ra->delay[RA_RECLAIM] > DELAY_LV4) {
80 pr_warn_ratelimited("Summary");
81 for (i = 0; i < NR_RA_STUBS; i++)
82 pr_warn_ratelimited(" %s=%llu %llu", stub_name[i],
83 ra->delay[i], ra->count[i]);
84 pr_warn_ratelimited("\n");
85 }
86
87 reclaimacct_collect_data();
88 reclaimacct_free(ra, ra->reclaim_type);
89 current->reclaim_acct = NULL;
90 }
91
reclaimacct_system_reclaim_end(struct reclaim_acct * ra)92 static void reclaimacct_system_reclaim_end(struct reclaim_acct *ra)
93 {
94 reclaimacct_free(ra, ra->reclaim_type);
95 }
96
reclaimacct_start(enum reclaim_type type,struct reclaim_acct * ra)97 void reclaimacct_start(enum reclaim_type type, struct reclaim_acct *ra)
98 {
99 if (reclaimacct_disable || reclaimacct_is_off)
100 return;
101
102 if (!current->reclaim_acct)
103 current->reclaim_acct = ra;
104
105 current->reclaim_acct->reclaim_type = type;
106 current->reclaim_acct->start[RA_RECLAIM] = ktime_get_ns();
107 }
108
reclaimacct_end(enum reclaim_type type)109 void reclaimacct_end(enum reclaim_type type)
110 {
111 if (!current->reclaim_acct)
112 return;
113
114 __reclaimacct_end(current->reclaim_acct, 0, RA_RECLAIM, NULL);
115
116 reclaimacct_collect_reclaim_efficiency();
117
118 if (is_system_reclaim(type))
119 reclaimacct_system_reclaim_end(current->reclaim_acct);
120 else
121 reclaimacct_directreclaim_end(current->reclaim_acct);
122 }
123
124 /* Reclaim accounting module initialize */
reclaimacct_init_handle(void * p)125 static int reclaimacct_init_handle(void *p)
126 {
127 int i;
128
129 if (!reclaimacct_initialize_show_data())
130 goto alloc_show_failed;
131
132 reclaimacct_is_off = false;
133 pr_info("enabled\n");
134 return 0;
135
136 alloc_show_failed:
137 reclaimacct_is_off = true;
138 pr_err("disabled\n");
139 return 0;
140 }
141
reclaimacct_module_init(void)142 static int __init reclaimacct_module_init(void)
143 {
144 struct task_struct *task = NULL;
145
146 task = kthread_run(reclaimacct_init_handle, NULL, "reclaimacct_init");
147 if (IS_ERR(task))
148 pr_err("run reclaimacct_init failed\n");
149 else
150 pr_info("run reclaimacct_init successfully\n");
151 return 0;
152 }
153
154 late_initcall(reclaimacct_module_init);
155
156 module_param_named(disable, reclaimacct_disable, int, 0644);
157