1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Copyright (c) 2021 SUSE LLC <rpalethorpe@suse.com> */
3 /*\
4 * [Description]
5 *
6 * Check that something (e.g. irqbalance daemon) is performing IRQ
7 * load balancing.
8 *
9 * On many systems userland needs to set /proc/irq/$IRQ/smp_affinity
10 * to prevent many IRQs being delivered to the same CPU.
11 *
12 * Note some drivers and IRQ controllers will distribute IRQs
13 * evenly. Some systems will have housekeeping CPUs configured. Some
14 * IRQs can not be masked etc. So this test is not appropriate for all
15 * scenarios.
16 *
17 * Furthermore, exactly how IRQs should be distributed is a
18 * performance and/or security issue. This is only a generic smoke
19 * test. It will hopefully detect misconfigured systems and total
20 * balancing failures which are often silent errors.
21 *
22 * Heuristic: Evidence of Change
23 *
24 * 1. Find IRQs with a non-zero count
25 * 2. Check if they are now disallowed
26 *
27 * There are two sources of information we need to parse:
28 *
29 * 1. /proc/interrupts
30 * 2. /proc/irq/$IRQ/smp_affinity
31 *
32 * We get the active IRQs and CPUs from /proc/interrupts. It also
33 * contains the per-CPU IRQ counts and info we do not care about.
34 *
35 * We get the IRQ masks from each active IRQ's smp_affinity file. This
36 * is a bitmask written out in hexadecimal format. It shows which CPUs
37 * an IRQ may be received by.
38 */
39
40 #include <stdlib.h>
41
42 #include "tst_test.h"
43 #include "tst_safe_stdio.h"
44 #include "tst_safe_file_at.h"
45
46 enum affinity {
47 ALLOW = '+',
48 DENY = '-',
49 };
50
51 static unsigned int *irq_stats;
52 static enum affinity *irq_affinity;
53
54 static unsigned int nr_cpus;
55 static unsigned int nr_irqs;
56 static unsigned int *irq_ids;
57
read_proc_file(const char * const path,size_t * const len_out)58 static char *read_proc_file(const char *const path, size_t *const len_out)
59 {
60 const size_t pg_len = SAFE_SYSCONF(_SC_PAGESIZE);
61 int fd = SAFE_OPEN(path, O_RDONLY);
62 size_t ret = 0, used_len = 0;
63 static size_t total_len;
64 static char *buf;
65
66 do {
67 if (used_len + 1 >= total_len) {
68 total_len += pg_len;
69 buf = SAFE_REALLOC(buf, total_len);
70 }
71
72 ret = SAFE_READ(0, fd,
73 buf + used_len,
74 total_len - used_len - 1);
75 used_len += ret;
76 } while (ret);
77
78 if (!used_len)
79 tst_brk(TBROK, "Empty %s?", path);
80
81 buf[used_len] = '\0';
82
83 SAFE_CLOSE(fd);
84
85 if (len_out)
86 *len_out = used_len;
87 return buf;
88 }
89
collect_irq_info(void)90 static void collect_irq_info(void)
91 {
92 char *buf, *c, *first_row;
93 char path[PATH_MAX];
94 size_t row, col, len;
95 long acc;
96 unsigned int cpu_total, bit;
97
98 nr_cpus = 0;
99 nr_irqs = 0;
100
101 buf = read_proc_file("/proc/interrupts", NULL);
102
103 /* Count CPUs, header columns are like /CPU[0-9]+/ */
104 for (c = buf; *c != '\0' && *c != '\n'; c++) {
105 if (!strncmp(c, "CPU", 3))
106 nr_cpus++;
107 }
108
109 c++;
110 first_row = c;
111 /* Count IRQs, real IRQs start with /[0-9]+:/ */
112 while (*c != '\0') {
113 switch (*c) {
114 case ' ':
115 case '\t':
116 case '\n':
117 case '0' ... '9':
118 c++;
119 break;
120 case ':':
121 nr_irqs++;
122 /* fall-through */
123 default:
124 while (*c != '\n' && *c != '\0')
125 c++;
126 }
127 }
128
129 tst_res(TINFO, "Found %u CPUS, %u IRQs", nr_cpus, nr_irqs);
130
131 irq_ids = SAFE_REALLOC(irq_ids, nr_irqs * sizeof(*irq_ids));
132 irq_stats = SAFE_REALLOC(irq_stats,
133 nr_cpus * (nr_irqs + 1) * sizeof(*irq_stats));
134 irq_affinity = SAFE_REALLOC(irq_affinity,
135 nr_cpus * nr_irqs * sizeof(*irq_affinity));
136
137 c = first_row;
138 acc = -1;
139 row = col = 0;
140 /* Parse columns containing IRQ counts and IRQ IDs into acc. Ignore
141 * everything else.
142 */
143 while (*c != '\0') {
144 switch (*c) {
145 case ' ':
146 case '\t':
147 if (acc >= 0) {
148 irq_stats[row * nr_cpus + col] = acc;
149 acc = -1;
150 col++;
151 }
152 break;
153 case '\n':
154 if (acc != -1)
155 tst_brk(TBROK, "Unexpected EOL");
156 col = 0;
157 row++;
158 break;
159 case '0' ... '9':
160 if (acc == -1)
161 acc = 0;
162
163 acc *= 10;
164 acc += *c - '0';
165 break;
166 case ':':
167 if (acc == -1 || col != 0)
168 tst_brk(TBROK, "Unexpected ':'");
169 irq_ids[row] = acc;
170 acc = -1;
171 break;
172 default:
173 acc = -1;
174 while (*c != '\n' && *c != '\0')
175 c++;
176 continue;
177 }
178
179 c++;
180 }
181
182 for (col = 0; col < nr_cpus; col++) {
183 cpu_total = 0;
184
185 for (row = 0; row < nr_irqs; row++)
186 cpu_total += irq_stats[row * nr_cpus + col];
187
188 irq_stats[row * nr_cpus + col] = cpu_total;
189 }
190
191 /* Read the CPU affinity masks for each IRQ. The first CPU is in the
192 * right most (least significant) bit. See bitmap_string() in the kernel
193 * (%*pb)
194 */
195 for (row = 0; row < nr_irqs; row++) {
196 sprintf(path, "/proc/irq/%u/smp_affinity", irq_ids[row]);
197 buf = read_proc_file(path, &len);
198 c = buf + len;
199 col = 0;
200
201 while (--c >= buf) {
202 if (col > nr_cpus) {
203 tst_res(TINFO, "%u/smp_affnity: %s",
204 irq_ids[row], buf);
205 tst_brk(TBROK, "More mask char bits than cpus");
206 }
207
208 switch (*c) {
209 case '\n':
210 case ' ':
211 case ',':
212 continue;
213 case '0' ... '9':
214 acc = *c - '0';
215 break;
216 case 'a' ... 'f':
217 acc = 10 + *c - 'a';
218 break;
219 default:
220 tst_res(TINFO, "%u/smp_affnity: %s",
221 irq_ids[row], buf);
222 tst_brk(TBROK, "Wasn't expecting 0x%02x", *c);
223 }
224
225 for (bit = 0; bit < 4 && col < nr_cpus; bit++) {
226 irq_affinity[row * nr_cpus + col++] =
227 (acc & (1 << bit)) ? ALLOW : DENY;
228 }
229 }
230
231 if (col < nr_cpus) {
232 tst_res(TINFO, "%u/smp_affnity: %s", irq_ids[row], buf);
233 tst_brk(TBROK, "Only found %zu cpus", col);
234 }
235 }
236 }
237
print_irq_info(void)238 static void print_irq_info(void)
239 {
240 size_t row, col;
241 unsigned int count;
242 enum affinity aff;
243
244 tst_printf(" IRQ ");
245 for (col = 0; col < nr_cpus; col++)
246 tst_printf("CPU%-8zu", col);
247
248 tst_printf("\n");
249
250 for (row = 0; row < nr_irqs; row++) {
251 tst_printf("%5u:", irq_ids[row]);
252
253 for (col = 0; col < nr_cpus; col++) {
254 count = irq_stats[row * nr_cpus + col];
255 aff = irq_affinity[row * nr_cpus + col];
256
257 tst_printf("%10u%c", count, aff);
258 }
259
260 tst_printf("\n");
261 }
262
263 tst_printf("Total:");
264
265 for (col = 0; col < nr_cpus; col++)
266 tst_printf("%10u ", irq_stats[row * nr_cpus + col]);
267
268 tst_printf("\n");
269 }
270
evidence_of_change(void)271 static void evidence_of_change(void)
272 {
273 size_t row, col, changed = 0;
274
275 for (row = 0; row < nr_irqs; row++) {
276 for (col = 0; col < nr_cpus; col++) {
277 if (!irq_stats[row * nr_cpus + col])
278 continue;
279
280 if (irq_affinity[row * nr_cpus + col] == ALLOW)
281 continue;
282
283 changed++;
284 }
285 }
286
287 tst_res(changed ? TPASS : TFAIL,
288 "Heuristic: Detected %zu irq-cpu pairs have been dissallowed",
289 changed);
290 }
291
setup(void)292 static void setup(void)
293 {
294 collect_irq_info();
295 print_irq_info();
296
297 if (nr_cpus < 1)
298 tst_brk(TBROK, "No CPUs found in /proc/interrupts?");
299
300 if (nr_irqs < 1)
301 tst_brk(TBROK, "No IRQs found in /proc/interrupts?");
302 }
303
run(void)304 static void run(void)
305 {
306 collect_irq_info();
307
308 evidence_of_change();
309 }
310
311 static struct tst_test test = {
312 .test_all = run,
313 .setup = setup,
314 .min_cpus = 2,
315 };
316