1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * CPPC (Collaborative Processor Performance Control) driver for
4 * interfacing with the CPUfreq layer and governors. See
5 * cppc_acpi.c for CPPC specific methods.
6 *
7 * (C) Copyright 2014, 2015 Linaro Ltd.
8 * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org>
9 */
10
11 #define pr_fmt(fmt) "CPPC Cpufreq:" fmt
12
13 #include <linux/arch_topology.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/delay.h>
17 #include <linux/cpu.h>
18 #include <linux/cpufreq.h>
19 #include <linux/irq_work.h>
20 #include <linux/kthread.h>
21 #include <linux/time.h>
22 #include <linux/vmalloc.h>
23 #include <uapi/linux/sched/types.h>
24
25 #include <linux/unaligned.h>
26
27 #include <acpi/cppc_acpi.h>
28
29 /*
30 * This list contains information parsed from per CPU ACPI _CPC and _PSD
31 * structures: e.g. the highest and lowest supported performance, capabilities,
32 * desired performance, level requested etc. Depending on the share_type, not
33 * all CPUs will have an entry in the list.
34 */
35 static LIST_HEAD(cpu_data_list);
36
37 static bool boost_supported;
38
39 struct cppc_workaround_oem_info {
40 char oem_id[ACPI_OEM_ID_SIZE + 1];
41 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
42 u32 oem_revision;
43 };
44
45 static struct cppc_workaround_oem_info wa_info[] = {
46 {
47 .oem_id = "HISI ",
48 .oem_table_id = "HIP07 ",
49 .oem_revision = 0,
50 }, {
51 .oem_id = "HISI ",
52 .oem_table_id = "HIP08 ",
53 .oem_revision = 0,
54 }
55 };
56
57 static struct cpufreq_driver cppc_cpufreq_driver;
58
59 static enum {
60 FIE_UNSET = -1,
61 FIE_ENABLED,
62 FIE_DISABLED
63 } fie_disabled = FIE_UNSET;
64
65 #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE
66 module_param(fie_disabled, int, 0444);
67 MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)");
68
69 /* Frequency invariance support */
70 struct cppc_freq_invariance {
71 int cpu;
72 struct irq_work irq_work;
73 struct kthread_work work;
74 struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;
75 struct cppc_cpudata *cpu_data;
76 };
77
78 static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);
79 static struct kthread_worker *kworker_fie;
80
81 static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu);
82 static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
83 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
84 struct cppc_perf_fb_ctrs *fb_ctrs_t1);
85
86 /**
87 * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance
88 * @work: The work item.
89 *
90 * The CPPC driver register itself with the topology core to provide its own
91 * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which
92 * gets called by the scheduler on every tick.
93 *
94 * Note that the arch specific counters have higher priority than CPPC counters,
95 * if available, though the CPPC driver doesn't need to have any special
96 * handling for that.
97 *
98 * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we
99 * reach here from hard-irq context), which then schedules a normal work item
100 * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable
101 * based on the counter updates since the last tick.
102 */
cppc_scale_freq_workfn(struct kthread_work * work)103 static void cppc_scale_freq_workfn(struct kthread_work *work)
104 {
105 struct cppc_freq_invariance *cppc_fi;
106 struct cppc_perf_fb_ctrs fb_ctrs = {0};
107 struct cppc_cpudata *cpu_data;
108 unsigned long local_freq_scale;
109 u64 perf;
110
111 cppc_fi = container_of(work, struct cppc_freq_invariance, work);
112 cpu_data = cppc_fi->cpu_data;
113
114 if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) {
115 pr_warn("%s: failed to read perf counters\n", __func__);
116 return;
117 }
118
119 perf = cppc_perf_from_fbctrs(cpu_data, &cppc_fi->prev_perf_fb_ctrs,
120 &fb_ctrs);
121 if (!perf)
122 return;
123
124 cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
125
126 perf <<= SCHED_CAPACITY_SHIFT;
127 local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf);
128
129 /* This can happen due to counter's overflow */
130 if (unlikely(local_freq_scale > 1024))
131 local_freq_scale = 1024;
132
133 per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale;
134 }
135
cppc_irq_work(struct irq_work * irq_work)136 static void cppc_irq_work(struct irq_work *irq_work)
137 {
138 struct cppc_freq_invariance *cppc_fi;
139
140 cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work);
141 kthread_queue_work(kworker_fie, &cppc_fi->work);
142 }
143
cppc_scale_freq_tick(void)144 static void cppc_scale_freq_tick(void)
145 {
146 struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id());
147
148 /*
149 * cppc_get_perf_ctrs() can potentially sleep, call that from the right
150 * context.
151 */
152 irq_work_queue(&cppc_fi->irq_work);
153 }
154
155 static struct scale_freq_data cppc_sftd = {
156 .source = SCALE_FREQ_SOURCE_CPPC,
157 .set_freq_scale = cppc_scale_freq_tick,
158 };
159
cppc_cpufreq_cpu_fie_init(struct cpufreq_policy * policy)160 static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
161 {
162 struct cppc_freq_invariance *cppc_fi;
163 int cpu, ret;
164
165 if (fie_disabled)
166 return;
167
168 for_each_cpu(cpu, policy->cpus) {
169 cppc_fi = &per_cpu(cppc_freq_inv, cpu);
170 cppc_fi->cpu = cpu;
171 cppc_fi->cpu_data = policy->driver_data;
172 kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn);
173 init_irq_work(&cppc_fi->irq_work, cppc_irq_work);
174
175 ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs);
176 if (ret) {
177 pr_warn("%s: failed to read perf counters for cpu:%d: %d\n",
178 __func__, cpu, ret);
179
180 /*
181 * Don't abort if the CPU was offline while the driver
182 * was getting registered.
183 */
184 if (cpu_online(cpu))
185 return;
186 }
187 }
188
189 /* Register for freq-invariance */
190 topology_set_scale_freq_source(&cppc_sftd, policy->cpus);
191 }
192
193 /*
194 * We free all the resources on policy's removal and not on CPU removal as the
195 * irq-work are per-cpu and the hotplug core takes care of flushing the pending
196 * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work
197 * fires on another CPU after the concerned CPU is removed, it won't harm.
198 *
199 * We just need to make sure to remove them all on policy->exit().
200 */
cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy * policy)201 static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
202 {
203 struct cppc_freq_invariance *cppc_fi;
204 int cpu;
205
206 if (fie_disabled)
207 return;
208
209 /* policy->cpus will be empty here, use related_cpus instead */
210 topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus);
211
212 for_each_cpu(cpu, policy->related_cpus) {
213 cppc_fi = &per_cpu(cppc_freq_inv, cpu);
214 irq_work_sync(&cppc_fi->irq_work);
215 kthread_cancel_work_sync(&cppc_fi->work);
216 }
217 }
218
cppc_freq_invariance_init(void)219 static void __init cppc_freq_invariance_init(void)
220 {
221 struct sched_attr attr = {
222 .size = sizeof(struct sched_attr),
223 .sched_policy = SCHED_DEADLINE,
224 .sched_nice = 0,
225 .sched_priority = 0,
226 /*
227 * Fake (unused) bandwidth; workaround to "fix"
228 * priority inheritance.
229 */
230 .sched_runtime = NSEC_PER_MSEC,
231 .sched_deadline = 10 * NSEC_PER_MSEC,
232 .sched_period = 10 * NSEC_PER_MSEC,
233 };
234 int ret;
235
236 if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) {
237 fie_disabled = FIE_ENABLED;
238 if (cppc_perf_ctrs_in_pcc()) {
239 pr_info("FIE not enabled on systems with registers in PCC\n");
240 fie_disabled = FIE_DISABLED;
241 }
242 }
243
244 if (fie_disabled)
245 return;
246
247 kworker_fie = kthread_create_worker(0, "cppc_fie");
248 if (IS_ERR(kworker_fie)) {
249 pr_warn("%s: failed to create kworker_fie: %ld\n", __func__,
250 PTR_ERR(kworker_fie));
251 fie_disabled = FIE_DISABLED;
252 return;
253 }
254
255 ret = sched_setattr_nocheck(kworker_fie->task, &attr);
256 if (ret) {
257 pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__,
258 ret);
259 kthread_destroy_worker(kworker_fie);
260 fie_disabled = FIE_DISABLED;
261 }
262 }
263
cppc_freq_invariance_exit(void)264 static void cppc_freq_invariance_exit(void)
265 {
266 if (fie_disabled)
267 return;
268
269 kthread_destroy_worker(kworker_fie);
270 }
271
272 #else
cppc_cpufreq_cpu_fie_init(struct cpufreq_policy * policy)273 static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
274 {
275 }
276
cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy * policy)277 static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
278 {
279 }
280
cppc_freq_invariance_init(void)281 static inline void cppc_freq_invariance_init(void)
282 {
283 }
284
cppc_freq_invariance_exit(void)285 static inline void cppc_freq_invariance_exit(void)
286 {
287 }
288 #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */
289
cppc_cpufreq_set_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)290 static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
291 unsigned int target_freq,
292 unsigned int relation)
293 {
294 struct cppc_cpudata *cpu_data = policy->driver_data;
295 unsigned int cpu = policy->cpu;
296 struct cpufreq_freqs freqs;
297 int ret = 0;
298
299 cpu_data->perf_ctrls.desired_perf =
300 cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
301 freqs.old = policy->cur;
302 freqs.new = target_freq;
303
304 cpufreq_freq_transition_begin(policy, &freqs);
305 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
306 cpufreq_freq_transition_end(policy, &freqs, ret != 0);
307
308 if (ret)
309 pr_debug("Failed to set target on CPU:%d. ret:%d\n",
310 cpu, ret);
311
312 return ret;
313 }
314
cppc_cpufreq_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)315 static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy,
316 unsigned int target_freq)
317 {
318 struct cppc_cpudata *cpu_data = policy->driver_data;
319 unsigned int cpu = policy->cpu;
320 u32 desired_perf;
321 int ret;
322
323 desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
324 cpu_data->perf_ctrls.desired_perf = desired_perf;
325 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
326
327 if (ret) {
328 pr_debug("Failed to set target on CPU:%d. ret:%d\n",
329 cpu, ret);
330 return 0;
331 }
332
333 return target_freq;
334 }
335
cppc_verify_policy(struct cpufreq_policy_data * policy)336 static int cppc_verify_policy(struct cpufreq_policy_data *policy)
337 {
338 cpufreq_verify_within_cpu_limits(policy);
339 return 0;
340 }
341
342 /*
343 * The PCC subspace describes the rate at which platform can accept commands
344 * on the shared PCC channel (including READs which do not count towards freq
345 * transition requests), so ideally we need to use the PCC values as a fallback
346 * if we don't have a platform specific transition_delay_us
347 */
348 #ifdef CONFIG_ARM64
349 #include <asm/cputype.h>
350
cppc_cpufreq_get_transition_delay_us(unsigned int cpu)351 static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
352 {
353 unsigned long implementor = read_cpuid_implementor();
354 unsigned long part_num = read_cpuid_part_number();
355
356 switch (implementor) {
357 case ARM_CPU_IMP_QCOM:
358 switch (part_num) {
359 case QCOM_CPU_PART_FALKOR_V1:
360 case QCOM_CPU_PART_FALKOR:
361 return 10000;
362 }
363 }
364 return cppc_get_transition_latency(cpu) / NSEC_PER_USEC;
365 }
366 #else
cppc_cpufreq_get_transition_delay_us(unsigned int cpu)367 static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
368 {
369 return cppc_get_transition_latency(cpu) / NSEC_PER_USEC;
370 }
371 #endif
372
373 #if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL)
374
375 static DEFINE_PER_CPU(unsigned int, efficiency_class);
376 static void cppc_cpufreq_register_em(struct cpufreq_policy *policy);
377
378 /* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */
379 #define CPPC_EM_CAP_STEP (20)
380 /* Increase the cost value by CPPC_EM_COST_STEP every performance state. */
381 #define CPPC_EM_COST_STEP (1)
382 /* Add a cost gap correspnding to the energy of 4 CPUs. */
383 #define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \
384 / CPPC_EM_CAP_STEP)
385
get_perf_level_count(struct cpufreq_policy * policy)386 static unsigned int get_perf_level_count(struct cpufreq_policy *policy)
387 {
388 struct cppc_perf_caps *perf_caps;
389 unsigned int min_cap, max_cap;
390 struct cppc_cpudata *cpu_data;
391 int cpu = policy->cpu;
392
393 cpu_data = policy->driver_data;
394 perf_caps = &cpu_data->perf_caps;
395 max_cap = arch_scale_cpu_capacity(cpu);
396 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
397 perf_caps->highest_perf);
398 if ((min_cap == 0) || (max_cap < min_cap))
399 return 0;
400 return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP;
401 }
402
403 /*
404 * The cost is defined as:
405 * cost = power * max_frequency / frequency
406 */
compute_cost(int cpu,int step)407 static inline unsigned long compute_cost(int cpu, int step)
408 {
409 return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) +
410 step * CPPC_EM_COST_STEP;
411 }
412
cppc_get_cpu_power(struct device * cpu_dev,unsigned long * power,unsigned long * KHz)413 static int cppc_get_cpu_power(struct device *cpu_dev,
414 unsigned long *power, unsigned long *KHz)
415 {
416 unsigned long perf_step, perf_prev, perf, perf_check;
417 unsigned int min_step, max_step, step, step_check;
418 unsigned long prev_freq = *KHz;
419 unsigned int min_cap, max_cap;
420 struct cpufreq_policy *policy;
421
422 struct cppc_perf_caps *perf_caps;
423 struct cppc_cpudata *cpu_data;
424
425 policy = cpufreq_cpu_get_raw(cpu_dev->id);
426 if (!policy)
427 return -EINVAL;
428
429 cpu_data = policy->driver_data;
430 perf_caps = &cpu_data->perf_caps;
431 max_cap = arch_scale_cpu_capacity(cpu_dev->id);
432 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
433 perf_caps->highest_perf);
434 perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf,
435 max_cap);
436 min_step = min_cap / CPPC_EM_CAP_STEP;
437 max_step = max_cap / CPPC_EM_CAP_STEP;
438
439 perf_prev = cppc_khz_to_perf(perf_caps, *KHz);
440 step = perf_prev / perf_step;
441
442 if (step > max_step)
443 return -EINVAL;
444
445 if (min_step == max_step) {
446 step = max_step;
447 perf = perf_caps->highest_perf;
448 } else if (step < min_step) {
449 step = min_step;
450 perf = perf_caps->lowest_perf;
451 } else {
452 step++;
453 if (step == max_step)
454 perf = perf_caps->highest_perf;
455 else
456 perf = step * perf_step;
457 }
458
459 *KHz = cppc_perf_to_khz(perf_caps, perf);
460 perf_check = cppc_khz_to_perf(perf_caps, *KHz);
461 step_check = perf_check / perf_step;
462
463 /*
464 * To avoid bad integer approximation, check that new frequency value
465 * increased and that the new frequency will be converted to the
466 * desired step value.
467 */
468 while ((*KHz == prev_freq) || (step_check != step)) {
469 perf++;
470 *KHz = cppc_perf_to_khz(perf_caps, perf);
471 perf_check = cppc_khz_to_perf(perf_caps, *KHz);
472 step_check = perf_check / perf_step;
473 }
474
475 /*
476 * With an artificial EM, only the cost value is used. Still the power
477 * is populated such as 0 < power < EM_MAX_POWER. This allows to add
478 * more sense to the artificial performance states.
479 */
480 *power = compute_cost(cpu_dev->id, step);
481
482 return 0;
483 }
484
cppc_get_cpu_cost(struct device * cpu_dev,unsigned long KHz,unsigned long * cost)485 static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz,
486 unsigned long *cost)
487 {
488 unsigned long perf_step, perf_prev;
489 struct cppc_perf_caps *perf_caps;
490 struct cpufreq_policy *policy;
491 struct cppc_cpudata *cpu_data;
492 unsigned int max_cap;
493 int step;
494
495 policy = cpufreq_cpu_get_raw(cpu_dev->id);
496 if (!policy)
497 return -EINVAL;
498
499 cpu_data = policy->driver_data;
500 perf_caps = &cpu_data->perf_caps;
501 max_cap = arch_scale_cpu_capacity(cpu_dev->id);
502
503 perf_prev = cppc_khz_to_perf(perf_caps, KHz);
504 perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap;
505 step = perf_prev / perf_step;
506
507 *cost = compute_cost(cpu_dev->id, step);
508
509 return 0;
510 }
511
populate_efficiency_class(void)512 static int populate_efficiency_class(void)
513 {
514 struct acpi_madt_generic_interrupt *gicc;
515 DECLARE_BITMAP(used_classes, 256) = {};
516 int class, cpu, index;
517
518 for_each_possible_cpu(cpu) {
519 gicc = acpi_cpu_get_madt_gicc(cpu);
520 class = gicc->efficiency_class;
521 bitmap_set(used_classes, class, 1);
522 }
523
524 if (bitmap_weight(used_classes, 256) <= 1) {
525 pr_debug("Efficiency classes are all equal (=%d). "
526 "No EM registered", class);
527 return -EINVAL;
528 }
529
530 /*
531 * Squeeze efficiency class values on [0:#efficiency_class-1].
532 * Values are per spec in [0:255].
533 */
534 index = 0;
535 for_each_set_bit(class, used_classes, 256) {
536 for_each_possible_cpu(cpu) {
537 gicc = acpi_cpu_get_madt_gicc(cpu);
538 if (gicc->efficiency_class == class)
539 per_cpu(efficiency_class, cpu) = index;
540 }
541 index++;
542 }
543 cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em;
544
545 return 0;
546 }
547
cppc_cpufreq_register_em(struct cpufreq_policy * policy)548 static void cppc_cpufreq_register_em(struct cpufreq_policy *policy)
549 {
550 struct cppc_cpudata *cpu_data;
551 struct em_data_callback em_cb =
552 EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost);
553
554 cpu_data = policy->driver_data;
555 em_dev_register_perf_domain(get_cpu_device(policy->cpu),
556 get_perf_level_count(policy), &em_cb,
557 cpu_data->shared_cpu_map, 0);
558 }
559
560 #else
populate_efficiency_class(void)561 static int populate_efficiency_class(void)
562 {
563 return 0;
564 }
565 #endif
566
cppc_cpufreq_get_cpu_data(unsigned int cpu)567 static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu)
568 {
569 struct cppc_cpudata *cpu_data;
570 int ret;
571
572 cpu_data = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL);
573 if (!cpu_data)
574 goto out;
575
576 if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL))
577 goto free_cpu;
578
579 ret = acpi_get_psd_map(cpu, cpu_data);
580 if (ret) {
581 pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret);
582 goto free_mask;
583 }
584
585 ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps);
586 if (ret) {
587 pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret);
588 goto free_mask;
589 }
590
591 list_add(&cpu_data->node, &cpu_data_list);
592
593 return cpu_data;
594
595 free_mask:
596 free_cpumask_var(cpu_data->shared_cpu_map);
597 free_cpu:
598 kfree(cpu_data);
599 out:
600 return NULL;
601 }
602
cppc_cpufreq_put_cpu_data(struct cpufreq_policy * policy)603 static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy)
604 {
605 struct cppc_cpudata *cpu_data = policy->driver_data;
606
607 list_del(&cpu_data->node);
608 free_cpumask_var(cpu_data->shared_cpu_map);
609 kfree(cpu_data);
610 policy->driver_data = NULL;
611 }
612
cppc_cpufreq_cpu_init(struct cpufreq_policy * policy)613 static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
614 {
615 unsigned int cpu = policy->cpu;
616 struct cppc_cpudata *cpu_data;
617 struct cppc_perf_caps *caps;
618 int ret;
619
620 cpu_data = cppc_cpufreq_get_cpu_data(cpu);
621 if (!cpu_data) {
622 pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu);
623 return -ENODEV;
624 }
625 caps = &cpu_data->perf_caps;
626 policy->driver_data = cpu_data;
627
628 /*
629 * Set min to lowest nonlinear perf to avoid any efficiency penalty (see
630 * Section 8.4.7.1.1.5 of ACPI 6.1 spec)
631 */
632 policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf);
633 policy->max = cppc_perf_to_khz(caps, caps->nominal_perf);
634
635 /*
636 * Set cpuinfo.min_freq to Lowest to make the full range of performance
637 * available if userspace wants to use any perf between lowest & lowest
638 * nonlinear perf
639 */
640 policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf);
641 policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->nominal_perf);
642
643 policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu);
644 policy->shared_type = cpu_data->shared_type;
645
646 switch (policy->shared_type) {
647 case CPUFREQ_SHARED_TYPE_HW:
648 case CPUFREQ_SHARED_TYPE_NONE:
649 /* Nothing to be done - we'll have a policy for each CPU */
650 break;
651 case CPUFREQ_SHARED_TYPE_ANY:
652 /*
653 * All CPUs in the domain will share a policy and all cpufreq
654 * operations will use a single cppc_cpudata structure stored
655 * in policy->driver_data.
656 */
657 cpumask_copy(policy->cpus, cpu_data->shared_cpu_map);
658 break;
659 default:
660 pr_debug("Unsupported CPU co-ord type: %d\n",
661 policy->shared_type);
662 ret = -EFAULT;
663 goto out;
664 }
665
666 policy->fast_switch_possible = cppc_allow_fast_switch();
667 policy->dvfs_possible_from_any_cpu = true;
668
669 /*
670 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost
671 * is supported.
672 */
673 if (caps->highest_perf > caps->nominal_perf)
674 boost_supported = true;
675
676 /* Set policy->cur to max now. The governors will adjust later. */
677 policy->cur = cppc_perf_to_khz(caps, caps->highest_perf);
678 cpu_data->perf_ctrls.desired_perf = caps->highest_perf;
679
680 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
681 if (ret) {
682 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
683 caps->highest_perf, cpu, ret);
684 goto out;
685 }
686
687 cppc_cpufreq_cpu_fie_init(policy);
688 return 0;
689
690 out:
691 cppc_cpufreq_put_cpu_data(policy);
692 return ret;
693 }
694
cppc_cpufreq_cpu_exit(struct cpufreq_policy * policy)695 static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy)
696 {
697 struct cppc_cpudata *cpu_data = policy->driver_data;
698 struct cppc_perf_caps *caps = &cpu_data->perf_caps;
699 unsigned int cpu = policy->cpu;
700 int ret;
701
702 cppc_cpufreq_cpu_fie_exit(policy);
703
704 cpu_data->perf_ctrls.desired_perf = caps->lowest_perf;
705
706 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
707 if (ret)
708 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
709 caps->lowest_perf, cpu, ret);
710
711 cppc_cpufreq_put_cpu_data(policy);
712 }
713
get_delta(u64 t1,u64 t0)714 static inline u64 get_delta(u64 t1, u64 t0)
715 {
716 if (t1 > t0 || t0 > ~(u32)0)
717 return t1 - t0;
718
719 return (u32)t1 - (u32)t0;
720 }
721
cppc_perf_from_fbctrs(struct cppc_cpudata * cpu_data,struct cppc_perf_fb_ctrs * fb_ctrs_t0,struct cppc_perf_fb_ctrs * fb_ctrs_t1)722 static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
723 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
724 struct cppc_perf_fb_ctrs *fb_ctrs_t1)
725 {
726 u64 delta_reference, delta_delivered;
727 u64 reference_perf;
728
729 reference_perf = fb_ctrs_t0->reference_perf;
730
731 delta_reference = get_delta(fb_ctrs_t1->reference,
732 fb_ctrs_t0->reference);
733 delta_delivered = get_delta(fb_ctrs_t1->delivered,
734 fb_ctrs_t0->delivered);
735
736 /*
737 * Avoid divide-by zero and unchanged feedback counters.
738 * Leave it for callers to handle.
739 */
740 if (!delta_reference || !delta_delivered)
741 return 0;
742
743 return (reference_perf * delta_delivered) / delta_reference;
744 }
745
cppc_get_perf_ctrs_sample(int cpu,struct cppc_perf_fb_ctrs * fb_ctrs_t0,struct cppc_perf_fb_ctrs * fb_ctrs_t1)746 static int cppc_get_perf_ctrs_sample(int cpu,
747 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
748 struct cppc_perf_fb_ctrs *fb_ctrs_t1)
749 {
750 int ret;
751
752 ret = cppc_get_perf_ctrs(cpu, fb_ctrs_t0);
753 if (ret)
754 return ret;
755
756 udelay(2); /* 2usec delay between sampling */
757
758 return cppc_get_perf_ctrs(cpu, fb_ctrs_t1);
759 }
760
cppc_cpufreq_get_rate(unsigned int cpu)761 static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)
762 {
763 struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0};
764 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
765 struct cppc_cpudata *cpu_data;
766 u64 delivered_perf;
767 int ret;
768
769 if (!policy)
770 return 0;
771
772 cpu_data = policy->driver_data;
773
774 cpufreq_cpu_put(policy);
775
776 ret = cppc_get_perf_ctrs_sample(cpu, &fb_ctrs_t0, &fb_ctrs_t1);
777 if (ret) {
778 if (ret == -EFAULT)
779 /* Any of the associated CPPC regs is 0. */
780 goto out_invalid_counters;
781 else
782 return 0;
783 }
784
785 delivered_perf = cppc_perf_from_fbctrs(cpu_data, &fb_ctrs_t0,
786 &fb_ctrs_t1);
787 if (!delivered_perf)
788 goto out_invalid_counters;
789
790 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf);
791
792 out_invalid_counters:
793 /*
794 * Feedback counters could be unchanged or 0 when a cpu enters a
795 * low-power idle state, e.g. clock-gated or power-gated.
796 * Use desired perf for reflecting frequency. Get the latest register
797 * value first as some platforms may update the actual delivered perf
798 * there; if failed, resort to the cached desired perf.
799 */
800 if (cppc_get_desired_perf(cpu, &delivered_perf))
801 delivered_perf = cpu_data->perf_ctrls.desired_perf;
802
803 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf);
804 }
805
cppc_cpufreq_set_boost(struct cpufreq_policy * policy,int state)806 static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
807 {
808 struct cppc_cpudata *cpu_data = policy->driver_data;
809 struct cppc_perf_caps *caps = &cpu_data->perf_caps;
810 int ret;
811
812 if (!boost_supported) {
813 pr_err("BOOST not supported by CPU or firmware\n");
814 return -EINVAL;
815 }
816
817 if (state)
818 policy->max = cppc_perf_to_khz(caps, caps->highest_perf);
819 else
820 policy->max = cppc_perf_to_khz(caps, caps->nominal_perf);
821 policy->cpuinfo.max_freq = policy->max;
822
823 ret = freq_qos_update_request(policy->max_freq_req, policy->max);
824 if (ret < 0)
825 return ret;
826
827 return 0;
828 }
829
show_freqdomain_cpus(struct cpufreq_policy * policy,char * buf)830 static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)
831 {
832 struct cppc_cpudata *cpu_data = policy->driver_data;
833
834 return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf);
835 }
836 cpufreq_freq_attr_ro(freqdomain_cpus);
837
838 static struct freq_attr *cppc_cpufreq_attr[] = {
839 &freqdomain_cpus,
840 NULL,
841 };
842
843 static struct cpufreq_driver cppc_cpufreq_driver = {
844 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
845 .verify = cppc_verify_policy,
846 .target = cppc_cpufreq_set_target,
847 .get = cppc_cpufreq_get_rate,
848 .fast_switch = cppc_cpufreq_fast_switch,
849 .init = cppc_cpufreq_cpu_init,
850 .exit = cppc_cpufreq_cpu_exit,
851 .set_boost = cppc_cpufreq_set_boost,
852 .attr = cppc_cpufreq_attr,
853 .name = "cppc_cpufreq",
854 };
855
856 /*
857 * HISI platform does not support delivered performance counter and
858 * reference performance counter. It can calculate the performance using the
859 * platform specific mechanism. We reuse the desired performance register to
860 * store the real performance calculated by the platform.
861 */
hisi_cppc_cpufreq_get_rate(unsigned int cpu)862 static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu)
863 {
864 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
865 struct cppc_cpudata *cpu_data;
866 u64 desired_perf;
867 int ret;
868
869 if (!policy)
870 return -ENODEV;
871
872 cpu_data = policy->driver_data;
873
874 cpufreq_cpu_put(policy);
875
876 ret = cppc_get_desired_perf(cpu, &desired_perf);
877 if (ret < 0)
878 return -EIO;
879
880 return cppc_perf_to_khz(&cpu_data->perf_caps, desired_perf);
881 }
882
cppc_check_hisi_workaround(void)883 static void cppc_check_hisi_workaround(void)
884 {
885 struct acpi_table_header *tbl;
886 acpi_status status = AE_OK;
887 int i;
888
889 status = acpi_get_table(ACPI_SIG_PCCT, 0, &tbl);
890 if (ACPI_FAILURE(status) || !tbl)
891 return;
892
893 for (i = 0; i < ARRAY_SIZE(wa_info); i++) {
894 if (!memcmp(wa_info[i].oem_id, tbl->oem_id, ACPI_OEM_ID_SIZE) &&
895 !memcmp(wa_info[i].oem_table_id, tbl->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
896 wa_info[i].oem_revision == tbl->oem_revision) {
897 /* Overwrite the get() callback */
898 cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate;
899 fie_disabled = FIE_DISABLED;
900 break;
901 }
902 }
903
904 acpi_put_table(tbl);
905 }
906
cppc_cpufreq_init(void)907 static int __init cppc_cpufreq_init(void)
908 {
909 int ret;
910
911 if (!acpi_cpc_valid())
912 return -ENODEV;
913
914 cppc_check_hisi_workaround();
915 cppc_freq_invariance_init();
916 populate_efficiency_class();
917
918 ret = cpufreq_register_driver(&cppc_cpufreq_driver);
919 if (ret)
920 cppc_freq_invariance_exit();
921
922 return ret;
923 }
924
free_cpu_data(void)925 static inline void free_cpu_data(void)
926 {
927 struct cppc_cpudata *iter, *tmp;
928
929 list_for_each_entry_safe(iter, tmp, &cpu_data_list, node) {
930 free_cpumask_var(iter->shared_cpu_map);
931 list_del(&iter->node);
932 kfree(iter);
933 }
934
935 }
936
cppc_cpufreq_exit(void)937 static void __exit cppc_cpufreq_exit(void)
938 {
939 cpufreq_unregister_driver(&cppc_cpufreq_driver);
940 cppc_freq_invariance_exit();
941
942 free_cpu_data();
943 }
944
945 module_exit(cppc_cpufreq_exit);
946 MODULE_AUTHOR("Ashwin Chaugule");
947 MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec");
948 MODULE_LICENSE("GPL");
949
950 late_initcall(cppc_cpufreq_init);
951
952 static const struct acpi_device_id cppc_acpi_ids[] __used = {
953 {ACPI_PROCESSOR_DEVICE_HID, },
954 {}
955 };
956
957 MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids);
958