1 /*
2 * POWERNV cpufreq driver for the IBM POWER processors
3 *
4 * (C) Copyright IBM 2014
5 *
6 * Author: Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 */
19
20 #define pr_fmt(fmt) "powernv-cpufreq: " fmt
21
22 #include <linux/kernel.h>
23 #include <linux/sysfs.h>
24 #include <linux/cpumask.h>
25 #include <linux/module.h>
26 #include <linux/cpufreq.h>
27 #include <linux/smp.h>
28 #include <linux/of.h>
29 #include <linux/reboot.h>
30 #include <linux/slab.h>
31 #include <linux/cpu.h>
32 #include <trace/events/power.h>
33
34 #include <asm/cputhreads.h>
35 #include <asm/firmware.h>
36 #include <asm/reg.h>
37 #include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */
38 #include <asm/opal.h>
39 #include <linux/timer.h>
40
41 #define POWERNV_MAX_PSTATES 256
42 #define PMSR_PSAFE_ENABLE (1UL << 30)
43 #define PMSR_SPR_EM_DISABLE (1UL << 31)
44 #define MAX_PSTATE_SHIFT 32
45 #define LPSTATE_SHIFT 48
46 #define GPSTATE_SHIFT 56
47
48 #define MAX_RAMP_DOWN_TIME 5120
49 /*
50 * On an idle system we want the global pstate to ramp-down from max value to
51 * min over a span of ~5 secs. Also we want it to initially ramp-down slowly and
52 * then ramp-down rapidly later on.
53 *
54 * This gives a percentage rampdown for time elapsed in milliseconds.
55 * ramp_down_percentage = ((ms * ms) >> 18)
56 * ~= 3.8 * (sec * sec)
57 *
58 * At 0 ms ramp_down_percent = 0
59 * At 5120 ms ramp_down_percent = 100
60 */
61 #define ramp_down_percent(time) ((time * time) >> 18)
62
63 /* Interval after which the timer is queued to bring down global pstate */
64 #define GPSTATE_TIMER_INTERVAL 2000
65
66 /**
67 * struct global_pstate_info - Per policy data structure to maintain history of
68 * global pstates
69 * @highest_lpstate_idx: The local pstate index from which we are
70 * ramping down
71 * @elapsed_time: Time in ms spent in ramping down from
72 * highest_lpstate_idx
73 * @last_sampled_time: Time from boot in ms when global pstates were
74 * last set
75 * @last_lpstate_idx, Last set value of local pstate and global
76 * last_gpstate_idx pstate in terms of cpufreq table index
77 * @timer: Is used for ramping down if cpu goes idle for
78 * a long time with global pstate held high
79 * @gpstate_lock: A spinlock to maintain synchronization between
80 * routines called by the timer handler and
81 * governer's target_index calls
82 */
83 struct global_pstate_info {
84 int highest_lpstate_idx;
85 unsigned int elapsed_time;
86 unsigned int last_sampled_time;
87 int last_lpstate_idx;
88 int last_gpstate_idx;
89 spinlock_t gpstate_lock;
90 struct timer_list timer;
91 };
92
93 static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1];
94 u32 pstate_sign_prefix;
95 static bool rebooting, throttled, occ_reset;
96
97 static const char * const throttle_reason[] = {
98 "No throttling",
99 "Power Cap",
100 "Processor Over Temperature",
101 "Power Supply Failure",
102 "Over Current",
103 "OCC Reset"
104 };
105
106 enum throttle_reason_type {
107 NO_THROTTLE = 0,
108 POWERCAP,
109 CPU_OVERTEMP,
110 POWER_SUPPLY_FAILURE,
111 OVERCURRENT,
112 OCC_RESET_THROTTLE,
113 OCC_MAX_REASON
114 };
115
116 static struct chip {
117 unsigned int id;
118 bool throttled;
119 bool restore;
120 u8 throttle_reason;
121 cpumask_t mask;
122 struct work_struct throttle;
123 int throttle_turbo;
124 int throttle_sub_turbo;
125 int reason[OCC_MAX_REASON];
126 } *chips;
127
128 static int nr_chips;
129 static DEFINE_PER_CPU(struct chip *, chip_info);
130
131 /*
132 * Note:
133 * The set of pstates consists of contiguous integers.
134 * powernv_pstate_info stores the index of the frequency table for
135 * max, min and nominal frequencies. It also stores number of
136 * available frequencies.
137 *
138 * powernv_pstate_info.nominal indicates the index to the highest
139 * non-turbo frequency.
140 */
141 static struct powernv_pstate_info {
142 unsigned int min;
143 unsigned int max;
144 unsigned int nominal;
145 unsigned int nr_pstates;
146 bool wof_enabled;
147 } powernv_pstate_info;
148
extract_pstate(u64 pmsr_val,unsigned int shift)149 static inline int extract_pstate(u64 pmsr_val, unsigned int shift)
150 {
151 int ret = ((pmsr_val >> shift) & 0xFF);
152
153 if (!ret)
154 return ret;
155
156 return (pstate_sign_prefix | ret);
157 }
158
159 #define extract_local_pstate(x) extract_pstate(x, LPSTATE_SHIFT)
160 #define extract_global_pstate(x) extract_pstate(x, GPSTATE_SHIFT)
161 #define extract_max_pstate(x) extract_pstate(x, MAX_PSTATE_SHIFT)
162
163 /* Use following macros for conversions between pstate_id and index */
idx_to_pstate(unsigned int i)164 static inline int idx_to_pstate(unsigned int i)
165 {
166 if (unlikely(i >= powernv_pstate_info.nr_pstates)) {
167 pr_warn_once("index %u is out of bound\n", i);
168 return powernv_freqs[powernv_pstate_info.nominal].driver_data;
169 }
170
171 return powernv_freqs[i].driver_data;
172 }
173
pstate_to_idx(int pstate)174 static inline unsigned int pstate_to_idx(int pstate)
175 {
176 int min = powernv_freqs[powernv_pstate_info.min].driver_data;
177 int max = powernv_freqs[powernv_pstate_info.max].driver_data;
178
179 if (min > 0) {
180 if (unlikely((pstate < max) || (pstate > min))) {
181 pr_warn_once("pstate %d is out of bound\n", pstate);
182 return powernv_pstate_info.nominal;
183 }
184 } else {
185 if (unlikely((pstate > max) || (pstate < min))) {
186 pr_warn_once("pstate %d is out of bound\n", pstate);
187 return powernv_pstate_info.nominal;
188 }
189 }
190 /*
191 * abs() is deliberately used so that is works with
192 * both monotonically increasing and decreasing
193 * pstate values
194 */
195 return abs(pstate - idx_to_pstate(powernv_pstate_info.max));
196 }
197
reset_gpstates(struct cpufreq_policy * policy)198 static inline void reset_gpstates(struct cpufreq_policy *policy)
199 {
200 struct global_pstate_info *gpstates = policy->driver_data;
201
202 gpstates->highest_lpstate_idx = 0;
203 gpstates->elapsed_time = 0;
204 gpstates->last_sampled_time = 0;
205 gpstates->last_lpstate_idx = 0;
206 gpstates->last_gpstate_idx = 0;
207 }
208
209 /*
210 * Initialize the freq table based on data obtained
211 * from the firmware passed via device-tree
212 */
init_powernv_pstates(void)213 static int init_powernv_pstates(void)
214 {
215 struct device_node *power_mgt;
216 int i, nr_pstates = 0;
217 const __be32 *pstate_ids, *pstate_freqs;
218 u32 len_ids, len_freqs;
219 u32 pstate_min, pstate_max, pstate_nominal;
220 u32 pstate_turbo, pstate_ultra_turbo;
221
222 power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
223 if (!power_mgt) {
224 pr_warn("power-mgt node not found\n");
225 return -ENODEV;
226 }
227
228 if (of_property_read_u32(power_mgt, "ibm,pstate-min", &pstate_min)) {
229 pr_warn("ibm,pstate-min node not found\n");
230 return -ENODEV;
231 }
232
233 if (of_property_read_u32(power_mgt, "ibm,pstate-max", &pstate_max)) {
234 pr_warn("ibm,pstate-max node not found\n");
235 return -ENODEV;
236 }
237
238 if (of_property_read_u32(power_mgt, "ibm,pstate-nominal",
239 &pstate_nominal)) {
240 pr_warn("ibm,pstate-nominal not found\n");
241 return -ENODEV;
242 }
243
244 if (of_property_read_u32(power_mgt, "ibm,pstate-ultra-turbo",
245 &pstate_ultra_turbo)) {
246 powernv_pstate_info.wof_enabled = false;
247 goto next;
248 }
249
250 if (of_property_read_u32(power_mgt, "ibm,pstate-turbo",
251 &pstate_turbo)) {
252 powernv_pstate_info.wof_enabled = false;
253 goto next;
254 }
255
256 if (pstate_turbo == pstate_ultra_turbo)
257 powernv_pstate_info.wof_enabled = false;
258 else
259 powernv_pstate_info.wof_enabled = true;
260
261 next:
262 pr_info("cpufreq pstate min %d nominal %d max %d\n", pstate_min,
263 pstate_nominal, pstate_max);
264 pr_info("Workload Optimized Frequency is %s in the platform\n",
265 (powernv_pstate_info.wof_enabled) ? "enabled" : "disabled");
266
267 pstate_ids = of_get_property(power_mgt, "ibm,pstate-ids", &len_ids);
268 if (!pstate_ids) {
269 pr_warn("ibm,pstate-ids not found\n");
270 return -ENODEV;
271 }
272
273 pstate_freqs = of_get_property(power_mgt, "ibm,pstate-frequencies-mhz",
274 &len_freqs);
275 if (!pstate_freqs) {
276 pr_warn("ibm,pstate-frequencies-mhz not found\n");
277 return -ENODEV;
278 }
279
280 if (len_ids != len_freqs) {
281 pr_warn("Entries in ibm,pstate-ids and "
282 "ibm,pstate-frequencies-mhz does not match\n");
283 }
284
285 nr_pstates = min(len_ids, len_freqs) / sizeof(u32);
286 if (!nr_pstates) {
287 pr_warn("No PStates found\n");
288 return -ENODEV;
289 }
290
291 powernv_pstate_info.nr_pstates = nr_pstates;
292 pr_debug("NR PStates %d\n", nr_pstates);
293
294 pstate_sign_prefix = pstate_min & ~0xFF;
295
296 for (i = 0; i < nr_pstates; i++) {
297 u32 id = be32_to_cpu(pstate_ids[i]);
298 u32 freq = be32_to_cpu(pstate_freqs[i]);
299
300 pr_debug("PState id %d freq %d MHz\n", id, freq);
301 powernv_freqs[i].frequency = freq * 1000; /* kHz */
302 powernv_freqs[i].driver_data = id;
303
304 if (id == pstate_max)
305 powernv_pstate_info.max = i;
306 if (id == pstate_nominal)
307 powernv_pstate_info.nominal = i;
308 if (id == pstate_min)
309 powernv_pstate_info.min = i;
310
311 if (powernv_pstate_info.wof_enabled && id == pstate_turbo) {
312 int j;
313
314 for (j = i - 1; j >= (int)powernv_pstate_info.max; j--)
315 powernv_freqs[j].flags = CPUFREQ_BOOST_FREQ;
316 }
317 }
318
319 /* End of list marker entry */
320 powernv_freqs[i].frequency = CPUFREQ_TABLE_END;
321 return 0;
322 }
323
324 /* Returns the CPU frequency corresponding to the pstate_id. */
pstate_id_to_freq(int pstate_id)325 static unsigned int pstate_id_to_freq(int pstate_id)
326 {
327 int i;
328
329 i = pstate_to_idx(pstate_id);
330 if (i >= powernv_pstate_info.nr_pstates || i < 0) {
331 pr_warn("PState id %d outside of PState table, "
332 "reporting nominal id %d instead\n",
333 pstate_id, idx_to_pstate(powernv_pstate_info.nominal));
334 i = powernv_pstate_info.nominal;
335 }
336
337 return powernv_freqs[i].frequency;
338 }
339
340 /*
341 * cpuinfo_nominal_freq_show - Show the nominal CPU frequency as indicated by
342 * the firmware
343 */
cpuinfo_nominal_freq_show(struct cpufreq_policy * policy,char * buf)344 static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy,
345 char *buf)
346 {
347 return sprintf(buf, "%u\n",
348 powernv_freqs[powernv_pstate_info.nominal].frequency);
349 }
350
351 struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq =
352 __ATTR_RO(cpuinfo_nominal_freq);
353
354 #define SCALING_BOOST_FREQS_ATTR_INDEX 2
355
356 static struct freq_attr *powernv_cpu_freq_attr[] = {
357 &cpufreq_freq_attr_scaling_available_freqs,
358 &cpufreq_freq_attr_cpuinfo_nominal_freq,
359 &cpufreq_freq_attr_scaling_boost_freqs,
360 NULL,
361 };
362
363 #define throttle_attr(name, member) \
364 static ssize_t name##_show(struct cpufreq_policy *policy, char *buf) \
365 { \
366 struct chip *chip = per_cpu(chip_info, policy->cpu); \
367 \
368 return sprintf(buf, "%u\n", chip->member); \
369 } \
370 \
371 static struct freq_attr throttle_attr_##name = __ATTR_RO(name) \
372
373 throttle_attr(unthrottle, reason[NO_THROTTLE]);
374 throttle_attr(powercap, reason[POWERCAP]);
375 throttle_attr(overtemp, reason[CPU_OVERTEMP]);
376 throttle_attr(supply_fault, reason[POWER_SUPPLY_FAILURE]);
377 throttle_attr(overcurrent, reason[OVERCURRENT]);
378 throttle_attr(occ_reset, reason[OCC_RESET_THROTTLE]);
379 throttle_attr(turbo_stat, throttle_turbo);
380 throttle_attr(sub_turbo_stat, throttle_sub_turbo);
381
382 static struct attribute *throttle_attrs[] = {
383 &throttle_attr_unthrottle.attr,
384 &throttle_attr_powercap.attr,
385 &throttle_attr_overtemp.attr,
386 &throttle_attr_supply_fault.attr,
387 &throttle_attr_overcurrent.attr,
388 &throttle_attr_occ_reset.attr,
389 &throttle_attr_turbo_stat.attr,
390 &throttle_attr_sub_turbo_stat.attr,
391 NULL,
392 };
393
394 static const struct attribute_group throttle_attr_grp = {
395 .name = "throttle_stats",
396 .attrs = throttle_attrs,
397 };
398
399 /* Helper routines */
400
401 /* Access helpers to power mgt SPR */
402
get_pmspr(unsigned long sprn)403 static inline unsigned long get_pmspr(unsigned long sprn)
404 {
405 switch (sprn) {
406 case SPRN_PMCR:
407 return mfspr(SPRN_PMCR);
408
409 case SPRN_PMICR:
410 return mfspr(SPRN_PMICR);
411
412 case SPRN_PMSR:
413 return mfspr(SPRN_PMSR);
414 }
415 BUG();
416 }
417
set_pmspr(unsigned long sprn,unsigned long val)418 static inline void set_pmspr(unsigned long sprn, unsigned long val)
419 {
420 switch (sprn) {
421 case SPRN_PMCR:
422 mtspr(SPRN_PMCR, val);
423 return;
424
425 case SPRN_PMICR:
426 mtspr(SPRN_PMICR, val);
427 return;
428 }
429 BUG();
430 }
431
432 /*
433 * Use objects of this type to query/update
434 * pstates on a remote CPU via smp_call_function.
435 */
436 struct powernv_smp_call_data {
437 unsigned int freq;
438 int pstate_id;
439 int gpstate_id;
440 };
441
442 /*
443 * powernv_read_cpu_freq: Reads the current frequency on this CPU.
444 *
445 * Called via smp_call_function.
446 *
447 * Note: The caller of the smp_call_function should pass an argument of
448 * the type 'struct powernv_smp_call_data *' along with this function.
449 *
450 * The current frequency on this CPU will be returned via
451 * ((struct powernv_smp_call_data *)arg)->freq;
452 */
powernv_read_cpu_freq(void * arg)453 static void powernv_read_cpu_freq(void *arg)
454 {
455 unsigned long pmspr_val;
456 struct powernv_smp_call_data *freq_data = arg;
457
458 pmspr_val = get_pmspr(SPRN_PMSR);
459 freq_data->pstate_id = extract_local_pstate(pmspr_val);
460 freq_data->freq = pstate_id_to_freq(freq_data->pstate_id);
461
462 pr_debug("cpu %d pmsr %016lX pstate_id %d frequency %d kHz\n",
463 raw_smp_processor_id(), pmspr_val, freq_data->pstate_id,
464 freq_data->freq);
465 }
466
467 /*
468 * powernv_cpufreq_get: Returns the CPU frequency as reported by the
469 * firmware for CPU 'cpu'. This value is reported through the sysfs
470 * file cpuinfo_cur_freq.
471 */
powernv_cpufreq_get(unsigned int cpu)472 static unsigned int powernv_cpufreq_get(unsigned int cpu)
473 {
474 struct powernv_smp_call_data freq_data;
475
476 smp_call_function_any(cpu_sibling_mask(cpu), powernv_read_cpu_freq,
477 &freq_data, 1);
478
479 return freq_data.freq;
480 }
481
482 /*
483 * set_pstate: Sets the pstate on this CPU.
484 *
485 * This is called via an smp_call_function.
486 *
487 * The caller must ensure that freq_data is of the type
488 * (struct powernv_smp_call_data *) and the pstate_id which needs to be set
489 * on this CPU should be present in freq_data->pstate_id.
490 */
set_pstate(void * data)491 static void set_pstate(void *data)
492 {
493 unsigned long val;
494 struct powernv_smp_call_data *freq_data = data;
495 unsigned long pstate_ul = freq_data->pstate_id;
496 unsigned long gpstate_ul = freq_data->gpstate_id;
497
498 val = get_pmspr(SPRN_PMCR);
499 val = val & 0x0000FFFFFFFFFFFFULL;
500
501 pstate_ul = pstate_ul & 0xFF;
502 gpstate_ul = gpstate_ul & 0xFF;
503
504 /* Set both global(bits 56..63) and local(bits 48..55) PStates */
505 val = val | (gpstate_ul << 56) | (pstate_ul << 48);
506
507 pr_debug("Setting cpu %d pmcr to %016lX\n",
508 raw_smp_processor_id(), val);
509 set_pmspr(SPRN_PMCR, val);
510 }
511
512 /*
513 * get_nominal_index: Returns the index corresponding to the nominal
514 * pstate in the cpufreq table
515 */
get_nominal_index(void)516 static inline unsigned int get_nominal_index(void)
517 {
518 return powernv_pstate_info.nominal;
519 }
520
powernv_cpufreq_throttle_check(void * data)521 static void powernv_cpufreq_throttle_check(void *data)
522 {
523 struct chip *chip;
524 unsigned int cpu = smp_processor_id();
525 unsigned long pmsr;
526 int pmsr_pmax;
527 unsigned int pmsr_pmax_idx;
528
529 pmsr = get_pmspr(SPRN_PMSR);
530 chip = this_cpu_read(chip_info);
531
532 /* Check for Pmax Capping */
533 pmsr_pmax = extract_max_pstate(pmsr);
534 pmsr_pmax_idx = pstate_to_idx(pmsr_pmax);
535 if (pmsr_pmax_idx != powernv_pstate_info.max) {
536 if (chip->throttled)
537 goto next;
538 chip->throttled = true;
539 if (pmsr_pmax_idx > powernv_pstate_info.nominal) {
540 pr_warn_once("CPU %d on Chip %u has Pmax(%d) reduced below nominal frequency(%d)\n",
541 cpu, chip->id, pmsr_pmax,
542 idx_to_pstate(powernv_pstate_info.nominal));
543 chip->throttle_sub_turbo++;
544 } else {
545 chip->throttle_turbo++;
546 }
547 trace_powernv_throttle(chip->id,
548 throttle_reason[chip->throttle_reason],
549 pmsr_pmax);
550 } else if (chip->throttled) {
551 chip->throttled = false;
552 trace_powernv_throttle(chip->id,
553 throttle_reason[chip->throttle_reason],
554 pmsr_pmax);
555 }
556
557 /* Check if Psafe_mode_active is set in PMSR. */
558 next:
559 if (pmsr & PMSR_PSAFE_ENABLE) {
560 throttled = true;
561 pr_info("Pstate set to safe frequency\n");
562 }
563
564 /* Check if SPR_EM_DISABLE is set in PMSR */
565 if (pmsr & PMSR_SPR_EM_DISABLE) {
566 throttled = true;
567 pr_info("Frequency Control disabled from OS\n");
568 }
569
570 if (throttled) {
571 pr_info("PMSR = %16lx\n", pmsr);
572 pr_warn("CPU Frequency could be throttled\n");
573 }
574 }
575
576 /**
577 * calc_global_pstate - Calculate global pstate
578 * @elapsed_time: Elapsed time in milliseconds
579 * @local_pstate_idx: New local pstate
580 * @highest_lpstate_idx: pstate from which its ramping down
581 *
582 * Finds the appropriate global pstate based on the pstate from which its
583 * ramping down and the time elapsed in ramping down. It follows a quadratic
584 * equation which ensures that it reaches ramping down to pmin in 5sec.
585 */
calc_global_pstate(unsigned int elapsed_time,int highest_lpstate_idx,int local_pstate_idx)586 static inline int calc_global_pstate(unsigned int elapsed_time,
587 int highest_lpstate_idx,
588 int local_pstate_idx)
589 {
590 int index_diff;
591
592 /*
593 * Using ramp_down_percent we get the percentage of rampdown
594 * that we are expecting to be dropping. Difference between
595 * highest_lpstate_idx and powernv_pstate_info.min will give a absolute
596 * number of how many pstates we will drop eventually by the end of
597 * 5 seconds, then just scale it get the number pstates to be dropped.
598 */
599 index_diff = ((int)ramp_down_percent(elapsed_time) *
600 (powernv_pstate_info.min - highest_lpstate_idx)) / 100;
601
602 /* Ensure that global pstate is >= to local pstate */
603 if (highest_lpstate_idx + index_diff >= local_pstate_idx)
604 return local_pstate_idx;
605 else
606 return highest_lpstate_idx + index_diff;
607 }
608
queue_gpstate_timer(struct global_pstate_info * gpstates)609 static inline void queue_gpstate_timer(struct global_pstate_info *gpstates)
610 {
611 unsigned int timer_interval;
612
613 /*
614 * Setting up timer to fire after GPSTATE_TIMER_INTERVAL ms, But
615 * if it exceeds MAX_RAMP_DOWN_TIME ms for ramp down time.
616 * Set timer such that it fires exactly at MAX_RAMP_DOWN_TIME
617 * seconds of ramp down time.
618 */
619 if ((gpstates->elapsed_time + GPSTATE_TIMER_INTERVAL)
620 > MAX_RAMP_DOWN_TIME)
621 timer_interval = MAX_RAMP_DOWN_TIME - gpstates->elapsed_time;
622 else
623 timer_interval = GPSTATE_TIMER_INTERVAL;
624
625 mod_timer(&gpstates->timer, jiffies + msecs_to_jiffies(timer_interval));
626 }
627
628 /**
629 * gpstate_timer_handler
630 *
631 * @data: pointer to cpufreq_policy on which timer was queued
632 *
633 * This handler brings down the global pstate closer to the local pstate
634 * according quadratic equation. Queues a new timer if it is still not equal
635 * to local pstate
636 */
gpstate_timer_handler(unsigned long data)637 void gpstate_timer_handler(unsigned long data)
638 {
639 struct cpufreq_policy *policy = (struct cpufreq_policy *)data;
640 struct global_pstate_info *gpstates = policy->driver_data;
641 int gpstate_idx, lpstate_idx;
642 unsigned long val;
643 unsigned int time_diff = jiffies_to_msecs(jiffies)
644 - gpstates->last_sampled_time;
645 struct powernv_smp_call_data freq_data;
646
647 if (!spin_trylock(&gpstates->gpstate_lock))
648 return;
649 /*
650 * If the timer has migrated to the different cpu then bring
651 * it back to one of the policy->cpus
652 */
653 if (!cpumask_test_cpu(raw_smp_processor_id(), policy->cpus)) {
654 gpstates->timer.expires = jiffies + msecs_to_jiffies(1);
655 add_timer_on(&gpstates->timer, cpumask_first(policy->cpus));
656 spin_unlock(&gpstates->gpstate_lock);
657 return;
658 }
659
660 /*
661 * If PMCR was last updated was using fast_swtich then
662 * We may have wrong in gpstate->last_lpstate_idx
663 * value. Hence, read from PMCR to get correct data.
664 */
665 val = get_pmspr(SPRN_PMCR);
666 freq_data.gpstate_id = extract_global_pstate(val);
667 freq_data.pstate_id = extract_local_pstate(val);
668 if (freq_data.gpstate_id == freq_data.pstate_id) {
669 reset_gpstates(policy);
670 spin_unlock(&gpstates->gpstate_lock);
671 return;
672 }
673
674 gpstates->last_sampled_time += time_diff;
675 gpstates->elapsed_time += time_diff;
676
677 if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
678 gpstate_idx = pstate_to_idx(freq_data.pstate_id);
679 lpstate_idx = gpstate_idx;
680 reset_gpstates(policy);
681 gpstates->highest_lpstate_idx = gpstate_idx;
682 } else {
683 lpstate_idx = pstate_to_idx(freq_data.pstate_id);
684 gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
685 gpstates->highest_lpstate_idx,
686 lpstate_idx);
687 }
688 freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
689 gpstates->last_gpstate_idx = gpstate_idx;
690 gpstates->last_lpstate_idx = lpstate_idx;
691 /*
692 * If local pstate is equal to global pstate, rampdown is over
693 * So timer is not required to be queued.
694 */
695 if (gpstate_idx != gpstates->last_lpstate_idx)
696 queue_gpstate_timer(gpstates);
697
698 set_pstate(&freq_data);
699 spin_unlock(&gpstates->gpstate_lock);
700 }
701
702 /*
703 * powernv_cpufreq_target_index: Sets the frequency corresponding to
704 * the cpufreq table entry indexed by new_index on the cpus in the
705 * mask policy->cpus
706 */
powernv_cpufreq_target_index(struct cpufreq_policy * policy,unsigned int new_index)707 static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
708 unsigned int new_index)
709 {
710 struct powernv_smp_call_data freq_data;
711 unsigned int cur_msec, gpstate_idx;
712 struct global_pstate_info *gpstates = policy->driver_data;
713
714 if (unlikely(rebooting) && new_index != get_nominal_index())
715 return 0;
716
717 if (!throttled) {
718 /* we don't want to be preempted while
719 * checking if the CPU frequency has been throttled
720 */
721 preempt_disable();
722 powernv_cpufreq_throttle_check(NULL);
723 preempt_enable();
724 }
725
726 cur_msec = jiffies_to_msecs(get_jiffies_64());
727
728 spin_lock(&gpstates->gpstate_lock);
729 freq_data.pstate_id = idx_to_pstate(new_index);
730
731 if (!gpstates->last_sampled_time) {
732 gpstate_idx = new_index;
733 gpstates->highest_lpstate_idx = new_index;
734 goto gpstates_done;
735 }
736
737 if (gpstates->last_gpstate_idx < new_index) {
738 gpstates->elapsed_time += cur_msec -
739 gpstates->last_sampled_time;
740
741 /*
742 * If its has been ramping down for more than MAX_RAMP_DOWN_TIME
743 * we should be resetting all global pstate related data. Set it
744 * equal to local pstate to start fresh.
745 */
746 if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
747 reset_gpstates(policy);
748 gpstates->highest_lpstate_idx = new_index;
749 gpstate_idx = new_index;
750 } else {
751 /* Elaspsed_time is less than 5 seconds, continue to rampdown */
752 gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
753 gpstates->highest_lpstate_idx,
754 new_index);
755 }
756 } else {
757 reset_gpstates(policy);
758 gpstates->highest_lpstate_idx = new_index;
759 gpstate_idx = new_index;
760 }
761
762 /*
763 * If local pstate is equal to global pstate, rampdown is over
764 * So timer is not required to be queued.
765 */
766 if (gpstate_idx != new_index)
767 queue_gpstate_timer(gpstates);
768 else
769 del_timer_sync(&gpstates->timer);
770
771 gpstates_done:
772 freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
773 gpstates->last_sampled_time = cur_msec;
774 gpstates->last_gpstate_idx = gpstate_idx;
775 gpstates->last_lpstate_idx = new_index;
776
777 spin_unlock(&gpstates->gpstate_lock);
778
779 /*
780 * Use smp_call_function to send IPI and execute the
781 * mtspr on target CPU. We could do that without IPI
782 * if current CPU is within policy->cpus (core)
783 */
784 smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1);
785 return 0;
786 }
787
powernv_cpufreq_cpu_init(struct cpufreq_policy * policy)788 static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy)
789 {
790 int base, i, ret;
791 struct kernfs_node *kn;
792 struct global_pstate_info *gpstates;
793
794 base = cpu_first_thread_sibling(policy->cpu);
795
796 for (i = 0; i < threads_per_core; i++)
797 cpumask_set_cpu(base + i, policy->cpus);
798
799 kn = kernfs_find_and_get(policy->kobj.sd, throttle_attr_grp.name);
800 if (!kn) {
801 int ret;
802
803 ret = sysfs_create_group(&policy->kobj, &throttle_attr_grp);
804 if (ret) {
805 pr_info("Failed to create throttle stats directory for cpu %d\n",
806 policy->cpu);
807 return ret;
808 }
809 } else {
810 kernfs_put(kn);
811 }
812
813 gpstates = kzalloc(sizeof(*gpstates), GFP_KERNEL);
814 if (!gpstates)
815 return -ENOMEM;
816
817 policy->driver_data = gpstates;
818
819 /* initialize timer */
820 init_timer_pinned_deferrable(&gpstates->timer);
821 gpstates->timer.data = (unsigned long)policy;
822 gpstates->timer.function = gpstate_timer_handler;
823 gpstates->timer.expires = jiffies +
824 msecs_to_jiffies(GPSTATE_TIMER_INTERVAL);
825 spin_lock_init(&gpstates->gpstate_lock);
826 ret = cpufreq_table_validate_and_show(policy, powernv_freqs);
827
828 if (ret < 0) {
829 kfree(policy->driver_data);
830 return ret;
831 }
832
833 policy->fast_switch_possible = true;
834 return ret;
835 }
836
powernv_cpufreq_cpu_exit(struct cpufreq_policy * policy)837 static int powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy)
838 {
839 /* timer is deleted in cpufreq_cpu_stop() */
840 kfree(policy->driver_data);
841
842 return 0;
843 }
844
powernv_cpufreq_reboot_notifier(struct notifier_block * nb,unsigned long action,void * unused)845 static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb,
846 unsigned long action, void *unused)
847 {
848 int cpu;
849 struct cpufreq_policy cpu_policy;
850
851 rebooting = true;
852 for_each_online_cpu(cpu) {
853 cpufreq_get_policy(&cpu_policy, cpu);
854 powernv_cpufreq_target_index(&cpu_policy, get_nominal_index());
855 }
856
857 return NOTIFY_DONE;
858 }
859
860 static struct notifier_block powernv_cpufreq_reboot_nb = {
861 .notifier_call = powernv_cpufreq_reboot_notifier,
862 };
863
powernv_cpufreq_work_fn(struct work_struct * work)864 void powernv_cpufreq_work_fn(struct work_struct *work)
865 {
866 struct chip *chip = container_of(work, struct chip, throttle);
867 unsigned int cpu;
868 cpumask_t mask;
869
870 get_online_cpus();
871 cpumask_and(&mask, &chip->mask, cpu_online_mask);
872 smp_call_function_any(&mask,
873 powernv_cpufreq_throttle_check, NULL, 0);
874
875 if (!chip->restore)
876 goto out;
877
878 chip->restore = false;
879 for_each_cpu(cpu, &mask) {
880 int index;
881 struct cpufreq_policy policy;
882
883 cpufreq_get_policy(&policy, cpu);
884 index = cpufreq_table_find_index_c(&policy, policy.cur);
885 powernv_cpufreq_target_index(&policy, index);
886 cpumask_andnot(&mask, &mask, policy.cpus);
887 }
888 out:
889 put_online_cpus();
890 }
891
powernv_cpufreq_occ_msg(struct notifier_block * nb,unsigned long msg_type,void * _msg)892 static int powernv_cpufreq_occ_msg(struct notifier_block *nb,
893 unsigned long msg_type, void *_msg)
894 {
895 struct opal_msg *msg = _msg;
896 struct opal_occ_msg omsg;
897 int i;
898
899 if (msg_type != OPAL_MSG_OCC)
900 return 0;
901
902 omsg.type = be64_to_cpu(msg->params[0]);
903
904 switch (omsg.type) {
905 case OCC_RESET:
906 occ_reset = true;
907 pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n");
908 /*
909 * powernv_cpufreq_throttle_check() is called in
910 * target() callback which can detect the throttle state
911 * for governors like ondemand.
912 * But static governors will not call target() often thus
913 * report throttling here.
914 */
915 if (!throttled) {
916 throttled = true;
917 pr_warn("CPU frequency is throttled for duration\n");
918 }
919
920 break;
921 case OCC_LOAD:
922 pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n");
923 break;
924 case OCC_THROTTLE:
925 omsg.chip = be64_to_cpu(msg->params[1]);
926 omsg.throttle_status = be64_to_cpu(msg->params[2]);
927
928 if (occ_reset) {
929 occ_reset = false;
930 throttled = false;
931 pr_info("OCC Active, CPU frequency is no longer throttled\n");
932
933 for (i = 0; i < nr_chips; i++) {
934 chips[i].restore = true;
935 schedule_work(&chips[i].throttle);
936 }
937
938 return 0;
939 }
940
941 for (i = 0; i < nr_chips; i++)
942 if (chips[i].id == omsg.chip)
943 break;
944
945 if (omsg.throttle_status >= 0 &&
946 omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS) {
947 chips[i].throttle_reason = omsg.throttle_status;
948 chips[i].reason[omsg.throttle_status]++;
949 }
950
951 if (!omsg.throttle_status)
952 chips[i].restore = true;
953
954 schedule_work(&chips[i].throttle);
955 }
956 return 0;
957 }
958
959 static struct notifier_block powernv_cpufreq_opal_nb = {
960 .notifier_call = powernv_cpufreq_occ_msg,
961 .next = NULL,
962 .priority = 0,
963 };
964
powernv_cpufreq_stop_cpu(struct cpufreq_policy * policy)965 static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy)
966 {
967 struct powernv_smp_call_data freq_data;
968 struct global_pstate_info *gpstates = policy->driver_data;
969
970 freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min);
971 freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min);
972 smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1);
973 del_timer_sync(&gpstates->timer);
974 }
975
powernv_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)976 static unsigned int powernv_fast_switch(struct cpufreq_policy *policy,
977 unsigned int target_freq)
978 {
979 int index;
980 struct powernv_smp_call_data freq_data;
981
982 index = cpufreq_table_find_index_dl(policy, target_freq);
983 freq_data.pstate_id = powernv_freqs[index].driver_data;
984 freq_data.gpstate_id = powernv_freqs[index].driver_data;
985 set_pstate(&freq_data);
986
987 return powernv_freqs[index].frequency;
988 }
989
990 static struct cpufreq_driver powernv_cpufreq_driver = {
991 .name = "powernv-cpufreq",
992 .flags = CPUFREQ_CONST_LOOPS,
993 .init = powernv_cpufreq_cpu_init,
994 .exit = powernv_cpufreq_cpu_exit,
995 .verify = cpufreq_generic_frequency_table_verify,
996 .target_index = powernv_cpufreq_target_index,
997 .fast_switch = powernv_fast_switch,
998 .get = powernv_cpufreq_get,
999 .stop_cpu = powernv_cpufreq_stop_cpu,
1000 .attr = powernv_cpu_freq_attr,
1001 };
1002
init_chip_info(void)1003 static int init_chip_info(void)
1004 {
1005 unsigned int *chip;
1006 unsigned int cpu, i;
1007 unsigned int prev_chip_id = UINT_MAX;
1008 int ret = 0;
1009
1010 chip = kcalloc(num_possible_cpus(), sizeof(*chip), GFP_KERNEL);
1011 if (!chip)
1012 return -ENOMEM;
1013
1014 for_each_possible_cpu(cpu) {
1015 unsigned int id = cpu_to_chip_id(cpu);
1016
1017 if (prev_chip_id != id) {
1018 prev_chip_id = id;
1019 chip[nr_chips++] = id;
1020 }
1021 }
1022
1023 chips = kcalloc(nr_chips, sizeof(struct chip), GFP_KERNEL);
1024 if (!chips) {
1025 ret = -ENOMEM;
1026 goto free_and_return;
1027 }
1028
1029 for (i = 0; i < nr_chips; i++) {
1030 chips[i].id = chip[i];
1031 cpumask_copy(&chips[i].mask, cpumask_of_node(chip[i]));
1032 INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn);
1033 for_each_cpu(cpu, &chips[i].mask)
1034 per_cpu(chip_info, cpu) = &chips[i];
1035 }
1036
1037 free_and_return:
1038 kfree(chip);
1039 return ret;
1040 }
1041
clean_chip_info(void)1042 static inline void clean_chip_info(void)
1043 {
1044 kfree(chips);
1045 }
1046
unregister_all_notifiers(void)1047 static inline void unregister_all_notifiers(void)
1048 {
1049 opal_message_notifier_unregister(OPAL_MSG_OCC,
1050 &powernv_cpufreq_opal_nb);
1051 unregister_reboot_notifier(&powernv_cpufreq_reboot_nb);
1052 }
1053
powernv_cpufreq_init(void)1054 static int __init powernv_cpufreq_init(void)
1055 {
1056 int rc = 0;
1057
1058 /* Don't probe on pseries (guest) platforms */
1059 if (!firmware_has_feature(FW_FEATURE_OPAL))
1060 return -ENODEV;
1061
1062 /* Discover pstates from device tree and init */
1063 rc = init_powernv_pstates();
1064 if (rc)
1065 goto out;
1066
1067 /* Populate chip info */
1068 rc = init_chip_info();
1069 if (rc)
1070 goto out;
1071
1072 register_reboot_notifier(&powernv_cpufreq_reboot_nb);
1073 opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb);
1074
1075 if (powernv_pstate_info.wof_enabled)
1076 powernv_cpufreq_driver.boost_enabled = true;
1077 else
1078 powernv_cpu_freq_attr[SCALING_BOOST_FREQS_ATTR_INDEX] = NULL;
1079
1080 rc = cpufreq_register_driver(&powernv_cpufreq_driver);
1081 if (rc) {
1082 pr_info("Failed to register the cpufreq driver (%d)\n", rc);
1083 goto cleanup_notifiers;
1084 }
1085
1086 if (powernv_pstate_info.wof_enabled)
1087 cpufreq_enable_boost_support();
1088
1089 return 0;
1090 cleanup_notifiers:
1091 unregister_all_notifiers();
1092 clean_chip_info();
1093 out:
1094 pr_info("Platform driver disabled. System does not support PState control\n");
1095 return rc;
1096 }
1097 module_init(powernv_cpufreq_init);
1098
powernv_cpufreq_exit(void)1099 static void __exit powernv_cpufreq_exit(void)
1100 {
1101 cpufreq_unregister_driver(&powernv_cpufreq_driver);
1102 unregister_all_notifiers();
1103 clean_chip_info();
1104 }
1105 module_exit(powernv_cpufreq_exit);
1106
1107 MODULE_LICENSE("GPL");
1108 MODULE_AUTHOR("Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>");
1109