1 /*
2 * linux/drivers/clocksource/arm_arch_timer.c
3 *
4 * Copyright (C) 2011 ARM Ltd.
5 * All Rights Reserved
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/device.h>
14 #include <linux/smp.h>
15 #include <linux/cpu.h>
16 #include <linux/cpu_pm.h>
17 #include <linux/clockchips.h>
18 #include <linux/clocksource.h>
19 #include <linux/interrupt.h>
20 #include <linux/of_irq.h>
21 #include <linux/of_address.h>
22 #include <linux/io.h>
23 #include <linux/slab.h>
24 #include <linux/sched_clock.h>
25 #include <linux/acpi.h>
26
27 #include <asm/arch_timer.h>
28 #include <asm/virt.h>
29
30 #include <clocksource/arm_arch_timer.h>
31
32 #define CNTTIDR 0x08
33 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
34
35 #define CNTVCT_LO 0x08
36 #define CNTVCT_HI 0x0c
37 #define CNTFRQ 0x10
38 #define CNTP_TVAL 0x28
39 #define CNTP_CTL 0x2c
40 #define CNTV_TVAL 0x38
41 #define CNTV_CTL 0x3c
42
43 #define ARCH_CP15_TIMER BIT(0)
44 #define ARCH_MEM_TIMER BIT(1)
45 static unsigned arch_timers_present __initdata;
46
47 static void __iomem *arch_counter_base;
48
49 struct arch_timer {
50 void __iomem *base;
51 struct clock_event_device evt;
52 };
53
54 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
55
56 static u32 arch_timer_rate;
57
58 enum ppi_nr {
59 PHYS_SECURE_PPI,
60 PHYS_NONSECURE_PPI,
61 VIRT_PPI,
62 HYP_PPI,
63 MAX_TIMER_PPI
64 };
65
66 static int arch_timer_ppi[MAX_TIMER_PPI];
67
68 static struct clock_event_device __percpu *arch_timer_evt;
69
70 static bool arch_timer_use_virtual = true;
71 static bool arch_timer_c3stop;
72 static bool arch_timer_mem_use_virtual;
73
74 /*
75 * Architected system timer support.
76 */
77
78 static __always_inline
arch_timer_reg_write(int access,enum arch_timer_reg reg,u32 val,struct clock_event_device * clk)79 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
80 struct clock_event_device *clk)
81 {
82 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
83 struct arch_timer *timer = to_arch_timer(clk);
84 switch (reg) {
85 case ARCH_TIMER_REG_CTRL:
86 writel_relaxed(val, timer->base + CNTP_CTL);
87 break;
88 case ARCH_TIMER_REG_TVAL:
89 writel_relaxed(val, timer->base + CNTP_TVAL);
90 break;
91 }
92 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
93 struct arch_timer *timer = to_arch_timer(clk);
94 switch (reg) {
95 case ARCH_TIMER_REG_CTRL:
96 writel_relaxed(val, timer->base + CNTV_CTL);
97 break;
98 case ARCH_TIMER_REG_TVAL:
99 writel_relaxed(val, timer->base + CNTV_TVAL);
100 break;
101 }
102 } else {
103 arch_timer_reg_write_cp15(access, reg, val);
104 }
105 }
106
107 static __always_inline
arch_timer_reg_read(int access,enum arch_timer_reg reg,struct clock_event_device * clk)108 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
109 struct clock_event_device *clk)
110 {
111 u32 val;
112
113 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
114 struct arch_timer *timer = to_arch_timer(clk);
115 switch (reg) {
116 case ARCH_TIMER_REG_CTRL:
117 val = readl_relaxed(timer->base + CNTP_CTL);
118 break;
119 case ARCH_TIMER_REG_TVAL:
120 val = readl_relaxed(timer->base + CNTP_TVAL);
121 break;
122 }
123 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
124 struct arch_timer *timer = to_arch_timer(clk);
125 switch (reg) {
126 case ARCH_TIMER_REG_CTRL:
127 val = readl_relaxed(timer->base + CNTV_CTL);
128 break;
129 case ARCH_TIMER_REG_TVAL:
130 val = readl_relaxed(timer->base + CNTV_TVAL);
131 break;
132 }
133 } else {
134 val = arch_timer_reg_read_cp15(access, reg);
135 }
136
137 return val;
138 }
139
timer_handler(const int access,struct clock_event_device * evt)140 static __always_inline irqreturn_t timer_handler(const int access,
141 struct clock_event_device *evt)
142 {
143 unsigned long ctrl;
144
145 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
146 if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
147 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
148 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
149 evt->event_handler(evt);
150 return IRQ_HANDLED;
151 }
152
153 return IRQ_NONE;
154 }
155
arch_timer_handler_virt(int irq,void * dev_id)156 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
157 {
158 struct clock_event_device *evt = dev_id;
159
160 return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
161 }
162
arch_timer_handler_phys(int irq,void * dev_id)163 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
164 {
165 struct clock_event_device *evt = dev_id;
166
167 return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
168 }
169
arch_timer_handler_phys_mem(int irq,void * dev_id)170 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
171 {
172 struct clock_event_device *evt = dev_id;
173
174 return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
175 }
176
arch_timer_handler_virt_mem(int irq,void * dev_id)177 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
178 {
179 struct clock_event_device *evt = dev_id;
180
181 return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
182 }
183
timer_shutdown(const int access,struct clock_event_device * clk)184 static __always_inline int timer_shutdown(const int access,
185 struct clock_event_device *clk)
186 {
187 unsigned long ctrl;
188
189 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
190 ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
191 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
192
193 return 0;
194 }
195
arch_timer_shutdown_virt(struct clock_event_device * clk)196 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
197 {
198 return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
199 }
200
arch_timer_shutdown_phys(struct clock_event_device * clk)201 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
202 {
203 return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
204 }
205
arch_timer_shutdown_virt_mem(struct clock_event_device * clk)206 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
207 {
208 return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
209 }
210
arch_timer_shutdown_phys_mem(struct clock_event_device * clk)211 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
212 {
213 return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
214 }
215
set_next_event(const int access,unsigned long evt,struct clock_event_device * clk)216 static __always_inline void set_next_event(const int access, unsigned long evt,
217 struct clock_event_device *clk)
218 {
219 unsigned long ctrl;
220 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
221 ctrl |= ARCH_TIMER_CTRL_ENABLE;
222 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
223 arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
224 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
225 }
226
arch_timer_set_next_event_virt(unsigned long evt,struct clock_event_device * clk)227 static int arch_timer_set_next_event_virt(unsigned long evt,
228 struct clock_event_device *clk)
229 {
230 set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
231 return 0;
232 }
233
arch_timer_set_next_event_phys(unsigned long evt,struct clock_event_device * clk)234 static int arch_timer_set_next_event_phys(unsigned long evt,
235 struct clock_event_device *clk)
236 {
237 set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
238 return 0;
239 }
240
arch_timer_set_next_event_virt_mem(unsigned long evt,struct clock_event_device * clk)241 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
242 struct clock_event_device *clk)
243 {
244 set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
245 return 0;
246 }
247
arch_timer_set_next_event_phys_mem(unsigned long evt,struct clock_event_device * clk)248 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
249 struct clock_event_device *clk)
250 {
251 set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
252 return 0;
253 }
254
__arch_timer_setup(unsigned type,struct clock_event_device * clk)255 static void __arch_timer_setup(unsigned type,
256 struct clock_event_device *clk)
257 {
258 clk->features = CLOCK_EVT_FEAT_ONESHOT;
259
260 if (type == ARCH_CP15_TIMER) {
261 if (arch_timer_c3stop)
262 clk->features |= CLOCK_EVT_FEAT_C3STOP;
263 clk->name = "arch_sys_timer";
264 clk->rating = 450;
265 clk->cpumask = cpumask_of(smp_processor_id());
266 if (arch_timer_use_virtual) {
267 clk->irq = arch_timer_ppi[VIRT_PPI];
268 clk->set_state_shutdown = arch_timer_shutdown_virt;
269 clk->set_next_event = arch_timer_set_next_event_virt;
270 } else {
271 clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
272 clk->set_state_shutdown = arch_timer_shutdown_phys;
273 clk->set_next_event = arch_timer_set_next_event_phys;
274 }
275 } else {
276 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
277 clk->name = "arch_mem_timer";
278 clk->rating = 400;
279 clk->cpumask = cpu_all_mask;
280 if (arch_timer_mem_use_virtual) {
281 clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
282 clk->set_next_event =
283 arch_timer_set_next_event_virt_mem;
284 } else {
285 clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
286 clk->set_next_event =
287 arch_timer_set_next_event_phys_mem;
288 }
289 }
290
291 clk->set_state_shutdown(clk);
292
293 clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
294 }
295
arch_timer_evtstrm_enable(int divider)296 static void arch_timer_evtstrm_enable(int divider)
297 {
298 u32 cntkctl = arch_timer_get_cntkctl();
299
300 cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
301 /* Set the divider and enable virtual event stream */
302 cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
303 | ARCH_TIMER_VIRT_EVT_EN;
304 arch_timer_set_cntkctl(cntkctl);
305 elf_hwcap |= HWCAP_EVTSTRM;
306 #ifdef CONFIG_COMPAT
307 compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
308 #endif
309 }
310
arch_timer_configure_evtstream(void)311 static void arch_timer_configure_evtstream(void)
312 {
313 int evt_stream_div, lsb;
314
315 /*
316 * As the event stream can at most be generated at half the frequency
317 * of the counter, use half the frequency when computing the divider.
318 */
319 evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ / 2;
320
321 /*
322 * Find the closest power of two to the divisor. If the adjacent bit
323 * of lsb (last set bit, starts from 0) is set, then we use (lsb + 1).
324 */
325 lsb = fls(evt_stream_div) - 1;
326 if (lsb > 0 && (evt_stream_div & BIT(lsb - 1)))
327 lsb++;
328
329 /* enable event stream */
330 arch_timer_evtstrm_enable(max(0, min(lsb, 15)));
331 }
332
arch_counter_set_user_access(void)333 static void arch_counter_set_user_access(void)
334 {
335 u32 cntkctl = arch_timer_get_cntkctl();
336
337 /* Disable user access to the timers and the physical counter */
338 /* Also disable virtual event stream */
339 cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
340 | ARCH_TIMER_USR_VT_ACCESS_EN
341 | ARCH_TIMER_VIRT_EVT_EN
342 | ARCH_TIMER_USR_PCT_ACCESS_EN);
343
344 /* Enable user access to the virtual counter */
345 if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_VCT_ACCESS))
346 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
347 else
348 cntkctl &= ~ARCH_TIMER_USR_VCT_ACCESS_EN;
349
350 arch_timer_set_cntkctl(cntkctl);
351 }
352
arch_timer_setup(struct clock_event_device * clk)353 static int arch_timer_setup(struct clock_event_device *clk)
354 {
355 __arch_timer_setup(ARCH_CP15_TIMER, clk);
356
357 if (arch_timer_use_virtual)
358 enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
359 else {
360 enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
361 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
362 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
363 }
364
365 arch_counter_set_user_access();
366 if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM))
367 arch_timer_configure_evtstream();
368
369 return 0;
370 }
371
372 static void
arch_timer_detect_rate(void __iomem * cntbase,struct device_node * np)373 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
374 {
375 /* Who has more than one independent system counter? */
376 if (arch_timer_rate)
377 return;
378
379 /*
380 * Try to determine the frequency from the device tree or CNTFRQ,
381 * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
382 */
383 if (!acpi_disabled ||
384 of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
385 if (cntbase)
386 arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
387 else
388 arch_timer_rate = arch_timer_get_cntfrq();
389 }
390
391 /* Check the timer frequency. */
392 if (arch_timer_rate == 0)
393 pr_warn("Architected timer frequency not available\n");
394 }
395
arch_timer_banner(unsigned type)396 static void arch_timer_banner(unsigned type)
397 {
398 pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
399 type & ARCH_CP15_TIMER ? "cp15" : "",
400 type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? " and " : "",
401 type & ARCH_MEM_TIMER ? "mmio" : "",
402 (unsigned long)arch_timer_rate / 1000000,
403 (unsigned long)(arch_timer_rate / 10000) % 100,
404 type & ARCH_CP15_TIMER ?
405 arch_timer_use_virtual ? "virt" : "phys" :
406 "",
407 type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? "/" : "",
408 type & ARCH_MEM_TIMER ?
409 arch_timer_mem_use_virtual ? "virt" : "phys" :
410 "");
411 }
412
arch_timer_get_rate(void)413 u32 arch_timer_get_rate(void)
414 {
415 return arch_timer_rate;
416 }
417
arch_counter_get_cntvct_mem(void)418 static u64 arch_counter_get_cntvct_mem(void)
419 {
420 u32 vct_lo, vct_hi, tmp_hi;
421
422 do {
423 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
424 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
425 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
426 } while (vct_hi != tmp_hi);
427
428 return ((u64) vct_hi << 32) | vct_lo;
429 }
430
431 /*
432 * Default to cp15 based access because arm64 uses this function for
433 * sched_clock() before DT is probed and the cp15 method is guaranteed
434 * to exist on arm64. arm doesn't use this before DT is probed so even
435 * if we don't have the cp15 accessors we won't have a problem.
436 */
437 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
438
arch_counter_read(struct clocksource * cs)439 static cycle_t arch_counter_read(struct clocksource *cs)
440 {
441 return arch_timer_read_counter();
442 }
443
arch_counter_read_cc(const struct cyclecounter * cc)444 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
445 {
446 return arch_timer_read_counter();
447 }
448
449 static struct clocksource clocksource_counter = {
450 .name = "arch_sys_counter",
451 .rating = 400,
452 .read = arch_counter_read,
453 .mask = CLOCKSOURCE_MASK(56),
454 .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
455 };
456
457 static struct cyclecounter cyclecounter = {
458 .read = arch_counter_read_cc,
459 .mask = CLOCKSOURCE_MASK(56),
460 };
461
462 static struct timecounter timecounter;
463
arch_timer_get_timecounter(void)464 struct timecounter *arch_timer_get_timecounter(void)
465 {
466 return &timecounter;
467 }
468
arch_counter_register(unsigned type)469 static void __init arch_counter_register(unsigned type)
470 {
471 u64 start_count;
472
473 /* Register the CP15 based counter if we have one */
474 if (type & ARCH_CP15_TIMER) {
475 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_use_virtual)
476 arch_timer_read_counter = arch_counter_get_cntvct;
477 else
478 arch_timer_read_counter = arch_counter_get_cntpct;
479 } else {
480 arch_timer_read_counter = arch_counter_get_cntvct_mem;
481
482 /* If the clocksource name is "arch_sys_counter" the
483 * VDSO will attempt to read the CP15-based counter.
484 * Ensure this does not happen when CP15-based
485 * counter is not available.
486 */
487 clocksource_counter.name = "arch_mem_counter";
488 }
489
490 start_count = arch_timer_read_counter();
491 clocksource_register_hz(&clocksource_counter, arch_timer_rate);
492 cyclecounter.mult = clocksource_counter.mult;
493 cyclecounter.shift = clocksource_counter.shift;
494 timecounter_init(&timecounter, &cyclecounter, start_count);
495
496 /* 56 bits minimum, so we assume worst case rollover */
497 sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
498 }
499
arch_timer_stop(struct clock_event_device * clk)500 static void arch_timer_stop(struct clock_event_device *clk)
501 {
502 pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
503 clk->irq, smp_processor_id());
504
505 if (arch_timer_use_virtual)
506 disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
507 else {
508 disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
509 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
510 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
511 }
512
513 clk->set_state_shutdown(clk);
514 }
515
arch_timer_cpu_notify(struct notifier_block * self,unsigned long action,void * hcpu)516 static int arch_timer_cpu_notify(struct notifier_block *self,
517 unsigned long action, void *hcpu)
518 {
519 /*
520 * Grab cpu pointer in each case to avoid spurious
521 * preemptible warnings
522 */
523 switch (action & ~CPU_TASKS_FROZEN) {
524 case CPU_STARTING:
525 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
526 break;
527 case CPU_DYING:
528 arch_timer_stop(this_cpu_ptr(arch_timer_evt));
529 break;
530 }
531
532 return NOTIFY_OK;
533 }
534
535 static struct notifier_block arch_timer_cpu_nb = {
536 .notifier_call = arch_timer_cpu_notify,
537 };
538
539 #ifdef CONFIG_CPU_PM
540 static unsigned int saved_cntkctl;
arch_timer_cpu_pm_notify(struct notifier_block * self,unsigned long action,void * hcpu)541 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
542 unsigned long action, void *hcpu)
543 {
544 if (action == CPU_PM_ENTER)
545 saved_cntkctl = arch_timer_get_cntkctl();
546 else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
547 arch_timer_set_cntkctl(saved_cntkctl);
548 return NOTIFY_OK;
549 }
550
551 static struct notifier_block arch_timer_cpu_pm_notifier = {
552 .notifier_call = arch_timer_cpu_pm_notify,
553 };
554
arch_timer_cpu_pm_init(void)555 static int __init arch_timer_cpu_pm_init(void)
556 {
557 return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
558 }
559 #else
arch_timer_cpu_pm_init(void)560 static int __init arch_timer_cpu_pm_init(void)
561 {
562 return 0;
563 }
564 #endif
565
arch_timer_register(void)566 static int __init arch_timer_register(void)
567 {
568 int err;
569 int ppi;
570
571 arch_timer_evt = alloc_percpu(struct clock_event_device);
572 if (!arch_timer_evt) {
573 err = -ENOMEM;
574 goto out;
575 }
576
577 if (arch_timer_use_virtual) {
578 ppi = arch_timer_ppi[VIRT_PPI];
579 err = request_percpu_irq(ppi, arch_timer_handler_virt,
580 "arch_timer", arch_timer_evt);
581 } else {
582 ppi = arch_timer_ppi[PHYS_SECURE_PPI];
583 err = request_percpu_irq(ppi, arch_timer_handler_phys,
584 "arch_timer", arch_timer_evt);
585 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
586 ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
587 err = request_percpu_irq(ppi, arch_timer_handler_phys,
588 "arch_timer", arch_timer_evt);
589 if (err)
590 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
591 arch_timer_evt);
592 }
593 }
594
595 if (err) {
596 pr_err("arch_timer: can't register interrupt %d (%d)\n",
597 ppi, err);
598 goto out_free;
599 }
600
601 err = register_cpu_notifier(&arch_timer_cpu_nb);
602 if (err)
603 goto out_free_irq;
604
605 err = arch_timer_cpu_pm_init();
606 if (err)
607 goto out_unreg_notify;
608
609 /* Immediately configure the timer on the boot CPU */
610 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
611
612 return 0;
613
614 out_unreg_notify:
615 unregister_cpu_notifier(&arch_timer_cpu_nb);
616 out_free_irq:
617 if (arch_timer_use_virtual)
618 free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
619 else {
620 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
621 arch_timer_evt);
622 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
623 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
624 arch_timer_evt);
625 }
626
627 out_free:
628 free_percpu(arch_timer_evt);
629 out:
630 return err;
631 }
632
arch_timer_mem_register(void __iomem * base,unsigned int irq)633 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
634 {
635 int ret;
636 irq_handler_t func;
637 struct arch_timer *t;
638
639 t = kzalloc(sizeof(*t), GFP_KERNEL);
640 if (!t)
641 return -ENOMEM;
642
643 t->base = base;
644 t->evt.irq = irq;
645 __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
646
647 if (arch_timer_mem_use_virtual)
648 func = arch_timer_handler_virt_mem;
649 else
650 func = arch_timer_handler_phys_mem;
651
652 ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
653 if (ret) {
654 pr_err("arch_timer: Failed to request mem timer irq\n");
655 kfree(t);
656 }
657
658 return ret;
659 }
660
661 static const struct of_device_id arch_timer_of_match[] __initconst = {
662 { .compatible = "arm,armv7-timer", },
663 { .compatible = "arm,armv8-timer", },
664 {},
665 };
666
667 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
668 { .compatible = "arm,armv7-timer-mem", },
669 {},
670 };
671
672 static bool __init
arch_timer_needs_probing(int type,const struct of_device_id * matches)673 arch_timer_needs_probing(int type, const struct of_device_id *matches)
674 {
675 struct device_node *dn;
676 bool needs_probing = false;
677
678 dn = of_find_matching_node(NULL, matches);
679 if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
680 needs_probing = true;
681 of_node_put(dn);
682
683 return needs_probing;
684 }
685
arch_timer_common_init(void)686 static void __init arch_timer_common_init(void)
687 {
688 unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
689
690 /* Wait until both nodes are probed if we have two timers */
691 if ((arch_timers_present & mask) != mask) {
692 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
693 return;
694 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
695 return;
696 }
697
698 arch_timer_banner(arch_timers_present);
699 arch_counter_register(arch_timers_present);
700 arch_timer_arch_init();
701 }
702
arch_timer_init(void)703 static void __init arch_timer_init(void)
704 {
705 /*
706 * If HYP mode is available, we know that the physical timer
707 * has been configured to be accessible from PL1. Use it, so
708 * that a guest can use the virtual timer instead.
709 *
710 * If no interrupt provided for virtual timer, we'll have to
711 * stick to the physical timer. It'd better be accessible...
712 */
713 if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
714 arch_timer_use_virtual = false;
715
716 if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
717 !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
718 pr_warn("arch_timer: No interrupt available, giving up\n");
719 return;
720 }
721 }
722
723 arch_timer_register();
724 arch_timer_common_init();
725 }
726
arch_timer_of_init(struct device_node * np)727 static void __init arch_timer_of_init(struct device_node *np)
728 {
729 int i;
730
731 if (arch_timers_present & ARCH_CP15_TIMER) {
732 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
733 return;
734 }
735
736 arch_timers_present |= ARCH_CP15_TIMER;
737 for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
738 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
739
740 arch_timer_detect_rate(NULL, np);
741
742 arch_timer_c3stop = !of_property_read_bool(np, "always-on");
743
744 /*
745 * If we cannot rely on firmware initializing the timer registers then
746 * we should use the physical timers instead.
747 */
748 if (IS_ENABLED(CONFIG_ARM) &&
749 of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
750 arch_timer_use_virtual = false;
751
752 arch_timer_init();
753 }
754 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
755 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
756
arch_timer_mem_init(struct device_node * np)757 static void __init arch_timer_mem_init(struct device_node *np)
758 {
759 struct device_node *frame, *best_frame = NULL;
760 void __iomem *cntctlbase, *base;
761 unsigned int irq;
762 u32 cnttidr;
763
764 arch_timers_present |= ARCH_MEM_TIMER;
765 cntctlbase = of_iomap(np, 0);
766 if (!cntctlbase) {
767 pr_err("arch_timer: Can't find CNTCTLBase\n");
768 return;
769 }
770
771 cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
772 iounmap(cntctlbase);
773
774 /*
775 * Try to find a virtual capable frame. Otherwise fall back to a
776 * physical capable frame.
777 */
778 for_each_available_child_of_node(np, frame) {
779 int n;
780
781 if (of_property_read_u32(frame, "frame-number", &n)) {
782 pr_err("arch_timer: Missing frame-number\n");
783 of_node_put(best_frame);
784 of_node_put(frame);
785 return;
786 }
787
788 if (cnttidr & CNTTIDR_VIRT(n)) {
789 of_node_put(best_frame);
790 best_frame = frame;
791 arch_timer_mem_use_virtual = true;
792 break;
793 }
794 of_node_put(best_frame);
795 best_frame = of_node_get(frame);
796 }
797
798 base = arch_counter_base = of_iomap(best_frame, 0);
799 if (!base) {
800 pr_err("arch_timer: Can't map frame's registers\n");
801 of_node_put(best_frame);
802 return;
803 }
804
805 if (arch_timer_mem_use_virtual)
806 irq = irq_of_parse_and_map(best_frame, 1);
807 else
808 irq = irq_of_parse_and_map(best_frame, 0);
809 of_node_put(best_frame);
810 if (!irq) {
811 pr_err("arch_timer: Frame missing %s irq",
812 arch_timer_mem_use_virtual ? "virt" : "phys");
813 return;
814 }
815
816 arch_timer_detect_rate(base, np);
817 arch_timer_mem_register(base, irq);
818 arch_timer_common_init();
819 }
820 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
821 arch_timer_mem_init);
822
823 #ifdef CONFIG_ACPI
map_generic_timer_interrupt(u32 interrupt,u32 flags)824 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
825 {
826 int trigger, polarity;
827
828 if (!interrupt)
829 return 0;
830
831 trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
832 : ACPI_LEVEL_SENSITIVE;
833
834 polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
835 : ACPI_ACTIVE_HIGH;
836
837 return acpi_register_gsi(NULL, interrupt, trigger, polarity);
838 }
839
840 /* Initialize per-processor generic timer */
arch_timer_acpi_init(struct acpi_table_header * table)841 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
842 {
843 struct acpi_table_gtdt *gtdt;
844
845 if (arch_timers_present & ARCH_CP15_TIMER) {
846 pr_warn("arch_timer: already initialized, skipping\n");
847 return -EINVAL;
848 }
849
850 gtdt = container_of(table, struct acpi_table_gtdt, header);
851
852 arch_timers_present |= ARCH_CP15_TIMER;
853
854 arch_timer_ppi[PHYS_SECURE_PPI] =
855 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
856 gtdt->secure_el1_flags);
857
858 arch_timer_ppi[PHYS_NONSECURE_PPI] =
859 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
860 gtdt->non_secure_el1_flags);
861
862 arch_timer_ppi[VIRT_PPI] =
863 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
864 gtdt->virtual_timer_flags);
865
866 arch_timer_ppi[HYP_PPI] =
867 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
868 gtdt->non_secure_el2_flags);
869
870 /* Get the frequency from CNTFRQ */
871 arch_timer_detect_rate(NULL, NULL);
872
873 /* Always-on capability */
874 arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
875
876 arch_timer_init();
877 return 0;
878 }
879 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
880 #endif
881