• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* irq.c: UltraSparc IRQ handling/init/registry.
2  *
3  * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net)
4  * Copyright (C) 1998  Eddie C. Dost    (ecd@skynet.be)
5  * Copyright (C) 1998  Jakub Jelinek    (jj@ultra.linux.cz)
6  */
7 
8 #include <linux/sched.h>
9 #include <linux/linkage.h>
10 #include <linux/ptrace.h>
11 #include <linux/errno.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/signal.h>
14 #include <linux/mm.h>
15 #include <linux/interrupt.h>
16 #include <linux/slab.h>
17 #include <linux/random.h>
18 #include <linux/init.h>
19 #include <linux/delay.h>
20 #include <linux/proc_fs.h>
21 #include <linux/seq_file.h>
22 #include <linux/ftrace.h>
23 #include <linux/irq.h>
24 #include <linux/kmemleak.h>
25 
26 #include <asm/ptrace.h>
27 #include <asm/processor.h>
28 #include <linux/atomic.h>
29 #include <asm/irq.h>
30 #include <asm/io.h>
31 #include <asm/iommu.h>
32 #include <asm/upa.h>
33 #include <asm/oplib.h>
34 #include <asm/prom.h>
35 #include <asm/timer.h>
36 #include <asm/smp.h>
37 #include <asm/starfire.h>
38 #include <asm/uaccess.h>
39 #include <asm/cache.h>
40 #include <asm/cpudata.h>
41 #include <asm/auxio.h>
42 #include <asm/head.h>
43 #include <asm/hypervisor.h>
44 #include <asm/cacheflush.h>
45 
46 #include "entry.h"
47 #include "cpumap.h"
48 #include "kstack.h"
49 
50 struct ino_bucket *ivector_table;
51 unsigned long ivector_table_pa;
52 
53 /* On several sun4u processors, it is illegal to mix bypass and
54  * non-bypass accesses.  Therefore we access all INO buckets
55  * using bypass accesses only.
56  */
bucket_get_chain_pa(unsigned long bucket_pa)57 static unsigned long bucket_get_chain_pa(unsigned long bucket_pa)
58 {
59 	unsigned long ret;
60 
61 	__asm__ __volatile__("ldxa	[%1] %2, %0"
62 			     : "=&r" (ret)
63 			     : "r" (bucket_pa +
64 				    offsetof(struct ino_bucket,
65 					     __irq_chain_pa)),
66 			       "i" (ASI_PHYS_USE_EC));
67 
68 	return ret;
69 }
70 
bucket_clear_chain_pa(unsigned long bucket_pa)71 static void bucket_clear_chain_pa(unsigned long bucket_pa)
72 {
73 	__asm__ __volatile__("stxa	%%g0, [%0] %1"
74 			     : /* no outputs */
75 			     : "r" (bucket_pa +
76 				    offsetof(struct ino_bucket,
77 					     __irq_chain_pa)),
78 			       "i" (ASI_PHYS_USE_EC));
79 }
80 
bucket_get_irq(unsigned long bucket_pa)81 static unsigned int bucket_get_irq(unsigned long bucket_pa)
82 {
83 	unsigned int ret;
84 
85 	__asm__ __volatile__("lduwa	[%1] %2, %0"
86 			     : "=&r" (ret)
87 			     : "r" (bucket_pa +
88 				    offsetof(struct ino_bucket,
89 					     __irq)),
90 			       "i" (ASI_PHYS_USE_EC));
91 
92 	return ret;
93 }
94 
bucket_set_irq(unsigned long bucket_pa,unsigned int irq)95 static void bucket_set_irq(unsigned long bucket_pa, unsigned int irq)
96 {
97 	__asm__ __volatile__("stwa	%0, [%1] %2"
98 			     : /* no outputs */
99 			     : "r" (irq),
100 			       "r" (bucket_pa +
101 				    offsetof(struct ino_bucket,
102 					     __irq)),
103 			       "i" (ASI_PHYS_USE_EC));
104 }
105 
106 #define irq_work_pa(__cpu)	&(trap_block[(__cpu)].irq_worklist_pa)
107 
108 static unsigned long hvirq_major __initdata;
early_hvirq_major(char * p)109 static int __init early_hvirq_major(char *p)
110 {
111 	int rc = kstrtoul(p, 10, &hvirq_major);
112 
113 	return rc;
114 }
115 early_param("hvirq", early_hvirq_major);
116 
117 static int hv_irq_version;
118 
119 /* Major version 2.0 of HV_GRP_INTR added support for the VIRQ cookie
120  * based interfaces, but:
121  *
122  * 1) Several OSs, Solaris and Linux included, use them even when only
123  *    negotiating version 1.0 (or failing to negotiate at all).  So the
124  *    hypervisor has a workaround that provides the VIRQ interfaces even
125  *    when only verion 1.0 of the API is in use.
126  *
127  * 2) Second, and more importantly, with major version 2.0 these VIRQ
128  *    interfaces only were actually hooked up for LDC interrupts, even
129  *    though the Hypervisor specification clearly stated:
130  *
131  *	The new interrupt API functions will be available to a guest
132  *	when it negotiates version 2.0 in the interrupt API group 0x2. When
133  *	a guest negotiates version 2.0, all interrupt sources will only
134  *	support using the cookie interface, and any attempt to use the
135  *	version 1.0 interrupt APIs numbered 0xa0 to 0xa6 will result in the
136  *	ENOTSUPPORTED error being returned.
137  *
138  *   with an emphasis on "all interrupt sources".
139  *
140  * To correct this, major version 3.0 was created which does actually
141  * support VIRQs for all interrupt sources (not just LDC devices).  So
142  * if we want to move completely over the cookie based VIRQs we must
143  * negotiate major version 3.0 or later of HV_GRP_INTR.
144  */
sun4v_cookie_only_virqs(void)145 static bool sun4v_cookie_only_virqs(void)
146 {
147 	if (hv_irq_version >= 3)
148 		return true;
149 	return false;
150 }
151 
irq_init_hv(void)152 static void __init irq_init_hv(void)
153 {
154 	unsigned long hv_error, major, minor = 0;
155 
156 	if (tlb_type != hypervisor)
157 		return;
158 
159 	if (hvirq_major)
160 		major = hvirq_major;
161 	else
162 		major = 3;
163 
164 	hv_error = sun4v_hvapi_register(HV_GRP_INTR, major, &minor);
165 	if (!hv_error)
166 		hv_irq_version = major;
167 	else
168 		hv_irq_version = 1;
169 
170 	pr_info("SUN4V: Using IRQ API major %d, cookie only virqs %s\n",
171 		hv_irq_version,
172 		sun4v_cookie_only_virqs() ? "enabled" : "disabled");
173 }
174 
175 /* This function is for the timer interrupt.*/
arch_probe_nr_irqs(void)176 int __init arch_probe_nr_irqs(void)
177 {
178 	return 1;
179 }
180 
181 #define DEFAULT_NUM_IVECS	(0xfffU)
182 static unsigned int nr_ivec = DEFAULT_NUM_IVECS;
183 #define NUM_IVECS (nr_ivec)
184 
size_nr_ivec(void)185 static unsigned int __init size_nr_ivec(void)
186 {
187 	if (tlb_type == hypervisor) {
188 		switch (sun4v_chip_type) {
189 		/* Athena's devhandle|devino is large.*/
190 		case SUN4V_CHIP_SPARC64X:
191 			nr_ivec = 0xffff;
192 			break;
193 		}
194 	}
195 	return nr_ivec;
196 }
197 
198 struct irq_handler_data {
199 	union {
200 		struct {
201 			unsigned int dev_handle;
202 			unsigned int dev_ino;
203 		};
204 		unsigned long sysino;
205 	};
206 	struct ino_bucket bucket;
207 	unsigned long	iclr;
208 	unsigned long	imap;
209 };
210 
irq_data_to_handle(struct irq_data * data)211 static inline unsigned int irq_data_to_handle(struct irq_data *data)
212 {
213 	struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data);
214 
215 	return ihd->dev_handle;
216 }
217 
irq_data_to_ino(struct irq_data * data)218 static inline unsigned int irq_data_to_ino(struct irq_data *data)
219 {
220 	struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data);
221 
222 	return ihd->dev_ino;
223 }
224 
irq_data_to_sysino(struct irq_data * data)225 static inline unsigned long irq_data_to_sysino(struct irq_data *data)
226 {
227 	struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data);
228 
229 	return ihd->sysino;
230 }
231 
irq_free(unsigned int irq)232 void irq_free(unsigned int irq)
233 {
234 	void *data = irq_get_handler_data(irq);
235 
236 	kfree(data);
237 	irq_set_handler_data(irq, NULL);
238 	irq_free_descs(irq, 1);
239 }
240 
irq_alloc(unsigned int dev_handle,unsigned int dev_ino)241 unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino)
242 {
243 	int irq;
244 
245 	irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL);
246 	if (irq <= 0)
247 		goto out;
248 
249 	return irq;
250 out:
251 	return 0;
252 }
253 
cookie_exists(u32 devhandle,unsigned int devino)254 static unsigned int cookie_exists(u32 devhandle, unsigned int devino)
255 {
256 	unsigned long hv_err, cookie;
257 	struct ino_bucket *bucket;
258 	unsigned int irq = 0U;
259 
260 	hv_err = sun4v_vintr_get_cookie(devhandle, devino, &cookie);
261 	if (hv_err) {
262 		pr_err("HV get cookie failed hv_err = %ld\n", hv_err);
263 		goto out;
264 	}
265 
266 	if (cookie & ((1UL << 63UL))) {
267 		cookie = ~cookie;
268 		bucket = (struct ino_bucket *) __va(cookie);
269 		irq = bucket->__irq;
270 	}
271 out:
272 	return irq;
273 }
274 
sysino_exists(u32 devhandle,unsigned int devino)275 static unsigned int sysino_exists(u32 devhandle, unsigned int devino)
276 {
277 	unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino);
278 	struct ino_bucket *bucket;
279 	unsigned int irq;
280 
281 	bucket = &ivector_table[sysino];
282 	irq = bucket_get_irq(__pa(bucket));
283 
284 	return irq;
285 }
286 
ack_bad_irq(unsigned int irq)287 void ack_bad_irq(unsigned int irq)
288 {
289 	pr_crit("BAD IRQ ack %d\n", irq);
290 }
291 
irq_install_pre_handler(int irq,void (* func)(unsigned int,void *,void *),void * arg1,void * arg2)292 void irq_install_pre_handler(int irq,
293 			     void (*func)(unsigned int, void *, void *),
294 			     void *arg1, void *arg2)
295 {
296 	pr_warn("IRQ pre handler NOT supported.\n");
297 }
298 
299 /*
300  * /proc/interrupts printing:
301  */
arch_show_interrupts(struct seq_file * p,int prec)302 int arch_show_interrupts(struct seq_file *p, int prec)
303 {
304 	int j;
305 
306 	seq_printf(p, "NMI: ");
307 	for_each_online_cpu(j)
308 		seq_printf(p, "%10u ", cpu_data(j).__nmi_count);
309 	seq_printf(p, "     Non-maskable interrupts\n");
310 	return 0;
311 }
312 
sun4u_compute_tid(unsigned long imap,unsigned long cpuid)313 static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid)
314 {
315 	unsigned int tid;
316 
317 	if (this_is_starfire) {
318 		tid = starfire_translate(imap, cpuid);
319 		tid <<= IMAP_TID_SHIFT;
320 		tid &= IMAP_TID_UPA;
321 	} else {
322 		if (tlb_type == cheetah || tlb_type == cheetah_plus) {
323 			unsigned long ver;
324 
325 			__asm__ ("rdpr %%ver, %0" : "=r" (ver));
326 			if ((ver >> 32UL) == __JALAPENO_ID ||
327 			    (ver >> 32UL) == __SERRANO_ID) {
328 				tid = cpuid << IMAP_TID_SHIFT;
329 				tid &= IMAP_TID_JBUS;
330 			} else {
331 				unsigned int a = cpuid & 0x1f;
332 				unsigned int n = (cpuid >> 5) & 0x1f;
333 
334 				tid = ((a << IMAP_AID_SHIFT) |
335 				       (n << IMAP_NID_SHIFT));
336 				tid &= (IMAP_AID_SAFARI |
337 					IMAP_NID_SAFARI);
338 			}
339 		} else {
340 			tid = cpuid << IMAP_TID_SHIFT;
341 			tid &= IMAP_TID_UPA;
342 		}
343 	}
344 
345 	return tid;
346 }
347 
348 #ifdef CONFIG_SMP
irq_choose_cpu(unsigned int irq,const struct cpumask * affinity)349 static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity)
350 {
351 	cpumask_t mask;
352 	int cpuid;
353 
354 	cpumask_copy(&mask, affinity);
355 	if (cpumask_equal(&mask, cpu_online_mask)) {
356 		cpuid = map_to_cpu(irq);
357 	} else {
358 		cpumask_t tmp;
359 
360 		cpumask_and(&tmp, cpu_online_mask, &mask);
361 		cpuid = cpumask_empty(&tmp) ? map_to_cpu(irq) : cpumask_first(&tmp);
362 	}
363 
364 	return cpuid;
365 }
366 #else
367 #define irq_choose_cpu(irq, affinity)	\
368 	real_hard_smp_processor_id()
369 #endif
370 
sun4u_irq_enable(struct irq_data * data)371 static void sun4u_irq_enable(struct irq_data *data)
372 {
373 	struct irq_handler_data *handler_data;
374 
375 	handler_data = irq_data_get_irq_handler_data(data);
376 	if (likely(handler_data)) {
377 		unsigned long cpuid, imap, val;
378 		unsigned int tid;
379 
380 		cpuid = irq_choose_cpu(data->irq,
381 				       irq_data_get_affinity_mask(data));
382 		imap = handler_data->imap;
383 
384 		tid = sun4u_compute_tid(imap, cpuid);
385 
386 		val = upa_readq(imap);
387 		val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS |
388 			 IMAP_AID_SAFARI | IMAP_NID_SAFARI);
389 		val |= tid | IMAP_VALID;
390 		upa_writeq(val, imap);
391 		upa_writeq(ICLR_IDLE, handler_data->iclr);
392 	}
393 }
394 
sun4u_set_affinity(struct irq_data * data,const struct cpumask * mask,bool force)395 static int sun4u_set_affinity(struct irq_data *data,
396 			       const struct cpumask *mask, bool force)
397 {
398 	struct irq_handler_data *handler_data;
399 
400 	handler_data = irq_data_get_irq_handler_data(data);
401 	if (likely(handler_data)) {
402 		unsigned long cpuid, imap, val;
403 		unsigned int tid;
404 
405 		cpuid = irq_choose_cpu(data->irq, mask);
406 		imap = handler_data->imap;
407 
408 		tid = sun4u_compute_tid(imap, cpuid);
409 
410 		val = upa_readq(imap);
411 		val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS |
412 			 IMAP_AID_SAFARI | IMAP_NID_SAFARI);
413 		val |= tid | IMAP_VALID;
414 		upa_writeq(val, imap);
415 		upa_writeq(ICLR_IDLE, handler_data->iclr);
416 	}
417 
418 	return 0;
419 }
420 
421 /* Don't do anything.  The desc->status check for IRQ_DISABLED in
422  * handler_irq() will skip the handler call and that will leave the
423  * interrupt in the sent state.  The next ->enable() call will hit the
424  * ICLR register to reset the state machine.
425  *
426  * This scheme is necessary, instead of clearing the Valid bit in the
427  * IMAP register, to handle the case of IMAP registers being shared by
428  * multiple INOs (and thus ICLR registers).  Since we use a different
429  * virtual IRQ for each shared IMAP instance, the generic code thinks
430  * there is only one user so it prematurely calls ->disable() on
431  * free_irq().
432  *
433  * We have to provide an explicit ->disable() method instead of using
434  * NULL to get the default.  The reason is that if the generic code
435  * sees that, it also hooks up a default ->shutdown method which
436  * invokes ->mask() which we do not want.  See irq_chip_set_defaults().
437  */
sun4u_irq_disable(struct irq_data * data)438 static void sun4u_irq_disable(struct irq_data *data)
439 {
440 }
441 
sun4u_irq_eoi(struct irq_data * data)442 static void sun4u_irq_eoi(struct irq_data *data)
443 {
444 	struct irq_handler_data *handler_data;
445 
446 	handler_data = irq_data_get_irq_handler_data(data);
447 	if (likely(handler_data))
448 		upa_writeq(ICLR_IDLE, handler_data->iclr);
449 }
450 
sun4v_irq_enable(struct irq_data * data)451 static void sun4v_irq_enable(struct irq_data *data)
452 {
453 	unsigned long cpuid = irq_choose_cpu(data->irq,
454 					     irq_data_get_affinity_mask(data));
455 	unsigned int ino = irq_data_to_sysino(data);
456 	int err;
457 
458 	err = sun4v_intr_settarget(ino, cpuid);
459 	if (err != HV_EOK)
460 		printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): "
461 		       "err(%d)\n", ino, cpuid, err);
462 	err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
463 	if (err != HV_EOK)
464 		printk(KERN_ERR "sun4v_intr_setstate(%x): "
465 		       "err(%d)\n", ino, err);
466 	err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
467 	if (err != HV_EOK)
468 		printk(KERN_ERR "sun4v_intr_setenabled(%x): err(%d)\n",
469 		       ino, err);
470 }
471 
sun4v_set_affinity(struct irq_data * data,const struct cpumask * mask,bool force)472 static int sun4v_set_affinity(struct irq_data *data,
473 			       const struct cpumask *mask, bool force)
474 {
475 	unsigned long cpuid = irq_choose_cpu(data->irq, mask);
476 	unsigned int ino = irq_data_to_sysino(data);
477 	int err;
478 
479 	err = sun4v_intr_settarget(ino, cpuid);
480 	if (err != HV_EOK)
481 		printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): "
482 		       "err(%d)\n", ino, cpuid, err);
483 
484 	return 0;
485 }
486 
sun4v_irq_disable(struct irq_data * data)487 static void sun4v_irq_disable(struct irq_data *data)
488 {
489 	unsigned int ino = irq_data_to_sysino(data);
490 	int err;
491 
492 	err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
493 	if (err != HV_EOK)
494 		printk(KERN_ERR "sun4v_intr_setenabled(%x): "
495 		       "err(%d)\n", ino, err);
496 }
497 
sun4v_irq_eoi(struct irq_data * data)498 static void sun4v_irq_eoi(struct irq_data *data)
499 {
500 	unsigned int ino = irq_data_to_sysino(data);
501 	int err;
502 
503 	err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
504 	if (err != HV_EOK)
505 		printk(KERN_ERR "sun4v_intr_setstate(%x): "
506 		       "err(%d)\n", ino, err);
507 }
508 
sun4v_virq_enable(struct irq_data * data)509 static void sun4v_virq_enable(struct irq_data *data)
510 {
511 	unsigned long dev_handle = irq_data_to_handle(data);
512 	unsigned long dev_ino = irq_data_to_ino(data);
513 	unsigned long cpuid;
514 	int err;
515 
516 	cpuid = irq_choose_cpu(data->irq, irq_data_get_affinity_mask(data));
517 
518 	err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
519 	if (err != HV_EOK)
520 		printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
521 		       "err(%d)\n",
522 		       dev_handle, dev_ino, cpuid, err);
523 	err = sun4v_vintr_set_state(dev_handle, dev_ino,
524 				    HV_INTR_STATE_IDLE);
525 	if (err != HV_EOK)
526 		printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
527 		       "HV_INTR_STATE_IDLE): err(%d)\n",
528 		       dev_handle, dev_ino, err);
529 	err = sun4v_vintr_set_valid(dev_handle, dev_ino,
530 				    HV_INTR_ENABLED);
531 	if (err != HV_EOK)
532 		printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
533 		       "HV_INTR_ENABLED): err(%d)\n",
534 		       dev_handle, dev_ino, err);
535 }
536 
sun4v_virt_set_affinity(struct irq_data * data,const struct cpumask * mask,bool force)537 static int sun4v_virt_set_affinity(struct irq_data *data,
538 				    const struct cpumask *mask, bool force)
539 {
540 	unsigned long dev_handle = irq_data_to_handle(data);
541 	unsigned long dev_ino = irq_data_to_ino(data);
542 	unsigned long cpuid;
543 	int err;
544 
545 	cpuid = irq_choose_cpu(data->irq, mask);
546 
547 	err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
548 	if (err != HV_EOK)
549 		printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
550 		       "err(%d)\n",
551 		       dev_handle, dev_ino, cpuid, err);
552 
553 	return 0;
554 }
555 
sun4v_virq_disable(struct irq_data * data)556 static void sun4v_virq_disable(struct irq_data *data)
557 {
558 	unsigned long dev_handle = irq_data_to_handle(data);
559 	unsigned long dev_ino = irq_data_to_ino(data);
560 	int err;
561 
562 
563 	err = sun4v_vintr_set_valid(dev_handle, dev_ino,
564 				    HV_INTR_DISABLED);
565 	if (err != HV_EOK)
566 		printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
567 		       "HV_INTR_DISABLED): err(%d)\n",
568 		       dev_handle, dev_ino, err);
569 }
570 
sun4v_virq_eoi(struct irq_data * data)571 static void sun4v_virq_eoi(struct irq_data *data)
572 {
573 	unsigned long dev_handle = irq_data_to_handle(data);
574 	unsigned long dev_ino = irq_data_to_ino(data);
575 	int err;
576 
577 	err = sun4v_vintr_set_state(dev_handle, dev_ino,
578 				    HV_INTR_STATE_IDLE);
579 	if (err != HV_EOK)
580 		printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx,"
581 		       "HV_INTR_STATE_IDLE): err(%d)\n",
582 		       dev_handle, dev_ino, err);
583 }
584 
585 static struct irq_chip sun4u_irq = {
586 	.name			= "sun4u",
587 	.irq_enable		= sun4u_irq_enable,
588 	.irq_disable		= sun4u_irq_disable,
589 	.irq_eoi		= sun4u_irq_eoi,
590 	.irq_set_affinity	= sun4u_set_affinity,
591 	.flags			= IRQCHIP_EOI_IF_HANDLED,
592 };
593 
594 static struct irq_chip sun4v_irq = {
595 	.name			= "sun4v",
596 	.irq_enable		= sun4v_irq_enable,
597 	.irq_disable		= sun4v_irq_disable,
598 	.irq_eoi		= sun4v_irq_eoi,
599 	.irq_set_affinity	= sun4v_set_affinity,
600 	.flags			= IRQCHIP_EOI_IF_HANDLED,
601 };
602 
603 static struct irq_chip sun4v_virq = {
604 	.name			= "vsun4v",
605 	.irq_enable		= sun4v_virq_enable,
606 	.irq_disable		= sun4v_virq_disable,
607 	.irq_eoi		= sun4v_virq_eoi,
608 	.irq_set_affinity	= sun4v_virt_set_affinity,
609 	.flags			= IRQCHIP_EOI_IF_HANDLED,
610 };
611 
build_irq(int inofixup,unsigned long iclr,unsigned long imap)612 unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap)
613 {
614 	struct irq_handler_data *handler_data;
615 	struct ino_bucket *bucket;
616 	unsigned int irq;
617 	int ino;
618 
619 	BUG_ON(tlb_type == hypervisor);
620 
621 	ino = (upa_readq(imap) & (IMAP_IGN | IMAP_INO)) + inofixup;
622 	bucket = &ivector_table[ino];
623 	irq = bucket_get_irq(__pa(bucket));
624 	if (!irq) {
625 		irq = irq_alloc(0, ino);
626 		bucket_set_irq(__pa(bucket), irq);
627 		irq_set_chip_and_handler_name(irq, &sun4u_irq,
628 					      handle_fasteoi_irq, "IVEC");
629 	}
630 
631 	handler_data = irq_get_handler_data(irq);
632 	if (unlikely(handler_data))
633 		goto out;
634 
635 	handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
636 	if (unlikely(!handler_data)) {
637 		prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
638 		prom_halt();
639 	}
640 	irq_set_handler_data(irq, handler_data);
641 
642 	handler_data->imap  = imap;
643 	handler_data->iclr  = iclr;
644 
645 out:
646 	return irq;
647 }
648 
sun4v_build_common(u32 devhandle,unsigned int devino,void (* handler_data_init)(struct irq_handler_data * data,u32 devhandle,unsigned int devino),struct irq_chip * chip)649 static unsigned int sun4v_build_common(u32 devhandle, unsigned int devino,
650 		void (*handler_data_init)(struct irq_handler_data *data,
651 		u32 devhandle, unsigned int devino),
652 		struct irq_chip *chip)
653 {
654 	struct irq_handler_data *data;
655 	unsigned int irq;
656 
657 	irq = irq_alloc(devhandle, devino);
658 	if (!irq)
659 		goto out;
660 
661 	data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
662 	if (unlikely(!data)) {
663 		pr_err("IRQ handler data allocation failed.\n");
664 		irq_free(irq);
665 		irq = 0;
666 		goto out;
667 	}
668 
669 	irq_set_handler_data(irq, data);
670 	handler_data_init(data, devhandle, devino);
671 	irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, "IVEC");
672 	data->imap = ~0UL;
673 	data->iclr = ~0UL;
674 out:
675 	return irq;
676 }
677 
cookie_assign(unsigned int irq,u32 devhandle,unsigned int devino)678 static unsigned long cookie_assign(unsigned int irq, u32 devhandle,
679 		unsigned int devino)
680 {
681 	struct irq_handler_data *ihd = irq_get_handler_data(irq);
682 	unsigned long hv_error, cookie;
683 
684 	/* handler_irq needs to find the irq. cookie is seen signed in
685 	 * sun4v_dev_mondo and treated as a non ivector_table delivery.
686 	 */
687 	ihd->bucket.__irq = irq;
688 	cookie = ~__pa(&ihd->bucket);
689 
690 	hv_error = sun4v_vintr_set_cookie(devhandle, devino, cookie);
691 	if (hv_error)
692 		pr_err("HV vintr set cookie failed = %ld\n", hv_error);
693 
694 	return hv_error;
695 }
696 
cookie_handler_data(struct irq_handler_data * data,u32 devhandle,unsigned int devino)697 static void cookie_handler_data(struct irq_handler_data *data,
698 				u32 devhandle, unsigned int devino)
699 {
700 	data->dev_handle = devhandle;
701 	data->dev_ino = devino;
702 }
703 
cookie_build_irq(u32 devhandle,unsigned int devino,struct irq_chip * chip)704 static unsigned int cookie_build_irq(u32 devhandle, unsigned int devino,
705 				     struct irq_chip *chip)
706 {
707 	unsigned long hv_error;
708 	unsigned int irq;
709 
710 	irq = sun4v_build_common(devhandle, devino, cookie_handler_data, chip);
711 
712 	hv_error = cookie_assign(irq, devhandle, devino);
713 	if (hv_error) {
714 		irq_free(irq);
715 		irq = 0;
716 	}
717 
718 	return irq;
719 }
720 
sun4v_build_cookie(u32 devhandle,unsigned int devino)721 static unsigned int sun4v_build_cookie(u32 devhandle, unsigned int devino)
722 {
723 	unsigned int irq;
724 
725 	irq = cookie_exists(devhandle, devino);
726 	if (irq)
727 		goto out;
728 
729 	irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
730 
731 out:
732 	return irq;
733 }
734 
sysino_set_bucket(unsigned int irq)735 static void sysino_set_bucket(unsigned int irq)
736 {
737 	struct irq_handler_data *ihd = irq_get_handler_data(irq);
738 	struct ino_bucket *bucket;
739 	unsigned long sysino;
740 
741 	sysino = sun4v_devino_to_sysino(ihd->dev_handle, ihd->dev_ino);
742 	BUG_ON(sysino >= nr_ivec);
743 	bucket = &ivector_table[sysino];
744 	bucket_set_irq(__pa(bucket), irq);
745 }
746 
sysino_handler_data(struct irq_handler_data * data,u32 devhandle,unsigned int devino)747 static void sysino_handler_data(struct irq_handler_data *data,
748 				u32 devhandle, unsigned int devino)
749 {
750 	unsigned long sysino;
751 
752 	sysino = sun4v_devino_to_sysino(devhandle, devino);
753 	data->sysino = sysino;
754 }
755 
sysino_build_irq(u32 devhandle,unsigned int devino,struct irq_chip * chip)756 static unsigned int sysino_build_irq(u32 devhandle, unsigned int devino,
757 				     struct irq_chip *chip)
758 {
759 	unsigned int irq;
760 
761 	irq = sun4v_build_common(devhandle, devino, sysino_handler_data, chip);
762 	if (!irq)
763 		goto out;
764 
765 	sysino_set_bucket(irq);
766 out:
767 	return irq;
768 }
769 
sun4v_build_sysino(u32 devhandle,unsigned int devino)770 static int sun4v_build_sysino(u32 devhandle, unsigned int devino)
771 {
772 	int irq;
773 
774 	irq = sysino_exists(devhandle, devino);
775 	if (irq)
776 		goto out;
777 
778 	irq = sysino_build_irq(devhandle, devino, &sun4v_irq);
779 out:
780 	return irq;
781 }
782 
sun4v_build_irq(u32 devhandle,unsigned int devino)783 unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino)
784 {
785 	unsigned int irq;
786 
787 	if (sun4v_cookie_only_virqs())
788 		irq = sun4v_build_cookie(devhandle, devino);
789 	else
790 		irq = sun4v_build_sysino(devhandle, devino);
791 
792 	return irq;
793 }
794 
sun4v_build_virq(u32 devhandle,unsigned int devino)795 unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino)
796 {
797 	int irq;
798 
799 	irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
800 	if (!irq)
801 		goto out;
802 
803 	/* This is borrowed from the original function.
804 	 */
805 	irq_set_status_flags(irq, IRQ_NOAUTOEN);
806 
807 out:
808 	return irq;
809 }
810 
811 void *hardirq_stack[NR_CPUS];
812 void *softirq_stack[NR_CPUS];
813 
handler_irq(int pil,struct pt_regs * regs)814 void __irq_entry handler_irq(int pil, struct pt_regs *regs)
815 {
816 	unsigned long pstate, bucket_pa;
817 	struct pt_regs *old_regs;
818 	void *orig_sp;
819 
820 	clear_softint(1 << pil);
821 
822 	old_regs = set_irq_regs(regs);
823 	irq_enter();
824 
825 	/* Grab an atomic snapshot of the pending IVECs.  */
826 	__asm__ __volatile__("rdpr	%%pstate, %0\n\t"
827 			     "wrpr	%0, %3, %%pstate\n\t"
828 			     "ldx	[%2], %1\n\t"
829 			     "stx	%%g0, [%2]\n\t"
830 			     "wrpr	%0, 0x0, %%pstate\n\t"
831 			     : "=&r" (pstate), "=&r" (bucket_pa)
832 			     : "r" (irq_work_pa(smp_processor_id())),
833 			       "i" (PSTATE_IE)
834 			     : "memory");
835 
836 	orig_sp = set_hardirq_stack();
837 
838 	while (bucket_pa) {
839 		unsigned long next_pa;
840 		unsigned int irq;
841 
842 		next_pa = bucket_get_chain_pa(bucket_pa);
843 		irq = bucket_get_irq(bucket_pa);
844 		bucket_clear_chain_pa(bucket_pa);
845 
846 		generic_handle_irq(irq);
847 
848 		bucket_pa = next_pa;
849 	}
850 
851 	restore_hardirq_stack(orig_sp);
852 
853 	irq_exit();
854 	set_irq_regs(old_regs);
855 }
856 
do_softirq_own_stack(void)857 void do_softirq_own_stack(void)
858 {
859 	void *orig_sp, *sp = softirq_stack[smp_processor_id()];
860 
861 	sp += THREAD_SIZE - 192 - STACK_BIAS;
862 
863 	__asm__ __volatile__("mov %%sp, %0\n\t"
864 			     "mov %1, %%sp"
865 			     : "=&r" (orig_sp)
866 			     : "r" (sp));
867 	__do_softirq();
868 	__asm__ __volatile__("mov %0, %%sp"
869 			     : : "r" (orig_sp));
870 }
871 
872 #ifdef CONFIG_HOTPLUG_CPU
fixup_irqs(void)873 void fixup_irqs(void)
874 {
875 	unsigned int irq;
876 
877 	for (irq = 0; irq < NR_IRQS; irq++) {
878 		struct irq_desc *desc = irq_to_desc(irq);
879 		struct irq_data *data;
880 		unsigned long flags;
881 
882 		if (!desc)
883 			continue;
884 		data = irq_desc_get_irq_data(desc);
885 		raw_spin_lock_irqsave(&desc->lock, flags);
886 		if (desc->action && !irqd_is_per_cpu(data)) {
887 			if (data->chip->irq_set_affinity)
888 				data->chip->irq_set_affinity(data,
889 					irq_data_get_affinity_mask(data),
890 					false);
891 		}
892 		raw_spin_unlock_irqrestore(&desc->lock, flags);
893 	}
894 
895 	tick_ops->disable_irq();
896 }
897 #endif
898 
899 struct sun5_timer {
900 	u64	count0;
901 	u64	limit0;
902 	u64	count1;
903 	u64	limit1;
904 };
905 
906 static struct sun5_timer *prom_timers;
907 static u64 prom_limit0, prom_limit1;
908 
map_prom_timers(void)909 static void map_prom_timers(void)
910 {
911 	struct device_node *dp;
912 	const unsigned int *addr;
913 
914 	/* PROM timer node hangs out in the top level of device siblings... */
915 	dp = of_find_node_by_path("/");
916 	dp = dp->child;
917 	while (dp) {
918 		if (!strcmp(dp->name, "counter-timer"))
919 			break;
920 		dp = dp->sibling;
921 	}
922 
923 	/* Assume if node is not present, PROM uses different tick mechanism
924 	 * which we should not care about.
925 	 */
926 	if (!dp) {
927 		prom_timers = (struct sun5_timer *) 0;
928 		return;
929 	}
930 
931 	/* If PROM is really using this, it must be mapped by him. */
932 	addr = of_get_property(dp, "address", NULL);
933 	if (!addr) {
934 		prom_printf("PROM does not have timer mapped, trying to continue.\n");
935 		prom_timers = (struct sun5_timer *) 0;
936 		return;
937 	}
938 	prom_timers = (struct sun5_timer *) ((unsigned long)addr[0]);
939 }
940 
kill_prom_timer(void)941 static void kill_prom_timer(void)
942 {
943 	if (!prom_timers)
944 		return;
945 
946 	/* Save them away for later. */
947 	prom_limit0 = prom_timers->limit0;
948 	prom_limit1 = prom_timers->limit1;
949 
950 	/* Just as in sun4c PROM uses timer which ticks at IRQ 14.
951 	 * We turn both off here just to be paranoid.
952 	 */
953 	prom_timers->limit0 = 0;
954 	prom_timers->limit1 = 0;
955 
956 	/* Wheee, eat the interrupt packet too... */
957 	__asm__ __volatile__(
958 "	mov	0x40, %%g2\n"
959 "	ldxa	[%%g0] %0, %%g1\n"
960 "	ldxa	[%%g2] %1, %%g1\n"
961 "	stxa	%%g0, [%%g0] %0\n"
962 "	membar	#Sync\n"
963 	: /* no outputs */
964 	: "i" (ASI_INTR_RECEIVE), "i" (ASI_INTR_R)
965 	: "g1", "g2");
966 }
967 
init_irqwork_curcpu(void)968 void notrace init_irqwork_curcpu(void)
969 {
970 	int cpu = hard_smp_processor_id();
971 
972 	trap_block[cpu].irq_worklist_pa = 0UL;
973 }
974 
975 /* Please be very careful with register_one_mondo() and
976  * sun4v_register_mondo_queues().
977  *
978  * On SMP this gets invoked from the CPU trampoline before
979  * the cpu has fully taken over the trap table from OBP,
980  * and it's kernel stack + %g6 thread register state is
981  * not fully cooked yet.
982  *
983  * Therefore you cannot make any OBP calls, not even prom_printf,
984  * from these two routines.
985  */
register_one_mondo(unsigned long paddr,unsigned long type,unsigned long qmask)986 static void notrace register_one_mondo(unsigned long paddr, unsigned long type,
987 				       unsigned long qmask)
988 {
989 	unsigned long num_entries = (qmask + 1) / 64;
990 	unsigned long status;
991 
992 	status = sun4v_cpu_qconf(type, paddr, num_entries);
993 	if (status != HV_EOK) {
994 		prom_printf("SUN4V: sun4v_cpu_qconf(%lu:%lx:%lu) failed, "
995 			    "err %lu\n", type, paddr, num_entries, status);
996 		prom_halt();
997 	}
998 }
999 
sun4v_register_mondo_queues(int this_cpu)1000 void notrace sun4v_register_mondo_queues(int this_cpu)
1001 {
1002 	struct trap_per_cpu *tb = &trap_block[this_cpu];
1003 
1004 	register_one_mondo(tb->cpu_mondo_pa, HV_CPU_QUEUE_CPU_MONDO,
1005 			   tb->cpu_mondo_qmask);
1006 	register_one_mondo(tb->dev_mondo_pa, HV_CPU_QUEUE_DEVICE_MONDO,
1007 			   tb->dev_mondo_qmask);
1008 	register_one_mondo(tb->resum_mondo_pa, HV_CPU_QUEUE_RES_ERROR,
1009 			   tb->resum_qmask);
1010 	register_one_mondo(tb->nonresum_mondo_pa, HV_CPU_QUEUE_NONRES_ERROR,
1011 			   tb->nonresum_qmask);
1012 }
1013 
1014 /* Each queue region must be a power of 2 multiple of 64 bytes in
1015  * size.  The base real address must be aligned to the size of the
1016  * region.  Thus, an 8KB queue must be 8KB aligned, for example.
1017  */
alloc_one_queue(unsigned long * pa_ptr,unsigned long qmask)1018 static void __init alloc_one_queue(unsigned long *pa_ptr, unsigned long qmask)
1019 {
1020 	unsigned long size = PAGE_ALIGN(qmask + 1);
1021 	unsigned long order = get_order(size);
1022 	unsigned long p;
1023 
1024 	p = __get_free_pages(GFP_KERNEL, order);
1025 	if (!p) {
1026 		prom_printf("SUN4V: Error, cannot allocate queue.\n");
1027 		prom_halt();
1028 	}
1029 
1030 	*pa_ptr = __pa(p);
1031 }
1032 
init_cpu_send_mondo_info(struct trap_per_cpu * tb)1033 static void __init init_cpu_send_mondo_info(struct trap_per_cpu *tb)
1034 {
1035 #ifdef CONFIG_SMP
1036 	unsigned long page;
1037 	void *mondo, *p;
1038 
1039 	BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > PAGE_SIZE);
1040 
1041 	/* Make sure mondo block is 64byte aligned */
1042 	p = kzalloc(127, GFP_KERNEL);
1043 	if (!p) {
1044 		prom_printf("SUN4V: Error, cannot allocate mondo block.\n");
1045 		prom_halt();
1046 	}
1047 	mondo = (void *)(((unsigned long)p + 63) & ~0x3f);
1048 	tb->cpu_mondo_block_pa = __pa(mondo);
1049 
1050 	page = get_zeroed_page(GFP_KERNEL);
1051 	if (!page) {
1052 		prom_printf("SUN4V: Error, cannot allocate cpu list page.\n");
1053 		prom_halt();
1054 	}
1055 
1056 	tb->cpu_list_pa = __pa(page);
1057 #endif
1058 }
1059 
1060 /* Allocate mondo and error queues for all possible cpus.  */
sun4v_init_mondo_queues(void)1061 static void __init sun4v_init_mondo_queues(void)
1062 {
1063 	int cpu;
1064 
1065 	for_each_possible_cpu(cpu) {
1066 		struct trap_per_cpu *tb = &trap_block[cpu];
1067 
1068 		alloc_one_queue(&tb->cpu_mondo_pa, tb->cpu_mondo_qmask);
1069 		alloc_one_queue(&tb->dev_mondo_pa, tb->dev_mondo_qmask);
1070 		alloc_one_queue(&tb->resum_mondo_pa, tb->resum_qmask);
1071 		alloc_one_queue(&tb->resum_kernel_buf_pa, tb->resum_qmask);
1072 		alloc_one_queue(&tb->nonresum_mondo_pa, tb->nonresum_qmask);
1073 		alloc_one_queue(&tb->nonresum_kernel_buf_pa,
1074 				tb->nonresum_qmask);
1075 	}
1076 }
1077 
init_send_mondo_info(void)1078 static void __init init_send_mondo_info(void)
1079 {
1080 	int cpu;
1081 
1082 	for_each_possible_cpu(cpu) {
1083 		struct trap_per_cpu *tb = &trap_block[cpu];
1084 
1085 		init_cpu_send_mondo_info(tb);
1086 	}
1087 }
1088 
1089 static struct irqaction timer_irq_action = {
1090 	.name = "timer",
1091 };
1092 
irq_ivector_init(void)1093 static void __init irq_ivector_init(void)
1094 {
1095 	unsigned long size, order;
1096 	unsigned int ivecs;
1097 
1098 	/* If we are doing cookie only VIRQs then we do not need the ivector
1099 	 * table to process interrupts.
1100 	 */
1101 	if (sun4v_cookie_only_virqs())
1102 		return;
1103 
1104 	ivecs = size_nr_ivec();
1105 	size = sizeof(struct ino_bucket) * ivecs;
1106 	order = get_order(size);
1107 	ivector_table = (struct ino_bucket *)
1108 		__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
1109 	if (!ivector_table) {
1110 		prom_printf("Fatal error, cannot allocate ivector_table\n");
1111 		prom_halt();
1112 	}
1113 	__flush_dcache_range((unsigned long) ivector_table,
1114 			     ((unsigned long) ivector_table) + size);
1115 
1116 	ivector_table_pa = __pa(ivector_table);
1117 }
1118 
1119 /* Only invoked on boot processor.*/
init_IRQ(void)1120 void __init init_IRQ(void)
1121 {
1122 	irq_init_hv();
1123 	irq_ivector_init();
1124 	map_prom_timers();
1125 	kill_prom_timer();
1126 
1127 	if (tlb_type == hypervisor)
1128 		sun4v_init_mondo_queues();
1129 
1130 	init_send_mondo_info();
1131 
1132 	if (tlb_type == hypervisor) {
1133 		/* Load up the boot cpu's entries.  */
1134 		sun4v_register_mondo_queues(hard_smp_processor_id());
1135 	}
1136 
1137 	/* We need to clear any IRQ's pending in the soft interrupt
1138 	 * registers, a spurious one could be left around from the
1139 	 * PROM timer which we just disabled.
1140 	 */
1141 	clear_softint(get_softint());
1142 
1143 	/* Now that ivector table is initialized, it is safe
1144 	 * to receive IRQ vector traps.  We will normally take
1145 	 * one or two right now, in case some device PROM used
1146 	 * to boot us wants to speak to us.  We just ignore them.
1147 	 */
1148 	__asm__ __volatile__("rdpr	%%pstate, %%g1\n\t"
1149 			     "or	%%g1, %0, %%g1\n\t"
1150 			     "wrpr	%%g1, 0x0, %%pstate"
1151 			     : /* No outputs */
1152 			     : "i" (PSTATE_IE)
1153 			     : "g1");
1154 
1155 	irq_to_desc(0)->action = &timer_irq_action;
1156 }
1157