1 #ifndef __ALPHA_SYSTEM_H
2 #define __ALPHA_SYSTEM_H
3
4 #include <asm/pal.h>
5 #include <asm/page.h>
6 #include <asm/barrier.h>
7
8 /*
9 * System defines.. Note that this is included both from .c and .S
10 * files, so it does only defines, not any C code.
11 */
12
13 /*
14 * We leave one page for the initial stack page, and one page for
15 * the initial process structure. Also, the console eats 3 MB for
16 * the initial bootloader (one of which we can reclaim later).
17 */
18 #define BOOT_PCB 0x20000000
19 #define BOOT_ADDR 0x20000000
20 /* Remove when official MILO sources have ELF support: */
21 #define BOOT_SIZE (16*1024)
22
23 #ifdef CONFIG_ALPHA_LEGACY_START_ADDRESS
24 #define KERNEL_START_PHYS 0x300000 /* Old bootloaders hardcoded this. */
25 #else
26 #define KERNEL_START_PHYS 0x1000000 /* required: Wildfire/Titan/Marvel */
27 #endif
28
29 #define KERNEL_START (PAGE_OFFSET+KERNEL_START_PHYS)
30 #define SWAPPER_PGD KERNEL_START
31 #define INIT_STACK (PAGE_OFFSET+KERNEL_START_PHYS+0x02000)
32 #define EMPTY_PGT (PAGE_OFFSET+KERNEL_START_PHYS+0x04000)
33 #define EMPTY_PGE (PAGE_OFFSET+KERNEL_START_PHYS+0x08000)
34 #define ZERO_PGE (PAGE_OFFSET+KERNEL_START_PHYS+0x0A000)
35
36 #define START_ADDR (PAGE_OFFSET+KERNEL_START_PHYS+0x10000)
37
38 /*
39 * This is setup by the secondary bootstrap loader. Because
40 * the zero page is zeroed out as soon as the vm system is
41 * initialized, we need to copy things out into a more permanent
42 * place.
43 */
44 #define PARAM ZERO_PGE
45 #define COMMAND_LINE ((char*)(PARAM + 0x0000))
46 #define INITRD_START (*(unsigned long *) (PARAM+0x100))
47 #define INITRD_SIZE (*(unsigned long *) (PARAM+0x108))
48
49 #ifndef __ASSEMBLY__
50 #include <linux/kernel.h>
51 #define AT_VECTOR_SIZE_ARCH 4 /* entries in ARCH_DLINFO */
52
53 /*
54 * This is the logout header that should be common to all platforms
55 * (assuming they are running OSF/1 PALcode, I guess).
56 */
57 struct el_common {
58 unsigned int size; /* size in bytes of logout area */
59 unsigned int sbz1 : 30; /* should be zero */
60 unsigned int err2 : 1; /* second error */
61 unsigned int retry : 1; /* retry flag */
62 unsigned int proc_offset; /* processor-specific offset */
63 unsigned int sys_offset; /* system-specific offset */
64 unsigned int code; /* machine check code */
65 unsigned int frame_rev; /* frame revision */
66 };
67
68 /* Machine Check Frame for uncorrectable errors (Large format)
69 * --- This is used to log uncorrectable errors such as
70 * double bit ECC errors.
71 * --- These errors are detected by both processor and systems.
72 */
73 struct el_common_EV5_uncorrectable_mcheck {
74 unsigned long shadow[8]; /* Shadow reg. 8-14, 25 */
75 unsigned long paltemp[24]; /* PAL TEMP REGS. */
76 unsigned long exc_addr; /* Address of excepting instruction*/
77 unsigned long exc_sum; /* Summary of arithmetic traps. */
78 unsigned long exc_mask; /* Exception mask (from exc_sum). */
79 unsigned long pal_base; /* Base address for PALcode. */
80 unsigned long isr; /* Interrupt Status Reg. */
81 unsigned long icsr; /* CURRENT SETUP OF EV5 IBOX */
82 unsigned long ic_perr_stat; /* I-CACHE Reg. <11> set Data parity
83 <12> set TAG parity*/
84 unsigned long dc_perr_stat; /* D-CACHE error Reg. Bits set to 1:
85 <2> Data error in bank 0
86 <3> Data error in bank 1
87 <4> Tag error in bank 0
88 <5> Tag error in bank 1 */
89 unsigned long va; /* Effective VA of fault or miss. */
90 unsigned long mm_stat; /* Holds the reason for D-stream
91 fault or D-cache parity errors */
92 unsigned long sc_addr; /* Address that was being accessed
93 when EV5 detected Secondary cache
94 failure. */
95 unsigned long sc_stat; /* Helps determine if the error was
96 TAG/Data parity(Secondary Cache)*/
97 unsigned long bc_tag_addr; /* Contents of EV5 BC_TAG_ADDR */
98 unsigned long ei_addr; /* Physical address of any transfer
99 that is logged in EV5 EI_STAT */
100 unsigned long fill_syndrome; /* For correcting ECC errors. */
101 unsigned long ei_stat; /* Helps identify reason of any
102 processor uncorrectable error
103 at its external interface. */
104 unsigned long ld_lock; /* Contents of EV5 LD_LOCK register*/
105 };
106
107 struct el_common_EV6_mcheck {
108 unsigned int FrameSize; /* Bytes, including this field */
109 unsigned int FrameFlags; /* <31> = Retry, <30> = Second Error */
110 unsigned int CpuOffset; /* Offset to CPU-specific info */
111 unsigned int SystemOffset; /* Offset to system-specific info */
112 unsigned int MCHK_Code;
113 unsigned int MCHK_Frame_Rev;
114 unsigned long I_STAT; /* EV6 Internal Processor Registers */
115 unsigned long DC_STAT; /* (See the 21264 Spec) */
116 unsigned long C_ADDR;
117 unsigned long DC1_SYNDROME;
118 unsigned long DC0_SYNDROME;
119 unsigned long C_STAT;
120 unsigned long C_STS;
121 unsigned long MM_STAT;
122 unsigned long EXC_ADDR;
123 unsigned long IER_CM;
124 unsigned long ISUM;
125 unsigned long RESERVED0;
126 unsigned long PAL_BASE;
127 unsigned long I_CTL;
128 unsigned long PCTX;
129 };
130
131 extern void halt(void) __attribute__((noreturn));
132 #define __halt() __asm__ __volatile__ ("call_pal %0 #halt" : : "i" (PAL_halt))
133
134 #define switch_to(P,N,L) \
135 do { \
136 (L) = alpha_switch_to(virt_to_phys(&task_thread_info(N)->pcb), (P)); \
137 check_mmu_context(); \
138 } while (0)
139
140 struct task_struct;
141 extern struct task_struct *alpha_switch_to(unsigned long, struct task_struct*);
142
143 #define imb() \
144 __asm__ __volatile__ ("call_pal %0 #imb" : : "i" (PAL_imb) : "memory")
145
146 #define draina() \
147 __asm__ __volatile__ ("call_pal %0 #draina" : : "i" (PAL_draina) : "memory")
148
149 enum implver_enum {
150 IMPLVER_EV4,
151 IMPLVER_EV5,
152 IMPLVER_EV6
153 };
154
155 #ifdef CONFIG_ALPHA_GENERIC
156 #define implver() \
157 ({ unsigned long __implver; \
158 __asm__ ("implver %0" : "=r"(__implver)); \
159 (enum implver_enum) __implver; })
160 #else
161 /* Try to eliminate some dead code. */
162 #ifdef CONFIG_ALPHA_EV4
163 #define implver() IMPLVER_EV4
164 #endif
165 #ifdef CONFIG_ALPHA_EV5
166 #define implver() IMPLVER_EV5
167 #endif
168 #if defined(CONFIG_ALPHA_EV6)
169 #define implver() IMPLVER_EV6
170 #endif
171 #endif
172
173 enum amask_enum {
174 AMASK_BWX = (1UL << 0),
175 AMASK_FIX = (1UL << 1),
176 AMASK_CIX = (1UL << 2),
177 AMASK_MAX = (1UL << 8),
178 AMASK_PRECISE_TRAP = (1UL << 9),
179 };
180
181 #define amask(mask) \
182 ({ unsigned long __amask, __input = (mask); \
183 __asm__ ("amask %1,%0" : "=r"(__amask) : "rI"(__input)); \
184 __amask; })
185
186 #define __CALL_PAL_R0(NAME, TYPE) \
187 extern inline TYPE NAME(void) \
188 { \
189 register TYPE __r0 __asm__("$0"); \
190 __asm__ __volatile__( \
191 "call_pal %1 # " #NAME \
192 :"=r" (__r0) \
193 :"i" (PAL_ ## NAME) \
194 :"$1", "$16", "$22", "$23", "$24", "$25"); \
195 return __r0; \
196 }
197
198 #define __CALL_PAL_W1(NAME, TYPE0) \
199 extern inline void NAME(TYPE0 arg0) \
200 { \
201 register TYPE0 __r16 __asm__("$16") = arg0; \
202 __asm__ __volatile__( \
203 "call_pal %1 # "#NAME \
204 : "=r"(__r16) \
205 : "i"(PAL_ ## NAME), "0"(__r16) \
206 : "$1", "$22", "$23", "$24", "$25"); \
207 }
208
209 #define __CALL_PAL_W2(NAME, TYPE0, TYPE1) \
210 extern inline void NAME(TYPE0 arg0, TYPE1 arg1) \
211 { \
212 register TYPE0 __r16 __asm__("$16") = arg0; \
213 register TYPE1 __r17 __asm__("$17") = arg1; \
214 __asm__ __volatile__( \
215 "call_pal %2 # "#NAME \
216 : "=r"(__r16), "=r"(__r17) \
217 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17) \
218 : "$1", "$22", "$23", "$24", "$25"); \
219 }
220
221 #define __CALL_PAL_RW1(NAME, RTYPE, TYPE0) \
222 extern inline RTYPE NAME(TYPE0 arg0) \
223 { \
224 register RTYPE __r0 __asm__("$0"); \
225 register TYPE0 __r16 __asm__("$16") = arg0; \
226 __asm__ __volatile__( \
227 "call_pal %2 # "#NAME \
228 : "=r"(__r16), "=r"(__r0) \
229 : "i"(PAL_ ## NAME), "0"(__r16) \
230 : "$1", "$22", "$23", "$24", "$25"); \
231 return __r0; \
232 }
233
234 #define __CALL_PAL_RW2(NAME, RTYPE, TYPE0, TYPE1) \
235 extern inline RTYPE NAME(TYPE0 arg0, TYPE1 arg1) \
236 { \
237 register RTYPE __r0 __asm__("$0"); \
238 register TYPE0 __r16 __asm__("$16") = arg0; \
239 register TYPE1 __r17 __asm__("$17") = arg1; \
240 __asm__ __volatile__( \
241 "call_pal %3 # "#NAME \
242 : "=r"(__r16), "=r"(__r17), "=r"(__r0) \
243 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17) \
244 : "$1", "$22", "$23", "$24", "$25"); \
245 return __r0; \
246 }
247
248 __CALL_PAL_W1(cflush, unsigned long);
249 __CALL_PAL_R0(rdmces, unsigned long);
250 __CALL_PAL_R0(rdps, unsigned long);
251 __CALL_PAL_R0(rdusp, unsigned long);
252 __CALL_PAL_RW1(swpipl, unsigned long, unsigned long);
253 __CALL_PAL_R0(whami, unsigned long);
254 __CALL_PAL_W2(wrent, void*, unsigned long);
255 __CALL_PAL_W1(wripir, unsigned long);
256 __CALL_PAL_W1(wrkgp, unsigned long);
257 __CALL_PAL_W1(wrmces, unsigned long);
258 __CALL_PAL_RW2(wrperfmon, unsigned long, unsigned long, unsigned long);
259 __CALL_PAL_W1(wrusp, unsigned long);
260 __CALL_PAL_W1(wrvptptr, unsigned long);
261
262 #define IPL_MIN 0
263 #define IPL_SW0 1
264 #define IPL_SW1 2
265 #define IPL_DEV0 3
266 #define IPL_DEV1 4
267 #define IPL_TIMER 5
268 #define IPL_PERF 6
269 #define IPL_POWERFAIL 6
270 #define IPL_MCHECK 7
271 #define IPL_MAX 7
272
273 #ifdef CONFIG_ALPHA_BROKEN_IRQ_MASK
274 #undef IPL_MIN
275 #define IPL_MIN __min_ipl
276 extern int __min_ipl;
277 #endif
278
279 #define getipl() (rdps() & 7)
280 #define setipl(ipl) ((void) swpipl(ipl))
281
282 #define local_irq_disable() do { setipl(IPL_MAX); barrier(); } while(0)
283 #define local_irq_enable() do { barrier(); setipl(IPL_MIN); } while(0)
284 #define local_save_flags(flags) ((flags) = rdps())
285 #define local_irq_save(flags) do { (flags) = swpipl(IPL_MAX); barrier(); } while(0)
286 #define local_irq_restore(flags) do { barrier(); setipl(flags); barrier(); } while(0)
287
288 #define irqs_disabled() (getipl() == IPL_MAX)
289
290 /*
291 * TB routines..
292 */
293 #define __tbi(nr,arg,arg1...) \
294 ({ \
295 register unsigned long __r16 __asm__("$16") = (nr); \
296 register unsigned long __r17 __asm__("$17"); arg; \
297 __asm__ __volatile__( \
298 "call_pal %3 #__tbi" \
299 :"=r" (__r16),"=r" (__r17) \
300 :"0" (__r16),"i" (PAL_tbi) ,##arg1 \
301 :"$0", "$1", "$22", "$23", "$24", "$25"); \
302 })
303
304 #define tbi(x,y) __tbi(x,__r17=(y),"1" (__r17))
305 #define tbisi(x) __tbi(1,__r17=(x),"1" (__r17))
306 #define tbisd(x) __tbi(2,__r17=(x),"1" (__r17))
307 #define tbis(x) __tbi(3,__r17=(x),"1" (__r17))
308 #define tbiap() __tbi(-1, /* no second argument */)
309 #define tbia() __tbi(-2, /* no second argument */)
310
311 /*
312 * Atomic exchange.
313 * Since it can be used to implement critical sections
314 * it must clobber "memory" (also for interrupts in UP).
315 */
316
317 static inline unsigned long
__xchg_u8(volatile char * m,unsigned long val)318 __xchg_u8(volatile char *m, unsigned long val)
319 {
320 unsigned long ret, tmp, addr64;
321
322 __asm__ __volatile__(
323 " andnot %4,7,%3\n"
324 " insbl %1,%4,%1\n"
325 "1: ldq_l %2,0(%3)\n"
326 " extbl %2,%4,%0\n"
327 " mskbl %2,%4,%2\n"
328 " or %1,%2,%2\n"
329 " stq_c %2,0(%3)\n"
330 " beq %2,2f\n"
331 #ifdef CONFIG_SMP
332 " mb\n"
333 #endif
334 ".subsection 2\n"
335 "2: br 1b\n"
336 ".previous"
337 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
338 : "r" ((long)m), "1" (val) : "memory");
339
340 return ret;
341 }
342
343 static inline unsigned long
__xchg_u16(volatile short * m,unsigned long val)344 __xchg_u16(volatile short *m, unsigned long val)
345 {
346 unsigned long ret, tmp, addr64;
347
348 __asm__ __volatile__(
349 " andnot %4,7,%3\n"
350 " inswl %1,%4,%1\n"
351 "1: ldq_l %2,0(%3)\n"
352 " extwl %2,%4,%0\n"
353 " mskwl %2,%4,%2\n"
354 " or %1,%2,%2\n"
355 " stq_c %2,0(%3)\n"
356 " beq %2,2f\n"
357 #ifdef CONFIG_SMP
358 " mb\n"
359 #endif
360 ".subsection 2\n"
361 "2: br 1b\n"
362 ".previous"
363 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
364 : "r" ((long)m), "1" (val) : "memory");
365
366 return ret;
367 }
368
369 static inline unsigned long
__xchg_u32(volatile int * m,unsigned long val)370 __xchg_u32(volatile int *m, unsigned long val)
371 {
372 unsigned long dummy;
373
374 __asm__ __volatile__(
375 "1: ldl_l %0,%4\n"
376 " bis $31,%3,%1\n"
377 " stl_c %1,%2\n"
378 " beq %1,2f\n"
379 #ifdef CONFIG_SMP
380 " mb\n"
381 #endif
382 ".subsection 2\n"
383 "2: br 1b\n"
384 ".previous"
385 : "=&r" (val), "=&r" (dummy), "=m" (*m)
386 : "rI" (val), "m" (*m) : "memory");
387
388 return val;
389 }
390
391 static inline unsigned long
__xchg_u64(volatile long * m,unsigned long val)392 __xchg_u64(volatile long *m, unsigned long val)
393 {
394 unsigned long dummy;
395
396 __asm__ __volatile__(
397 "1: ldq_l %0,%4\n"
398 " bis $31,%3,%1\n"
399 " stq_c %1,%2\n"
400 " beq %1,2f\n"
401 #ifdef CONFIG_SMP
402 " mb\n"
403 #endif
404 ".subsection 2\n"
405 "2: br 1b\n"
406 ".previous"
407 : "=&r" (val), "=&r" (dummy), "=m" (*m)
408 : "rI" (val), "m" (*m) : "memory");
409
410 return val;
411 }
412
413 /* This function doesn't exist, so you'll get a linker error
414 if something tries to do an invalid xchg(). */
415 extern void __xchg_called_with_bad_pointer(void);
416
417 #define __xchg(ptr, x, size) \
418 ({ \
419 unsigned long __xchg__res; \
420 volatile void *__xchg__ptr = (ptr); \
421 switch (size) { \
422 case 1: __xchg__res = __xchg_u8(__xchg__ptr, x); break; \
423 case 2: __xchg__res = __xchg_u16(__xchg__ptr, x); break; \
424 case 4: __xchg__res = __xchg_u32(__xchg__ptr, x); break; \
425 case 8: __xchg__res = __xchg_u64(__xchg__ptr, x); break; \
426 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \
427 } \
428 __xchg__res; \
429 })
430
431 #define xchg(ptr,x) \
432 ({ \
433 __typeof__(*(ptr)) _x_ = (x); \
434 (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
435 })
436
437 static inline unsigned long
__xchg_u8_local(volatile char * m,unsigned long val)438 __xchg_u8_local(volatile char *m, unsigned long val)
439 {
440 unsigned long ret, tmp, addr64;
441
442 __asm__ __volatile__(
443 " andnot %4,7,%3\n"
444 " insbl %1,%4,%1\n"
445 "1: ldq_l %2,0(%3)\n"
446 " extbl %2,%4,%0\n"
447 " mskbl %2,%4,%2\n"
448 " or %1,%2,%2\n"
449 " stq_c %2,0(%3)\n"
450 " beq %2,2f\n"
451 ".subsection 2\n"
452 "2: br 1b\n"
453 ".previous"
454 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
455 : "r" ((long)m), "1" (val) : "memory");
456
457 return ret;
458 }
459
460 static inline unsigned long
__xchg_u16_local(volatile short * m,unsigned long val)461 __xchg_u16_local(volatile short *m, unsigned long val)
462 {
463 unsigned long ret, tmp, addr64;
464
465 __asm__ __volatile__(
466 " andnot %4,7,%3\n"
467 " inswl %1,%4,%1\n"
468 "1: ldq_l %2,0(%3)\n"
469 " extwl %2,%4,%0\n"
470 " mskwl %2,%4,%2\n"
471 " or %1,%2,%2\n"
472 " stq_c %2,0(%3)\n"
473 " beq %2,2f\n"
474 ".subsection 2\n"
475 "2: br 1b\n"
476 ".previous"
477 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
478 : "r" ((long)m), "1" (val) : "memory");
479
480 return ret;
481 }
482
483 static inline unsigned long
__xchg_u32_local(volatile int * m,unsigned long val)484 __xchg_u32_local(volatile int *m, unsigned long val)
485 {
486 unsigned long dummy;
487
488 __asm__ __volatile__(
489 "1: ldl_l %0,%4\n"
490 " bis $31,%3,%1\n"
491 " stl_c %1,%2\n"
492 " beq %1,2f\n"
493 ".subsection 2\n"
494 "2: br 1b\n"
495 ".previous"
496 : "=&r" (val), "=&r" (dummy), "=m" (*m)
497 : "rI" (val), "m" (*m) : "memory");
498
499 return val;
500 }
501
502 static inline unsigned long
__xchg_u64_local(volatile long * m,unsigned long val)503 __xchg_u64_local(volatile long *m, unsigned long val)
504 {
505 unsigned long dummy;
506
507 __asm__ __volatile__(
508 "1: ldq_l %0,%4\n"
509 " bis $31,%3,%1\n"
510 " stq_c %1,%2\n"
511 " beq %1,2f\n"
512 ".subsection 2\n"
513 "2: br 1b\n"
514 ".previous"
515 : "=&r" (val), "=&r" (dummy), "=m" (*m)
516 : "rI" (val), "m" (*m) : "memory");
517
518 return val;
519 }
520
521 #define __xchg_local(ptr, x, size) \
522 ({ \
523 unsigned long __xchg__res; \
524 volatile void *__xchg__ptr = (ptr); \
525 switch (size) { \
526 case 1: __xchg__res = __xchg_u8_local(__xchg__ptr, x); break; \
527 case 2: __xchg__res = __xchg_u16_local(__xchg__ptr, x); break; \
528 case 4: __xchg__res = __xchg_u32_local(__xchg__ptr, x); break; \
529 case 8: __xchg__res = __xchg_u64_local(__xchg__ptr, x); break; \
530 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \
531 } \
532 __xchg__res; \
533 })
534
535 #define xchg_local(ptr,x) \
536 ({ \
537 __typeof__(*(ptr)) _x_ = (x); \
538 (__typeof__(*(ptr))) __xchg_local((ptr), (unsigned long)_x_, \
539 sizeof(*(ptr))); \
540 })
541
542 /*
543 * Atomic compare and exchange. Compare OLD with MEM, if identical,
544 * store NEW in MEM. Return the initial value in MEM. Success is
545 * indicated by comparing RETURN with OLD.
546 *
547 * The memory barrier should be placed in SMP only when we actually
548 * make the change. If we don't change anything (so if the returned
549 * prev is equal to old) then we aren't acquiring anything new and
550 * we don't need any memory barrier as far I can tell.
551 */
552
553 #define __HAVE_ARCH_CMPXCHG 1
554
555 static inline unsigned long
__cmpxchg_u8(volatile char * m,long old,long new)556 __cmpxchg_u8(volatile char *m, long old, long new)
557 {
558 unsigned long prev, tmp, cmp, addr64;
559
560 __asm__ __volatile__(
561 " andnot %5,7,%4\n"
562 " insbl %1,%5,%1\n"
563 "1: ldq_l %2,0(%4)\n"
564 " extbl %2,%5,%0\n"
565 " cmpeq %0,%6,%3\n"
566 " beq %3,2f\n"
567 " mskbl %2,%5,%2\n"
568 " or %1,%2,%2\n"
569 " stq_c %2,0(%4)\n"
570 " beq %2,3f\n"
571 #ifdef CONFIG_SMP
572 " mb\n"
573 #endif
574 "2:\n"
575 ".subsection 2\n"
576 "3: br 1b\n"
577 ".previous"
578 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
579 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
580
581 return prev;
582 }
583
584 static inline unsigned long
__cmpxchg_u16(volatile short * m,long old,long new)585 __cmpxchg_u16(volatile short *m, long old, long new)
586 {
587 unsigned long prev, tmp, cmp, addr64;
588
589 __asm__ __volatile__(
590 " andnot %5,7,%4\n"
591 " inswl %1,%5,%1\n"
592 "1: ldq_l %2,0(%4)\n"
593 " extwl %2,%5,%0\n"
594 " cmpeq %0,%6,%3\n"
595 " beq %3,2f\n"
596 " mskwl %2,%5,%2\n"
597 " or %1,%2,%2\n"
598 " stq_c %2,0(%4)\n"
599 " beq %2,3f\n"
600 #ifdef CONFIG_SMP
601 " mb\n"
602 #endif
603 "2:\n"
604 ".subsection 2\n"
605 "3: br 1b\n"
606 ".previous"
607 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
608 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
609
610 return prev;
611 }
612
613 static inline unsigned long
__cmpxchg_u32(volatile int * m,int old,int new)614 __cmpxchg_u32(volatile int *m, int old, int new)
615 {
616 unsigned long prev, cmp;
617
618 __asm__ __volatile__(
619 "1: ldl_l %0,%5\n"
620 " cmpeq %0,%3,%1\n"
621 " beq %1,2f\n"
622 " mov %4,%1\n"
623 " stl_c %1,%2\n"
624 " beq %1,3f\n"
625 #ifdef CONFIG_SMP
626 " mb\n"
627 #endif
628 "2:\n"
629 ".subsection 2\n"
630 "3: br 1b\n"
631 ".previous"
632 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
633 : "r"((long) old), "r"(new), "m"(*m) : "memory");
634
635 return prev;
636 }
637
638 static inline unsigned long
__cmpxchg_u64(volatile long * m,unsigned long old,unsigned long new)639 __cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new)
640 {
641 unsigned long prev, cmp;
642
643 __asm__ __volatile__(
644 "1: ldq_l %0,%5\n"
645 " cmpeq %0,%3,%1\n"
646 " beq %1,2f\n"
647 " mov %4,%1\n"
648 " stq_c %1,%2\n"
649 " beq %1,3f\n"
650 #ifdef CONFIG_SMP
651 " mb\n"
652 #endif
653 "2:\n"
654 ".subsection 2\n"
655 "3: br 1b\n"
656 ".previous"
657 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
658 : "r"((long) old), "r"(new), "m"(*m) : "memory");
659
660 return prev;
661 }
662
663 /* This function doesn't exist, so you'll get a linker error
664 if something tries to do an invalid cmpxchg(). */
665 extern void __cmpxchg_called_with_bad_pointer(void);
666
667 static __always_inline unsigned long
__cmpxchg(volatile void * ptr,unsigned long old,unsigned long new,int size)668 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
669 {
670 switch (size) {
671 case 1:
672 return __cmpxchg_u8(ptr, old, new);
673 case 2:
674 return __cmpxchg_u16(ptr, old, new);
675 case 4:
676 return __cmpxchg_u32(ptr, old, new);
677 case 8:
678 return __cmpxchg_u64(ptr, old, new);
679 }
680 __cmpxchg_called_with_bad_pointer();
681 return old;
682 }
683
684 #define cmpxchg(ptr, o, n) \
685 ({ \
686 __typeof__(*(ptr)) _o_ = (o); \
687 __typeof__(*(ptr)) _n_ = (n); \
688 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
689 (unsigned long)_n_, sizeof(*(ptr))); \
690 })
691 #define cmpxchg64(ptr, o, n) \
692 ({ \
693 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
694 cmpxchg((ptr), (o), (n)); \
695 })
696
697 static inline unsigned long
__cmpxchg_u8_local(volatile char * m,long old,long new)698 __cmpxchg_u8_local(volatile char *m, long old, long new)
699 {
700 unsigned long prev, tmp, cmp, addr64;
701
702 __asm__ __volatile__(
703 " andnot %5,7,%4\n"
704 " insbl %1,%5,%1\n"
705 "1: ldq_l %2,0(%4)\n"
706 " extbl %2,%5,%0\n"
707 " cmpeq %0,%6,%3\n"
708 " beq %3,2f\n"
709 " mskbl %2,%5,%2\n"
710 " or %1,%2,%2\n"
711 " stq_c %2,0(%4)\n"
712 " beq %2,3f\n"
713 "2:\n"
714 ".subsection 2\n"
715 "3: br 1b\n"
716 ".previous"
717 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
718 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
719
720 return prev;
721 }
722
723 static inline unsigned long
__cmpxchg_u16_local(volatile short * m,long old,long new)724 __cmpxchg_u16_local(volatile short *m, long old, long new)
725 {
726 unsigned long prev, tmp, cmp, addr64;
727
728 __asm__ __volatile__(
729 " andnot %5,7,%4\n"
730 " inswl %1,%5,%1\n"
731 "1: ldq_l %2,0(%4)\n"
732 " extwl %2,%5,%0\n"
733 " cmpeq %0,%6,%3\n"
734 " beq %3,2f\n"
735 " mskwl %2,%5,%2\n"
736 " or %1,%2,%2\n"
737 " stq_c %2,0(%4)\n"
738 " beq %2,3f\n"
739 "2:\n"
740 ".subsection 2\n"
741 "3: br 1b\n"
742 ".previous"
743 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
744 : "r" ((long)m), "Ir" (old), "1" (new) : "memory");
745
746 return prev;
747 }
748
749 static inline unsigned long
__cmpxchg_u32_local(volatile int * m,int old,int new)750 __cmpxchg_u32_local(volatile int *m, int old, int new)
751 {
752 unsigned long prev, cmp;
753
754 __asm__ __volatile__(
755 "1: ldl_l %0,%5\n"
756 " cmpeq %0,%3,%1\n"
757 " beq %1,2f\n"
758 " mov %4,%1\n"
759 " stl_c %1,%2\n"
760 " beq %1,3f\n"
761 "2:\n"
762 ".subsection 2\n"
763 "3: br 1b\n"
764 ".previous"
765 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
766 : "r"((long) old), "r"(new), "m"(*m) : "memory");
767
768 return prev;
769 }
770
771 static inline unsigned long
__cmpxchg_u64_local(volatile long * m,unsigned long old,unsigned long new)772 __cmpxchg_u64_local(volatile long *m, unsigned long old, unsigned long new)
773 {
774 unsigned long prev, cmp;
775
776 __asm__ __volatile__(
777 "1: ldq_l %0,%5\n"
778 " cmpeq %0,%3,%1\n"
779 " beq %1,2f\n"
780 " mov %4,%1\n"
781 " stq_c %1,%2\n"
782 " beq %1,3f\n"
783 "2:\n"
784 ".subsection 2\n"
785 "3: br 1b\n"
786 ".previous"
787 : "=&r"(prev), "=&r"(cmp), "=m"(*m)
788 : "r"((long) old), "r"(new), "m"(*m) : "memory");
789
790 return prev;
791 }
792
793 static __always_inline unsigned long
__cmpxchg_local(volatile void * ptr,unsigned long old,unsigned long new,int size)794 __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
795 int size)
796 {
797 switch (size) {
798 case 1:
799 return __cmpxchg_u8_local(ptr, old, new);
800 case 2:
801 return __cmpxchg_u16_local(ptr, old, new);
802 case 4:
803 return __cmpxchg_u32_local(ptr, old, new);
804 case 8:
805 return __cmpxchg_u64_local(ptr, old, new);
806 }
807 __cmpxchg_called_with_bad_pointer();
808 return old;
809 }
810
811 #define cmpxchg_local(ptr, o, n) \
812 ({ \
813 __typeof__(*(ptr)) _o_ = (o); \
814 __typeof__(*(ptr)) _n_ = (n); \
815 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
816 (unsigned long)_n_, sizeof(*(ptr))); \
817 })
818 #define cmpxchg64_local(ptr, o, n) \
819 ({ \
820 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
821 cmpxchg_local((ptr), (o), (n)); \
822 })
823
824
825 #endif /* __ASSEMBLY__ */
826
827 #define arch_align_stack(x) (x)
828
829 #endif
830