• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2010 Tilera Corporation. All Rights Reserved.
3  *
4  *   This program is free software; you can redistribute it and/or
5  *   modify it under the terms of the GNU General Public License
6  *   as published by the Free Software Foundation, version 2.
7  *
8  *   This program is distributed in the hope that it will be useful, but
9  *   WITHOUT ANY WARRANTY; without even the implied warranty of
10  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11  *   NON INFRINGEMENT.  See the GNU General Public License for
12  *   more details.
13  *
14  * From i386 code copyright (C) 1995  Linus Torvalds
15  */
16 
17 #include <linux/signal.h>
18 #include <linux/sched.h>
19 #include <linux/kernel.h>
20 #include <linux/errno.h>
21 #include <linux/string.h>
22 #include <linux/types.h>
23 #include <linux/ptrace.h>
24 #include <linux/mman.h>
25 #include <linux/mm.h>
26 #include <linux/smp.h>
27 #include <linux/interrupt.h>
28 #include <linux/init.h>
29 #include <linux/tty.h>
30 #include <linux/vt_kern.h>		/* For unblank_screen() */
31 #include <linux/highmem.h>
32 #include <linux/module.h>
33 #include <linux/kprobes.h>
34 #include <linux/hugetlb.h>
35 #include <linux/syscalls.h>
36 #include <linux/uaccess.h>
37 
38 #include <asm/pgalloc.h>
39 #include <asm/sections.h>
40 #include <asm/traps.h>
41 #include <asm/syscalls.h>
42 
43 #include <arch/interrupts.h>
44 
force_sig_info_fault(const char * type,int si_signo,int si_code,unsigned long address,int fault_num,struct task_struct * tsk,struct pt_regs * regs)45 static noinline void force_sig_info_fault(const char *type, int si_signo,
46 					  int si_code, unsigned long address,
47 					  int fault_num,
48 					  struct task_struct *tsk,
49 					  struct pt_regs *regs)
50 {
51 	siginfo_t info;
52 
53 	if (unlikely(tsk->pid < 2)) {
54 		panic("Signal %d (code %d) at %#lx sent to %s!",
55 		      si_signo, si_code & 0xffff, address,
56 		      is_idle_task(tsk) ? "the idle task" : "init");
57 	}
58 
59 	info.si_signo = si_signo;
60 	info.si_errno = 0;
61 	info.si_code = si_code;
62 	info.si_addr = (void __user *)address;
63 	info.si_trapno = fault_num;
64 	trace_unhandled_signal(type, regs, address, si_signo);
65 	force_sig_info(si_signo, &info, tsk);
66 }
67 
68 #ifndef __tilegx__
69 /*
70  * Synthesize the fault a PL0 process would get by doing a word-load of
71  * an unaligned address or a high kernel address.
72  */
SYSCALL_DEFINE2(cmpxchg_badaddr,unsigned long,address,struct pt_regs *,regs)73 SYSCALL_DEFINE2(cmpxchg_badaddr, unsigned long, address,
74 		struct pt_regs *, regs)
75 {
76 	if (address >= PAGE_OFFSET)
77 		force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR,
78 				     address, INT_DTLB_MISS, current, regs);
79 	else
80 		force_sig_info_fault("atomic alignment fault", SIGBUS,
81 				     BUS_ADRALN, address,
82 				     INT_UNALIGN_DATA, current, regs);
83 
84 	/*
85 	 * Adjust pc to point at the actual instruction, which is unusual
86 	 * for syscalls normally, but is appropriate when we are claiming
87 	 * that a syscall swint1 caused a page fault or bus error.
88 	 */
89 	regs->pc -= 8;
90 
91 	/*
92 	 * Mark this as a caller-save interrupt, like a normal page fault,
93 	 * so that when we go through the signal handler path we will
94 	 * properly restore r0, r1, and r2 for the signal handler arguments.
95 	 */
96 	regs->flags |= PT_FLAGS_CALLER_SAVES;
97 
98 	return 0;
99 }
100 #endif
101 
vmalloc_sync_one(pgd_t * pgd,unsigned long address)102 static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
103 {
104 	unsigned index = pgd_index(address);
105 	pgd_t *pgd_k;
106 	pud_t *pud, *pud_k;
107 	pmd_t *pmd, *pmd_k;
108 
109 	pgd += index;
110 	pgd_k = init_mm.pgd + index;
111 
112 	if (!pgd_present(*pgd_k))
113 		return NULL;
114 
115 	pud = pud_offset(pgd, address);
116 	pud_k = pud_offset(pgd_k, address);
117 	if (!pud_present(*pud_k))
118 		return NULL;
119 
120 	pmd = pmd_offset(pud, address);
121 	pmd_k = pmd_offset(pud_k, address);
122 	if (!pmd_present(*pmd_k))
123 		return NULL;
124 	if (!pmd_present(*pmd)) {
125 		set_pmd(pmd, *pmd_k);
126 		arch_flush_lazy_mmu_mode();
127 	} else
128 		BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k));
129 	return pmd_k;
130 }
131 
132 /*
133  * Handle a fault on the vmalloc area.
134  */
vmalloc_fault(pgd_t * pgd,unsigned long address)135 static inline int vmalloc_fault(pgd_t *pgd, unsigned long address)
136 {
137 	pmd_t *pmd_k;
138 	pte_t *pte_k;
139 
140 	/* Make sure we are in vmalloc area */
141 	if (!(address >= VMALLOC_START && address < VMALLOC_END))
142 		return -1;
143 
144 	/*
145 	 * Synchronize this task's top level page-table
146 	 * with the 'reference' page table.
147 	 */
148 	pmd_k = vmalloc_sync_one(pgd, address);
149 	if (!pmd_k)
150 		return -1;
151 	if (pmd_huge(*pmd_k))
152 		return 0;   /* support TILE huge_vmap() API */
153 	pte_k = pte_offset_kernel(pmd_k, address);
154 	if (!pte_present(*pte_k))
155 		return -1;
156 	return 0;
157 }
158 
159 /* Wait until this PTE has completed migration. */
wait_for_migration(pte_t * pte)160 static void wait_for_migration(pte_t *pte)
161 {
162 	if (pte_migrating(*pte)) {
163 		/*
164 		 * Wait until the migrater fixes up this pte.
165 		 * We scale the loop count by the clock rate so we'll wait for
166 		 * a few seconds here.
167 		 */
168 		int retries = 0;
169 		int bound = get_clock_rate();
170 		while (pte_migrating(*pte)) {
171 			barrier();
172 			if (++retries > bound)
173 				panic("Hit migrating PTE (%#llx) and"
174 				      " page PFN %#lx still migrating",
175 				      pte->val, pte_pfn(*pte));
176 		}
177 	}
178 }
179 
180 /*
181  * It's not generally safe to use "current" to get the page table pointer,
182  * since we might be running an oprofile interrupt in the middle of a
183  * task switch.
184  */
get_current_pgd(void)185 static pgd_t *get_current_pgd(void)
186 {
187 	HV_Context ctx = hv_inquire_context();
188 	unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT;
189 	struct page *pgd_page = pfn_to_page(pgd_pfn);
190 	BUG_ON(PageHighMem(pgd_page));   /* oops, HIGHPTE? */
191 	return (pgd_t *) __va(ctx.page_table);
192 }
193 
194 /*
195  * We can receive a page fault from a migrating PTE at any time.
196  * Handle it by just waiting until the fault resolves.
197  *
198  * It's also possible to get a migrating kernel PTE that resolves
199  * itself during the downcall from hypervisor to Linux.  We just check
200  * here to see if the PTE seems valid, and if so we retry it.
201  *
202  * NOTE! We MUST NOT take any locks for this case.  We may be in an
203  * interrupt or a critical region, and must do as little as possible.
204  * Similarly, we can't use atomic ops here, since we may be handling a
205  * fault caused by an atomic op access.
206  *
207  * If we find a migrating PTE while we're in an NMI context, and we're
208  * at a PC that has a registered exception handler, we don't wait,
209  * since this thread may (e.g.) have been interrupted while migrating
210  * its own stack, which would then cause us to self-deadlock.
211  */
handle_migrating_pte(pgd_t * pgd,int fault_num,unsigned long address,unsigned long pc,int is_kernel_mode,int write)212 static int handle_migrating_pte(pgd_t *pgd, int fault_num,
213 				unsigned long address, unsigned long pc,
214 				int is_kernel_mode, int write)
215 {
216 	pud_t *pud;
217 	pmd_t *pmd;
218 	pte_t *pte;
219 	pte_t pteval;
220 
221 	if (pgd_addr_invalid(address))
222 		return 0;
223 
224 	pgd += pgd_index(address);
225 	pud = pud_offset(pgd, address);
226 	if (!pud || !pud_present(*pud))
227 		return 0;
228 	pmd = pmd_offset(pud, address);
229 	if (!pmd || !pmd_present(*pmd))
230 		return 0;
231 	pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) :
232 		pte_offset_kernel(pmd, address);
233 	pteval = *pte;
234 	if (pte_migrating(pteval)) {
235 		if (in_nmi() && search_exception_tables(pc))
236 			return 0;
237 		wait_for_migration(pte);
238 		return 1;
239 	}
240 
241 	if (!is_kernel_mode || !pte_present(pteval))
242 		return 0;
243 	if (fault_num == INT_ITLB_MISS) {
244 		if (pte_exec(pteval))
245 			return 1;
246 	} else if (write) {
247 		if (pte_write(pteval))
248 			return 1;
249 	} else {
250 		if (pte_read(pteval))
251 			return 1;
252 	}
253 
254 	return 0;
255 }
256 
257 /*
258  * This routine is responsible for faulting in user pages.
259  * It passes the work off to one of the appropriate routines.
260  * It returns true if the fault was successfully handled.
261  */
handle_page_fault(struct pt_regs * regs,int fault_num,int is_page_fault,unsigned long address,int write)262 static int handle_page_fault(struct pt_regs *regs,
263 			     int fault_num,
264 			     int is_page_fault,
265 			     unsigned long address,
266 			     int write)
267 {
268 	struct task_struct *tsk;
269 	struct mm_struct *mm;
270 	struct vm_area_struct *vma;
271 	unsigned long stack_offset;
272 	int fault;
273 	int si_code;
274 	int is_kernel_mode;
275 	pgd_t *pgd;
276 
277 	/* on TILE, protection faults are always writes */
278 	if (!is_page_fault)
279 		write = 1;
280 
281 	is_kernel_mode = (EX1_PL(regs->ex1) != USER_PL);
282 
283 	tsk = validate_current();
284 
285 	/*
286 	 * Check to see if we might be overwriting the stack, and bail
287 	 * out if so.  The page fault code is a relatively likely
288 	 * place to get trapped in an infinite regress, and once we
289 	 * overwrite the whole stack, it becomes very hard to recover.
290 	 */
291 	stack_offset = stack_pointer & (THREAD_SIZE-1);
292 	if (stack_offset < THREAD_SIZE / 8) {
293 		pr_alert("Potential stack overrun: sp %#lx\n",
294 		       stack_pointer);
295 		show_regs(regs);
296 		pr_alert("Killing current process %d/%s\n",
297 		       tsk->pid, tsk->comm);
298 		do_group_exit(SIGKILL);
299 	}
300 
301 	/*
302 	 * Early on, we need to check for migrating PTE entries;
303 	 * see homecache.c.  If we find a migrating PTE, we wait until
304 	 * the backing page claims to be done migrating, then we proceed.
305 	 * For kernel PTEs, we rewrite the PTE and return and retry.
306 	 * Otherwise, we treat the fault like a normal "no PTE" fault,
307 	 * rather than trying to patch up the existing PTE.
308 	 */
309 	pgd = get_current_pgd();
310 	if (handle_migrating_pte(pgd, fault_num, address, regs->pc,
311 				 is_kernel_mode, write))
312 		return 1;
313 
314 	si_code = SEGV_MAPERR;
315 
316 	/*
317 	 * We fault-in kernel-space virtual memory on-demand. The
318 	 * 'reference' page table is init_mm.pgd.
319 	 *
320 	 * NOTE! We MUST NOT take any locks for this case. We may
321 	 * be in an interrupt or a critical region, and should
322 	 * only copy the information from the master page table,
323 	 * nothing more.
324 	 *
325 	 * This verifies that the fault happens in kernel space
326 	 * and that the fault was not a protection fault.
327 	 */
328 	if (unlikely(address >= TASK_SIZE &&
329 		     !is_arch_mappable_range(address, 0))) {
330 		if (is_kernel_mode && is_page_fault &&
331 		    vmalloc_fault(pgd, address) >= 0)
332 			return 1;
333 		/*
334 		 * Don't take the mm semaphore here. If we fixup a prefetch
335 		 * fault we could otherwise deadlock.
336 		 */
337 		mm = NULL;  /* happy compiler */
338 		vma = NULL;
339 		goto bad_area_nosemaphore;
340 	}
341 
342 	/*
343 	 * If we're trying to touch user-space addresses, we must
344 	 * be either at PL0, or else with interrupts enabled in the
345 	 * kernel, so either way we can re-enable interrupts here
346 	 * unless we are doing atomic access to user space with
347 	 * interrupts disabled.
348 	 */
349 	if (!(regs->flags & PT_FLAGS_DISABLE_IRQ))
350 		local_irq_enable();
351 
352 	mm = tsk->mm;
353 
354 	/*
355 	 * If we're in an interrupt, have no user context or are running in an
356 	 * atomic region then we must not take the fault.
357 	 */
358 	if (in_atomic() || !mm) {
359 		vma = NULL;  /* happy compiler */
360 		goto bad_area_nosemaphore;
361 	}
362 
363 	/*
364 	 * When running in the kernel we expect faults to occur only to
365 	 * addresses in user space.  All other faults represent errors in the
366 	 * kernel and should generate an OOPS.  Unfortunately, in the case of an
367 	 * erroneous fault occurring in a code path which already holds mmap_sem
368 	 * we will deadlock attempting to validate the fault against the
369 	 * address space.  Luckily the kernel only validly references user
370 	 * space from well defined areas of code, which are listed in the
371 	 * exceptions table.
372 	 *
373 	 * As the vast majority of faults will be valid we will only perform
374 	 * the source reference check when there is a possibility of a deadlock.
375 	 * Attempt to lock the address space, if we cannot we then validate the
376 	 * source.  If this is invalid we can skip the address space check,
377 	 * thus avoiding the deadlock.
378 	 */
379 	if (!down_read_trylock(&mm->mmap_sem)) {
380 		if (is_kernel_mode &&
381 		    !search_exception_tables(regs->pc)) {
382 			vma = NULL;  /* happy compiler */
383 			goto bad_area_nosemaphore;
384 		}
385 		down_read(&mm->mmap_sem);
386 	}
387 
388 	vma = find_vma(mm, address);
389 	if (!vma)
390 		goto bad_area;
391 	if (vma->vm_start <= address)
392 		goto good_area;
393 	if (!(vma->vm_flags & VM_GROWSDOWN))
394 		goto bad_area;
395 	if (regs->sp < PAGE_OFFSET) {
396 		/*
397 		 * accessing the stack below sp is always a bug.
398 		 */
399 		if (address < regs->sp)
400 			goto bad_area;
401 	}
402 	if (expand_stack(vma, address))
403 		goto bad_area;
404 
405 /*
406  * Ok, we have a good vm_area for this memory access, so
407  * we can handle it..
408  */
409 good_area:
410 	si_code = SEGV_ACCERR;
411 	if (fault_num == INT_ITLB_MISS) {
412 		if (!(vma->vm_flags & VM_EXEC))
413 			goto bad_area;
414 	} else if (write) {
415 #ifdef TEST_VERIFY_AREA
416 		if (!is_page_fault && regs->cs == KERNEL_CS)
417 			pr_err("WP fault at "REGFMT"\n", regs->eip);
418 #endif
419 		if (!(vma->vm_flags & VM_WRITE))
420 			goto bad_area;
421 	} else {
422 		if (!is_page_fault || !(vma->vm_flags & VM_READ))
423 			goto bad_area;
424 	}
425 
426  survive:
427 	/*
428 	 * If for any reason at all we couldn't handle the fault,
429 	 * make sure we exit gracefully rather than endlessly redo
430 	 * the fault.
431 	 */
432 	fault = handle_mm_fault(mm, vma, address, write);
433 	if (unlikely(fault & VM_FAULT_ERROR)) {
434 		if (fault & VM_FAULT_OOM)
435 			goto out_of_memory;
436 		else if (fault & VM_FAULT_SIGBUS)
437 			goto do_sigbus;
438 		BUG();
439 	}
440 	if (fault & VM_FAULT_MAJOR)
441 		tsk->maj_flt++;
442 	else
443 		tsk->min_flt++;
444 
445 #if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
446 	/*
447 	 * If this was an asynchronous fault,
448 	 * restart the appropriate engine.
449 	 */
450 	switch (fault_num) {
451 #if CHIP_HAS_TILE_DMA()
452 	case INT_DMATLB_MISS:
453 	case INT_DMATLB_MISS_DWNCL:
454 	case INT_DMATLB_ACCESS:
455 	case INT_DMATLB_ACCESS_DWNCL:
456 		__insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
457 		break;
458 #endif
459 #if CHIP_HAS_SN_PROC()
460 	case INT_SNITLB_MISS:
461 	case INT_SNITLB_MISS_DWNCL:
462 		__insn_mtspr(SPR_SNCTL,
463 			     __insn_mfspr(SPR_SNCTL) &
464 			     ~SPR_SNCTL__FRZPROC_MASK);
465 		break;
466 #endif
467 	}
468 #endif
469 
470 	up_read(&mm->mmap_sem);
471 	return 1;
472 
473 /*
474  * Something tried to access memory that isn't in our memory map..
475  * Fix it, but check if it's kernel or user first..
476  */
477 bad_area:
478 	up_read(&mm->mmap_sem);
479 
480 bad_area_nosemaphore:
481 	/* User mode accesses just cause a SIGSEGV */
482 	if (!is_kernel_mode) {
483 		/*
484 		 * It's possible to have interrupts off here.
485 		 */
486 		local_irq_enable();
487 
488 		force_sig_info_fault("segfault", SIGSEGV, si_code, address,
489 				     fault_num, tsk, regs);
490 		return 0;
491 	}
492 
493 no_context:
494 	/* Are we prepared to handle this kernel fault?  */
495 	if (fixup_exception(regs))
496 		return 0;
497 
498 /*
499  * Oops. The kernel tried to access some bad page. We'll have to
500  * terminate things with extreme prejudice.
501  */
502 
503 	bust_spinlocks(1);
504 
505 	/* FIXME: no lookup_address() yet */
506 #ifdef SUPPORT_LOOKUP_ADDRESS
507 	if (fault_num == INT_ITLB_MISS) {
508 		pte_t *pte = lookup_address(address);
509 
510 		if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
511 			pr_crit("kernel tried to execute"
512 			       " non-executable page - exploit attempt?"
513 			       " (uid: %d)\n", current->uid);
514 	}
515 #endif
516 	if (address < PAGE_SIZE)
517 		pr_alert("Unable to handle kernel NULL pointer dereference\n");
518 	else
519 		pr_alert("Unable to handle kernel paging request\n");
520 	pr_alert(" at virtual address "REGFMT", pc "REGFMT"\n",
521 		 address, regs->pc);
522 
523 	show_regs(regs);
524 
525 	if (unlikely(tsk->pid < 2)) {
526 		panic("Kernel page fault running %s!",
527 		      is_idle_task(tsk) ? "the idle task" : "init");
528 	}
529 
530 	/*
531 	 * More FIXME: we should probably copy the i386 here and
532 	 * implement a generic die() routine.  Not today.
533 	 */
534 #ifdef SUPPORT_DIE
535 	die("Oops", regs);
536 #endif
537 	bust_spinlocks(1);
538 
539 	do_group_exit(SIGKILL);
540 
541 /*
542  * We ran out of memory, or some other thing happened to us that made
543  * us unable to handle the page fault gracefully.
544  */
545 out_of_memory:
546 	up_read(&mm->mmap_sem);
547 	if (is_global_init(tsk)) {
548 		yield();
549 		down_read(&mm->mmap_sem);
550 		goto survive;
551 	}
552 	pr_alert("VM: killing process %s\n", tsk->comm);
553 	if (!is_kernel_mode)
554 		do_group_exit(SIGKILL);
555 	goto no_context;
556 
557 do_sigbus:
558 	up_read(&mm->mmap_sem);
559 
560 	/* Kernel mode? Handle exceptions or die */
561 	if (is_kernel_mode)
562 		goto no_context;
563 
564 	force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address,
565 			     fault_num, tsk, regs);
566 	return 0;
567 }
568 
569 #ifndef __tilegx__
570 
571 /* We must release ICS before panicking or we won't get anywhere. */
572 #define ics_panic(fmt, ...) do { \
573 	__insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \
574 	panic(fmt, __VA_ARGS__); \
575 } while (0)
576 
577 /*
578  * When we take an ITLB or DTLB fault or access violation in the
579  * supervisor while the critical section bit is set, the hypervisor is
580  * reluctant to write new values into the EX_CONTEXT_K_x registers,
581  * since that might indicate we have not yet squirreled the SPR
582  * contents away and can thus safely take a recursive interrupt.
583  * Accordingly, the hypervisor passes us the PC via SYSTEM_SAVE_K_2.
584  *
585  * Note that this routine is called before homecache_tlb_defer_enter(),
586  * which means that we can properly unlock any atomics that might
587  * be used there (good), but also means we must be very sensitive
588  * to not touch any data structures that might be located in memory
589  * that could migrate, as we could be entering the kernel on a dataplane
590  * cpu that has been deferring kernel TLB updates.  This means, for
591  * example, that we can't migrate init_mm or its pgd.
592  */
do_page_fault_ics(struct pt_regs * regs,int fault_num,unsigned long address,unsigned long info)593 struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num,
594 				      unsigned long address,
595 				      unsigned long info)
596 {
597 	unsigned long pc = info & ~1;
598 	int write = info & 1;
599 	pgd_t *pgd = get_current_pgd();
600 
601 	/* Retval is 1 at first since we will handle the fault fully. */
602 	struct intvec_state state = {
603 		do_page_fault, fault_num, address, write, 1
604 	};
605 
606 	/* Validate that we are plausibly in the right routine. */
607 	if ((pc & 0x7) != 0 || pc < PAGE_OFFSET ||
608 	    (fault_num != INT_DTLB_MISS &&
609 	     fault_num != INT_DTLB_ACCESS)) {
610 		unsigned long old_pc = regs->pc;
611 		regs->pc = pc;
612 		ics_panic("Bad ICS page fault args:"
613 			  " old PC %#lx, fault %d/%d at %#lx\n",
614 			  old_pc, fault_num, write, address);
615 	}
616 
617 	/* We might be faulting on a vmalloc page, so check that first. */
618 	if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0)
619 		return state;
620 
621 	/*
622 	 * If we faulted with ICS set in sys_cmpxchg, we are providing
623 	 * a user syscall service that should generate a signal on
624 	 * fault.  We didn't set up a kernel stack on initial entry to
625 	 * sys_cmpxchg, but instead had one set up by the fault, which
626 	 * (because sys_cmpxchg never releases ICS) came to us via the
627 	 * SYSTEM_SAVE_K_2 mechanism, and thus EX_CONTEXT_K_[01] are
628 	 * still referencing the original user code.  We release the
629 	 * atomic lock and rewrite pt_regs so that it appears that we
630 	 * came from user-space directly, and after we finish the
631 	 * fault we'll go back to user space and re-issue the swint.
632 	 * This way the backtrace information is correct if we need to
633 	 * emit a stack dump at any point while handling this.
634 	 *
635 	 * Must match register use in sys_cmpxchg().
636 	 */
637 	if (pc >= (unsigned long) sys_cmpxchg &&
638 	    pc < (unsigned long) __sys_cmpxchg_end) {
639 #ifdef CONFIG_SMP
640 		/* Don't unlock before we could have locked. */
641 		if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) {
642 			int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
643 			__atomic_fault_unlock(lock_ptr);
644 		}
645 #endif
646 		regs->sp = regs->regs[27];
647 	}
648 
649 	/*
650 	 * We can also fault in the atomic assembly, in which
651 	 * case we use the exception table to do the first-level fixup.
652 	 * We may re-fixup again in the real fault handler if it
653 	 * turns out the faulting address is just bad, and not,
654 	 * for example, migrating.
655 	 */
656 	else if (pc >= (unsigned long) __start_atomic_asm_code &&
657 		   pc < (unsigned long) __end_atomic_asm_code) {
658 		const struct exception_table_entry *fixup;
659 #ifdef CONFIG_SMP
660 		/* Unlock the atomic lock. */
661 		int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
662 		__atomic_fault_unlock(lock_ptr);
663 #endif
664 		fixup = search_exception_tables(pc);
665 		if (!fixup)
666 			ics_panic("ICS atomic fault not in table:"
667 				  " PC %#lx, fault %d", pc, fault_num);
668 		regs->pc = fixup->fixup;
669 		regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0);
670 	}
671 
672 	/*
673 	 * Now that we have released the atomic lock (if necessary),
674 	 * it's safe to spin if the PTE that caused the fault was migrating.
675 	 */
676 	if (fault_num == INT_DTLB_ACCESS)
677 		write = 1;
678 	if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write))
679 		return state;
680 
681 	/* Return zero so that we continue on with normal fault handling. */
682 	state.retval = 0;
683 	return state;
684 }
685 
686 #endif /* !__tilegx__ */
687 
688 /*
689  * This routine handles page faults.  It determines the address, and the
690  * problem, and then passes it handle_page_fault() for normal DTLB and
691  * ITLB issues, and for DMA or SN processor faults when we are in user
692  * space.  For the latter, if we're in kernel mode, we just save the
693  * interrupt away appropriately and return immediately.  We can't do
694  * page faults for user code while in kernel mode.
695  */
do_page_fault(struct pt_regs * regs,int fault_num,unsigned long address,unsigned long write)696 void do_page_fault(struct pt_regs *regs, int fault_num,
697 		   unsigned long address, unsigned long write)
698 {
699 	int is_page_fault;
700 
701 	/* This case should have been handled by do_page_fault_ics(). */
702 	BUG_ON(write & ~1);
703 
704 #if CHIP_HAS_TILE_DMA()
705 	/*
706 	 * If it's a DMA fault, suspend the transfer while we're
707 	 * handling the miss; we'll restart after it's handled.  If we
708 	 * don't suspend, it's possible that this process could swap
709 	 * out and back in, and restart the engine since the DMA is
710 	 * still 'running'.
711 	 */
712 	if (fault_num == INT_DMATLB_MISS ||
713 	    fault_num == INT_DMATLB_ACCESS ||
714 	    fault_num == INT_DMATLB_MISS_DWNCL ||
715 	    fault_num == INT_DMATLB_ACCESS_DWNCL) {
716 		__insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
717 		while (__insn_mfspr(SPR_DMA_USER_STATUS) &
718 		       SPR_DMA_STATUS__BUSY_MASK)
719 			;
720 	}
721 #endif
722 
723 	/* Validate fault num and decide if this is a first-time page fault. */
724 	switch (fault_num) {
725 	case INT_ITLB_MISS:
726 	case INT_DTLB_MISS:
727 #if CHIP_HAS_TILE_DMA()
728 	case INT_DMATLB_MISS:
729 	case INT_DMATLB_MISS_DWNCL:
730 #endif
731 #if CHIP_HAS_SN_PROC()
732 	case INT_SNITLB_MISS:
733 	case INT_SNITLB_MISS_DWNCL:
734 #endif
735 		is_page_fault = 1;
736 		break;
737 
738 	case INT_DTLB_ACCESS:
739 #if CHIP_HAS_TILE_DMA()
740 	case INT_DMATLB_ACCESS:
741 	case INT_DMATLB_ACCESS_DWNCL:
742 #endif
743 		is_page_fault = 0;
744 		break;
745 
746 	default:
747 		panic("Bad fault number %d in do_page_fault", fault_num);
748 	}
749 
750 #if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
751 	if (EX1_PL(regs->ex1) != USER_PL) {
752 		struct async_tlb *async;
753 		switch (fault_num) {
754 #if CHIP_HAS_TILE_DMA()
755 		case INT_DMATLB_MISS:
756 		case INT_DMATLB_ACCESS:
757 		case INT_DMATLB_MISS_DWNCL:
758 		case INT_DMATLB_ACCESS_DWNCL:
759 			async = &current->thread.dma_async_tlb;
760 			break;
761 #endif
762 #if CHIP_HAS_SN_PROC()
763 		case INT_SNITLB_MISS:
764 		case INT_SNITLB_MISS_DWNCL:
765 			async = &current->thread.sn_async_tlb;
766 			break;
767 #endif
768 		default:
769 			async = NULL;
770 		}
771 		if (async) {
772 
773 			/*
774 			 * No vmalloc check required, so we can allow
775 			 * interrupts immediately at this point.
776 			 */
777 			local_irq_enable();
778 
779 			set_thread_flag(TIF_ASYNC_TLB);
780 			if (async->fault_num != 0) {
781 				panic("Second async fault %d;"
782 				      " old fault was %d (%#lx/%ld)",
783 				      fault_num, async->fault_num,
784 				      address, write);
785 			}
786 			BUG_ON(fault_num == 0);
787 			async->fault_num = fault_num;
788 			async->is_fault = is_page_fault;
789 			async->is_write = write;
790 			async->address = address;
791 			return;
792 		}
793 	}
794 #endif
795 
796 	handle_page_fault(regs, fault_num, is_page_fault, address, write);
797 }
798 
799 
800 #if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
801 /*
802  * Check an async_tlb structure to see if a deferred fault is waiting,
803  * and if so pass it to the page-fault code.
804  */
handle_async_page_fault(struct pt_regs * regs,struct async_tlb * async)805 static void handle_async_page_fault(struct pt_regs *regs,
806 				    struct async_tlb *async)
807 {
808 	if (async->fault_num) {
809 		/*
810 		 * Clear async->fault_num before calling the page-fault
811 		 * handler so that if we re-interrupt before returning
812 		 * from the function we have somewhere to put the
813 		 * information from the new interrupt.
814 		 */
815 		int fault_num = async->fault_num;
816 		async->fault_num = 0;
817 		handle_page_fault(regs, fault_num, async->is_fault,
818 				  async->address, async->is_write);
819 	}
820 }
821 
822 /*
823  * This routine effectively re-issues asynchronous page faults
824  * when we are returning to user space.
825  */
do_async_page_fault(struct pt_regs * regs)826 void do_async_page_fault(struct pt_regs *regs)
827 {
828 	/*
829 	 * Clear thread flag early.  If we re-interrupt while processing
830 	 * code here, we will reset it and recall this routine before
831 	 * returning to user space.
832 	 */
833 	clear_thread_flag(TIF_ASYNC_TLB);
834 
835 #if CHIP_HAS_TILE_DMA()
836 	handle_async_page_fault(regs, &current->thread.dma_async_tlb);
837 #endif
838 #if CHIP_HAS_SN_PROC()
839 	handle_async_page_fault(regs, &current->thread.sn_async_tlb);
840 #endif
841 }
842 #endif /* CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC() */
843 
844 
vmalloc_sync_all(void)845 void vmalloc_sync_all(void)
846 {
847 #ifdef __tilegx__
848 	/* Currently all L1 kernel pmd's are static and shared. */
849 	BUG_ON(pgd_index(VMALLOC_END) != pgd_index(VMALLOC_START));
850 #else
851 	/*
852 	 * Note that races in the updates of insync and start aren't
853 	 * problematic: insync can only get set bits added, and updates to
854 	 * start are only improving performance (without affecting correctness
855 	 * if undone).
856 	 */
857 	static DECLARE_BITMAP(insync, PTRS_PER_PGD);
858 	static unsigned long start = PAGE_OFFSET;
859 	unsigned long address;
860 
861 	BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK);
862 	for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) {
863 		if (!test_bit(pgd_index(address), insync)) {
864 			unsigned long flags;
865 			struct list_head *pos;
866 
867 			spin_lock_irqsave(&pgd_lock, flags);
868 			list_for_each(pos, &pgd_list)
869 				if (!vmalloc_sync_one(list_to_pgd(pos),
870 								address)) {
871 					/* Must be at first entry in list. */
872 					BUG_ON(pos != pgd_list.next);
873 					break;
874 				}
875 			spin_unlock_irqrestore(&pgd_lock, flags);
876 			if (pos != pgd_list.next)
877 				set_bit(pgd_index(address), insync);
878 		}
879 		if (address == start && test_bit(pgd_index(address), insync))
880 			start = address + PGDIR_SIZE;
881 	}
882 #endif
883 }
884