1 /*
2 * arch/sh/mm/tlb-flush_64.c
3 *
4 * Copyright (C) 2000, 2001 Paolo Alberelli
5 * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes)
6 * Copyright (C) 2003 - 2009 Paul Mundt
7 *
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
10 * for more details.
11 */
12 #include <linux/signal.h>
13 #include <linux/rwsem.h>
14 #include <linux/sched.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/string.h>
18 #include <linux/types.h>
19 #include <linux/ptrace.h>
20 #include <linux/mman.h>
21 #include <linux/mm.h>
22 #include <linux/smp.h>
23 #include <linux/perf_event.h>
24 #include <linux/interrupt.h>
25 #include <asm/io.h>
26 #include <asm/tlb.h>
27 #include <asm/uaccess.h>
28 #include <asm/pgalloc.h>
29 #include <asm/mmu_context.h>
30
31 extern void die(const char *,struct pt_regs *,long);
32
33 #define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" )
34 #define PPROT(flag) PFLAG(pgprot_val(prot),flag)
35
print_prots(pgprot_t prot)36 static inline void print_prots(pgprot_t prot)
37 {
38 printk("prot is 0x%016llx\n",pgprot_val(prot));
39
40 printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
41 PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
42 }
43
print_vma(struct vm_area_struct * vma)44 static inline void print_vma(struct vm_area_struct *vma)
45 {
46 printk("vma start 0x%08lx\n", vma->vm_start);
47 printk("vma end 0x%08lx\n", vma->vm_end);
48
49 print_prots(vma->vm_page_prot);
50 printk("vm_flags 0x%08lx\n", vma->vm_flags);
51 }
52
print_task(struct task_struct * tsk)53 static inline void print_task(struct task_struct *tsk)
54 {
55 printk("Task pid %d\n", task_pid_nr(tsk));
56 }
57
lookup_pte(struct mm_struct * mm,unsigned long address)58 static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
59 {
60 pgd_t *dir;
61 pud_t *pud;
62 pmd_t *pmd;
63 pte_t *pte;
64 pte_t entry;
65
66 dir = pgd_offset(mm, address);
67 if (pgd_none(*dir))
68 return NULL;
69
70 pud = pud_offset(dir, address);
71 if (pud_none(*pud))
72 return NULL;
73
74 pmd = pmd_offset(pud, address);
75 if (pmd_none(*pmd))
76 return NULL;
77
78 pte = pte_offset_kernel(pmd, address);
79 entry = *pte;
80 if (pte_none(entry) || !pte_present(entry))
81 return NULL;
82
83 return pte;
84 }
85
86 /*
87 * This routine handles page faults. It determines the address,
88 * and the problem, and then passes it off to one of the appropriate
89 * routines.
90 */
do_page_fault(struct pt_regs * regs,unsigned long writeaccess,unsigned long textaccess,unsigned long address)91 asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
92 unsigned long textaccess, unsigned long address)
93 {
94 struct task_struct *tsk;
95 struct mm_struct *mm;
96 struct vm_area_struct * vma;
97 const struct exception_table_entry *fixup;
98 pte_t *pte;
99 int fault;
100
101 /* SIM
102 * Note this is now called with interrupts still disabled
103 * This is to cope with being called for a missing IO port
104 * address with interrupts disabled. This should be fixed as
105 * soon as we have a better 'fast path' miss handler.
106 *
107 * Plus take care how you try and debug this stuff.
108 * For example, writing debug data to a port which you
109 * have just faulted on is not going to work.
110 */
111
112 tsk = current;
113 mm = tsk->mm;
114
115 /* Not an IO address, so reenable interrupts */
116 local_irq_enable();
117
118 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
119
120 /*
121 * If we're in an interrupt or have no user
122 * context, we must not take the fault..
123 */
124 if (in_atomic() || !mm)
125 goto no_context;
126
127 /* TLB misses upon some cache flushes get done under cli() */
128 down_read(&mm->mmap_sem);
129
130 vma = find_vma(mm, address);
131
132 if (!vma) {
133 #ifdef DEBUG_FAULT
134 print_task(tsk);
135 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
136 __func__, __LINE__,
137 address,regs->pc,textaccess,writeaccess);
138 show_regs(regs);
139 #endif
140 goto bad_area;
141 }
142 if (vma->vm_start <= address) {
143 goto good_area;
144 }
145
146 if (!(vma->vm_flags & VM_GROWSDOWN)) {
147 #ifdef DEBUG_FAULT
148 print_task(tsk);
149 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
150 __func__, __LINE__,
151 address,regs->pc,textaccess,writeaccess);
152 show_regs(regs);
153
154 print_vma(vma);
155 #endif
156 goto bad_area;
157 }
158 if (expand_stack(vma, address)) {
159 #ifdef DEBUG_FAULT
160 print_task(tsk);
161 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
162 __func__, __LINE__,
163 address,regs->pc,textaccess,writeaccess);
164 show_regs(regs);
165 #endif
166 goto bad_area;
167 }
168 /*
169 * Ok, we have a good vm_area for this memory access, so
170 * we can handle it..
171 */
172 good_area:
173 if (textaccess) {
174 if (!(vma->vm_flags & VM_EXEC))
175 goto bad_area;
176 } else {
177 if (writeaccess) {
178 if (!(vma->vm_flags & VM_WRITE))
179 goto bad_area;
180 } else {
181 if (!(vma->vm_flags & VM_READ))
182 goto bad_area;
183 }
184 }
185
186 /*
187 * If for any reason at all we couldn't handle the fault,
188 * make sure we exit gracefully rather than endlessly redo
189 * the fault.
190 */
191 fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0);
192 if (unlikely(fault & VM_FAULT_ERROR)) {
193 if (fault & VM_FAULT_OOM)
194 goto out_of_memory;
195 else if (fault & VM_FAULT_SIGBUS)
196 goto do_sigbus;
197 BUG();
198 }
199
200 if (fault & VM_FAULT_MAJOR) {
201 tsk->maj_flt++;
202 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
203 regs, address);
204 } else {
205 tsk->min_flt++;
206 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
207 regs, address);
208 }
209
210 /* If we get here, the page fault has been handled. Do the TLB refill
211 now from the newly-setup PTE, to avoid having to fault again right
212 away on the same instruction. */
213 pte = lookup_pte (mm, address);
214 if (!pte) {
215 /* From empirical evidence, we can get here, due to
216 !pte_present(pte). (e.g. if a swap-in occurs, and the page
217 is swapped back out again before the process that wanted it
218 gets rescheduled?) */
219 goto no_pte;
220 }
221
222 __do_tlb_refill(address, textaccess, pte);
223
224 no_pte:
225
226 up_read(&mm->mmap_sem);
227 return;
228
229 /*
230 * Something tried to access memory that isn't in our memory map..
231 * Fix it, but check if it's kernel or user first..
232 */
233 bad_area:
234 #ifdef DEBUG_FAULT
235 printk("fault:bad area\n");
236 #endif
237 up_read(&mm->mmap_sem);
238
239 if (user_mode(regs)) {
240 static int count=0;
241 siginfo_t info;
242 if (count < 4) {
243 /* This is really to help debug faults when starting
244 * usermode, so only need a few */
245 count++;
246 printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
247 address, task_pid_nr(current), current->comm,
248 (unsigned long) regs->pc);
249 #if 0
250 show_regs(regs);
251 #endif
252 }
253 if (is_global_init(tsk)) {
254 panic("INIT had user mode bad_area\n");
255 }
256 tsk->thread.address = address;
257 tsk->thread.error_code = writeaccess;
258 info.si_signo = SIGSEGV;
259 info.si_errno = 0;
260 info.si_addr = (void *) address;
261 force_sig_info(SIGSEGV, &info, tsk);
262 return;
263 }
264
265 no_context:
266 #ifdef DEBUG_FAULT
267 printk("fault:No context\n");
268 #endif
269 /* Are we prepared to handle this kernel fault? */
270 fixup = search_exception_tables(regs->pc);
271 if (fixup) {
272 regs->pc = fixup->fixup;
273 return;
274 }
275
276 /*
277 * Oops. The kernel tried to access some bad page. We'll have to
278 * terminate things with extreme prejudice.
279 *
280 */
281 if (address < PAGE_SIZE)
282 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
283 else
284 printk(KERN_ALERT "Unable to handle kernel paging request");
285 printk(" at virtual address %08lx\n", address);
286 printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
287 die("Oops", regs, writeaccess);
288 do_exit(SIGKILL);
289
290 /*
291 * We ran out of memory, or some other thing happened to us that made
292 * us unable to handle the page fault gracefully.
293 */
294 out_of_memory:
295 up_read(&mm->mmap_sem);
296 if (!user_mode(regs))
297 goto no_context;
298 pagefault_out_of_memory();
299 return;
300
301 do_sigbus:
302 printk("fault:Do sigbus\n");
303 up_read(&mm->mmap_sem);
304
305 /*
306 * Send a sigbus, regardless of whether we were in kernel
307 * or user mode.
308 */
309 tsk->thread.address = address;
310 tsk->thread.error_code = writeaccess;
311 tsk->thread.trap_no = 14;
312 force_sig(SIGBUS, tsk);
313
314 /* Kernel mode? Handle exceptions or die */
315 if (!user_mode(regs))
316 goto no_context;
317 }
318
local_flush_tlb_one(unsigned long asid,unsigned long page)319 void local_flush_tlb_one(unsigned long asid, unsigned long page)
320 {
321 unsigned long long match, pteh=0, lpage;
322 unsigned long tlb;
323
324 /*
325 * Sign-extend based on neff.
326 */
327 lpage = neff_sign_extend(page);
328 match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
329 match |= lpage;
330
331 for_each_itlb_entry(tlb) {
332 asm volatile ("getcfg %1, 0, %0"
333 : "=r" (pteh)
334 : "r" (tlb) );
335
336 if (pteh == match) {
337 __flush_tlb_slot(tlb);
338 break;
339 }
340 }
341
342 for_each_dtlb_entry(tlb) {
343 asm volatile ("getcfg %1, 0, %0"
344 : "=r" (pteh)
345 : "r" (tlb) );
346
347 if (pteh == match) {
348 __flush_tlb_slot(tlb);
349 break;
350 }
351
352 }
353 }
354
local_flush_tlb_page(struct vm_area_struct * vma,unsigned long page)355 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
356 {
357 unsigned long flags;
358
359 if (vma->vm_mm) {
360 page &= PAGE_MASK;
361 local_irq_save(flags);
362 local_flush_tlb_one(get_asid(), page);
363 local_irq_restore(flags);
364 }
365 }
366
local_flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)367 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
368 unsigned long end)
369 {
370 unsigned long flags;
371 unsigned long long match, pteh=0, pteh_epn, pteh_low;
372 unsigned long tlb;
373 unsigned int cpu = smp_processor_id();
374 struct mm_struct *mm;
375
376 mm = vma->vm_mm;
377 if (cpu_context(cpu, mm) == NO_CONTEXT)
378 return;
379
380 local_irq_save(flags);
381
382 start &= PAGE_MASK;
383 end &= PAGE_MASK;
384
385 match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;
386
387 /* Flush ITLB */
388 for_each_itlb_entry(tlb) {
389 asm volatile ("getcfg %1, 0, %0"
390 : "=r" (pteh)
391 : "r" (tlb) );
392
393 pteh_epn = pteh & PAGE_MASK;
394 pteh_low = pteh & ~PAGE_MASK;
395
396 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
397 __flush_tlb_slot(tlb);
398 }
399
400 /* Flush DTLB */
401 for_each_dtlb_entry(tlb) {
402 asm volatile ("getcfg %1, 0, %0"
403 : "=r" (pteh)
404 : "r" (tlb) );
405
406 pteh_epn = pteh & PAGE_MASK;
407 pteh_low = pteh & ~PAGE_MASK;
408
409 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
410 __flush_tlb_slot(tlb);
411 }
412
413 local_irq_restore(flags);
414 }
415
local_flush_tlb_mm(struct mm_struct * mm)416 void local_flush_tlb_mm(struct mm_struct *mm)
417 {
418 unsigned long flags;
419 unsigned int cpu = smp_processor_id();
420
421 if (cpu_context(cpu, mm) == NO_CONTEXT)
422 return;
423
424 local_irq_save(flags);
425
426 cpu_context(cpu, mm) = NO_CONTEXT;
427 if (mm == current->mm)
428 activate_context(mm, cpu);
429
430 local_irq_restore(flags);
431 }
432
local_flush_tlb_all(void)433 void local_flush_tlb_all(void)
434 {
435 /* Invalidate all, including shared pages, excluding fixed TLBs */
436 unsigned long flags, tlb;
437
438 local_irq_save(flags);
439
440 /* Flush each ITLB entry */
441 for_each_itlb_entry(tlb)
442 __flush_tlb_slot(tlb);
443
444 /* Flush each DTLB entry */
445 for_each_dtlb_entry(tlb)
446 __flush_tlb_slot(tlb);
447
448 local_irq_restore(flags);
449 }
450
local_flush_tlb_kernel_range(unsigned long start,unsigned long end)451 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
452 {
453 /* FIXME: Optimize this later.. */
454 flush_tlb_all();
455 }
456
__flush_tlb_global(void)457 void __flush_tlb_global(void)
458 {
459 flush_tlb_all();
460 }
461
__update_tlb(struct vm_area_struct * vma,unsigned long address,pte_t pte)462 void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
463 {
464 }
465