• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 
2 /*
3  *    Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
4  *			 <benh@kernel.crashing.org>
5  *
6  *  This program is free software; you can redistribute it and/or
7  *  modify it under the terms of the GNU General Public License
8  *  as published by the Free Software Foundation; either version
9  *  2 of the License, or (at your option) any later version.
10  */
11 
12 #include <linux/errno.h>
13 #include <linux/sched.h>
14 #include <linux/kernel.h>
15 #include <linux/mm.h>
16 #include <linux/smp.h>
17 #include <linux/stddef.h>
18 #include <linux/unistd.h>
19 #include <linux/slab.h>
20 #include <linux/user.h>
21 #include <linux/elf.h>
22 #include <linux/security.h>
23 #include <linux/memblock.h>
24 
25 #include <asm/pgtable.h>
26 #include <asm/processor.h>
27 #include <asm/mmu.h>
28 #include <asm/mmu_context.h>
29 #include <asm/prom.h>
30 #include <asm/machdep.h>
31 #include <asm/cputable.h>
32 #include <asm/sections.h>
33 #include <asm/firmware.h>
34 #include <asm/vdso.h>
35 #include <asm/vdso_datapage.h>
36 #include <asm/setup.h>
37 
38 #undef DEBUG
39 
40 #ifdef DEBUG
41 #define DBG(fmt...) printk(fmt)
42 #else
43 #define DBG(fmt...)
44 #endif
45 
46 /* Max supported size for symbol names */
47 #define MAX_SYMNAME	64
48 
49 /* The alignment of the vDSO */
50 #define VDSO_ALIGNMENT	(1 << 16)
51 
52 static unsigned int vdso32_pages;
53 static void *vdso32_kbase;
54 static struct page **vdso32_pagelist;
55 unsigned long vdso32_sigtramp;
56 unsigned long vdso32_rt_sigtramp;
57 
58 #ifdef CONFIG_VDSO32
59 extern char vdso32_start, vdso32_end;
60 #endif
61 
62 #ifdef CONFIG_PPC64
63 extern char vdso64_start, vdso64_end;
64 static void *vdso64_kbase = &vdso64_start;
65 static unsigned int vdso64_pages;
66 static struct page **vdso64_pagelist;
67 unsigned long vdso64_rt_sigtramp;
68 #endif /* CONFIG_PPC64 */
69 
70 static int vdso_ready;
71 
72 /*
73  * The vdso data page (aka. systemcfg for old ppc64 fans) is here.
74  * Once the early boot kernel code no longer needs to muck around
75  * with it, it will become dynamically allocated
76  */
77 static union {
78 	struct vdso_data	data;
79 	u8			page[PAGE_SIZE];
80 } vdso_data_store __page_aligned_data;
81 struct vdso_data *vdso_data = &vdso_data_store.data;
82 
83 /* Format of the patch table */
84 struct vdso_patch_def
85 {
86 	unsigned long	ftr_mask, ftr_value;
87 	const char	*gen_name;
88 	const char	*fix_name;
89 };
90 
91 /* Table of functions to patch based on the CPU type/revision
92  *
93  * Currently, we only change sync_dicache to do nothing on processors
94  * with a coherent icache
95  */
96 static struct vdso_patch_def vdso_patches[] = {
97 	{
98 		CPU_FTR_COHERENT_ICACHE, CPU_FTR_COHERENT_ICACHE,
99 		"__kernel_sync_dicache", "__kernel_sync_dicache_p5"
100 	},
101 	{
102 		CPU_FTR_USE_TB, 0,
103 		"__kernel_gettimeofday", NULL
104 	},
105 	{
106 		CPU_FTR_USE_TB, 0,
107 		"__kernel_clock_gettime", NULL
108 	},
109 	{
110 		CPU_FTR_USE_TB, 0,
111 		"__kernel_clock_getres", NULL
112 	},
113 	{
114 		CPU_FTR_USE_TB, 0,
115 		"__kernel_get_tbfreq", NULL
116 	},
117 	{
118 		CPU_FTR_USE_TB, 0,
119 		"__kernel_time", NULL
120 	},
121 };
122 
123 /*
124  * Some infos carried around for each of them during parsing at
125  * boot time.
126  */
127 struct lib32_elfinfo
128 {
129 	Elf32_Ehdr	*hdr;		/* ptr to ELF */
130 	Elf32_Sym	*dynsym;	/* ptr to .dynsym section */
131 	unsigned long	dynsymsize;	/* size of .dynsym section */
132 	char		*dynstr;	/* ptr to .dynstr section */
133 	unsigned long	text;		/* offset of .text section in .so */
134 };
135 
136 struct lib64_elfinfo
137 {
138 	Elf64_Ehdr	*hdr;
139 	Elf64_Sym	*dynsym;
140 	unsigned long	dynsymsize;
141 	char		*dynstr;
142 	unsigned long	text;
143 };
144 
145 
146 /*
147  * This is called from binfmt_elf, we create the special vma for the
148  * vDSO and insert it into the mm struct tree
149  */
arch_setup_additional_pages(struct linux_binprm * bprm,int uses_interp)150 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
151 {
152 	struct mm_struct *mm = current->mm;
153 	struct page **vdso_pagelist;
154 	unsigned long vdso_pages;
155 	unsigned long vdso_base;
156 	int rc;
157 
158 	if (!vdso_ready)
159 		return 0;
160 
161 #ifdef CONFIG_PPC64
162 	if (is_32bit_task()) {
163 		vdso_pagelist = vdso32_pagelist;
164 		vdso_pages = vdso32_pages;
165 		vdso_base = VDSO32_MBASE;
166 	} else {
167 		vdso_pagelist = vdso64_pagelist;
168 		vdso_pages = vdso64_pages;
169 		/*
170 		 * On 64bit we don't have a preferred map address. This
171 		 * allows get_unmapped_area to find an area near other mmaps
172 		 * and most likely share a SLB entry.
173 		 */
174 		vdso_base = 0;
175 	}
176 #else
177 	vdso_pagelist = vdso32_pagelist;
178 	vdso_pages = vdso32_pages;
179 	vdso_base = VDSO32_MBASE;
180 #endif
181 
182 	current->mm->context.vdso_base = 0;
183 
184 	/* vDSO has a problem and was disabled, just don't "enable" it for the
185 	 * process
186 	 */
187 	if (vdso_pages == 0)
188 		return 0;
189 	/* Add a page to the vdso size for the data page */
190 	vdso_pages ++;
191 
192 	/*
193 	 * pick a base address for the vDSO in process space. We try to put it
194 	 * at vdso_base which is the "natural" base for it, but we might fail
195 	 * and end up putting it elsewhere.
196 	 * Add enough to the size so that the result can be aligned.
197 	 */
198 	down_write(&mm->mmap_sem);
199 	vdso_base = get_unmapped_area(NULL, vdso_base,
200 				      (vdso_pages << PAGE_SHIFT) +
201 				      ((VDSO_ALIGNMENT - 1) & PAGE_MASK),
202 				      0, 0);
203 	if (IS_ERR_VALUE(vdso_base)) {
204 		rc = vdso_base;
205 		goto fail_mmapsem;
206 	}
207 
208 	/* Add required alignment. */
209 	vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT);
210 
211 	/*
212 	 * Put vDSO base into mm struct. We need to do this before calling
213 	 * install_special_mapping or the perf counter mmap tracking code
214 	 * will fail to recognise it as a vDSO (since arch_vma_name fails).
215 	 */
216 	current->mm->context.vdso_base = vdso_base;
217 
218 	/*
219 	 * our vma flags don't have VM_WRITE so by default, the process isn't
220 	 * allowed to write those pages.
221 	 * gdb can break that with ptrace interface, and thus trigger COW on
222 	 * those pages but it's then your responsibility to never do that on
223 	 * the "data" page of the vDSO or you'll stop getting kernel updates
224 	 * and your nice userland gettimeofday will be totally dead.
225 	 * It's fine to use that for setting breakpoints in the vDSO code
226 	 * pages though.
227 	 */
228 	rc = install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT,
229 				     VM_READ|VM_EXEC|
230 				     VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
231 				     vdso_pagelist);
232 	if (rc) {
233 		current->mm->context.vdso_base = 0;
234 		goto fail_mmapsem;
235 	}
236 
237 	up_write(&mm->mmap_sem);
238 	return 0;
239 
240  fail_mmapsem:
241 	up_write(&mm->mmap_sem);
242 	return rc;
243 }
244 
arch_vma_name(struct vm_area_struct * vma)245 const char *arch_vma_name(struct vm_area_struct *vma)
246 {
247 	if (vma->vm_mm && vma->vm_start == vma->vm_mm->context.vdso_base)
248 		return "[vdso]";
249 	return NULL;
250 }
251 
252 
253 
254 #ifdef CONFIG_VDSO32
find_section32(Elf32_Ehdr * ehdr,const char * secname,unsigned long * size)255 static void * __init find_section32(Elf32_Ehdr *ehdr, const char *secname,
256 				  unsigned long *size)
257 {
258 	Elf32_Shdr *sechdrs;
259 	unsigned int i;
260 	char *secnames;
261 
262 	/* Grab section headers and strings so we can tell who is who */
263 	sechdrs = (void *)ehdr + ehdr->e_shoff;
264 	secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
265 
266 	/* Find the section they want */
267 	for (i = 1; i < ehdr->e_shnum; i++) {
268 		if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) {
269 			if (size)
270 				*size = sechdrs[i].sh_size;
271 			return (void *)ehdr + sechdrs[i].sh_offset;
272 		}
273 	}
274 	*size = 0;
275 	return NULL;
276 }
277 
find_symbol32(struct lib32_elfinfo * lib,const char * symname)278 static Elf32_Sym * __init find_symbol32(struct lib32_elfinfo *lib,
279 					const char *symname)
280 {
281 	unsigned int i;
282 	char name[MAX_SYMNAME], *c;
283 
284 	for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) {
285 		if (lib->dynsym[i].st_name == 0)
286 			continue;
287 		strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
288 			MAX_SYMNAME);
289 		c = strchr(name, '@');
290 		if (c)
291 			*c = 0;
292 		if (strcmp(symname, name) == 0)
293 			return &lib->dynsym[i];
294 	}
295 	return NULL;
296 }
297 
298 /* Note that we assume the section is .text and the symbol is relative to
299  * the library base
300  */
find_function32(struct lib32_elfinfo * lib,const char * symname)301 static unsigned long __init find_function32(struct lib32_elfinfo *lib,
302 					    const char *symname)
303 {
304 	Elf32_Sym *sym = find_symbol32(lib, symname);
305 
306 	if (sym == NULL) {
307 		printk(KERN_WARNING "vDSO32: function %s not found !\n",
308 		       symname);
309 		return 0;
310 	}
311 	return sym->st_value - VDSO32_LBASE;
312 }
313 
vdso_do_func_patch32(struct lib32_elfinfo * v32,struct lib64_elfinfo * v64,const char * orig,const char * fix)314 static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32,
315 				       struct lib64_elfinfo *v64,
316 				       const char *orig, const char *fix)
317 {
318 	Elf32_Sym *sym32_gen, *sym32_fix;
319 
320 	sym32_gen = find_symbol32(v32, orig);
321 	if (sym32_gen == NULL) {
322 		printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", orig);
323 		return -1;
324 	}
325 	if (fix == NULL) {
326 		sym32_gen->st_name = 0;
327 		return 0;
328 	}
329 	sym32_fix = find_symbol32(v32, fix);
330 	if (sym32_fix == NULL) {
331 		printk(KERN_ERR "vDSO32: Can't find symbol %s !\n", fix);
332 		return -1;
333 	}
334 	sym32_gen->st_value = sym32_fix->st_value;
335 	sym32_gen->st_size = sym32_fix->st_size;
336 	sym32_gen->st_info = sym32_fix->st_info;
337 	sym32_gen->st_other = sym32_fix->st_other;
338 	sym32_gen->st_shndx = sym32_fix->st_shndx;
339 
340 	return 0;
341 }
342 #else /* !CONFIG_VDSO32 */
find_function32(struct lib32_elfinfo * lib,const char * symname)343 static unsigned long __init find_function32(struct lib32_elfinfo *lib,
344 					    const char *symname)
345 {
346 	return 0;
347 }
348 
vdso_do_func_patch32(struct lib32_elfinfo * v32,struct lib64_elfinfo * v64,const char * orig,const char * fix)349 static int __init vdso_do_func_patch32(struct lib32_elfinfo *v32,
350 				       struct lib64_elfinfo *v64,
351 				       const char *orig, const char *fix)
352 {
353 	return 0;
354 }
355 #endif /* CONFIG_VDSO32 */
356 
357 
358 #ifdef CONFIG_PPC64
359 
find_section64(Elf64_Ehdr * ehdr,const char * secname,unsigned long * size)360 static void * __init find_section64(Elf64_Ehdr *ehdr, const char *secname,
361 				  unsigned long *size)
362 {
363 	Elf64_Shdr *sechdrs;
364 	unsigned int i;
365 	char *secnames;
366 
367 	/* Grab section headers and strings so we can tell who is who */
368 	sechdrs = (void *)ehdr + ehdr->e_shoff;
369 	secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset;
370 
371 	/* Find the section they want */
372 	for (i = 1; i < ehdr->e_shnum; i++) {
373 		if (strcmp(secnames+sechdrs[i].sh_name, secname) == 0) {
374 			if (size)
375 				*size = sechdrs[i].sh_size;
376 			return (void *)ehdr + sechdrs[i].sh_offset;
377 		}
378 	}
379 	if (size)
380 		*size = 0;
381 	return NULL;
382 }
383 
find_symbol64(struct lib64_elfinfo * lib,const char * symname)384 static Elf64_Sym * __init find_symbol64(struct lib64_elfinfo *lib,
385 					const char *symname)
386 {
387 	unsigned int i;
388 	char name[MAX_SYMNAME], *c;
389 
390 	for (i = 0; i < (lib->dynsymsize / sizeof(Elf64_Sym)); i++) {
391 		if (lib->dynsym[i].st_name == 0)
392 			continue;
393 		strlcpy(name, lib->dynstr + lib->dynsym[i].st_name,
394 			MAX_SYMNAME);
395 		c = strchr(name, '@');
396 		if (c)
397 			*c = 0;
398 		if (strcmp(symname, name) == 0)
399 			return &lib->dynsym[i];
400 	}
401 	return NULL;
402 }
403 
404 /* Note that we assume the section is .text and the symbol is relative to
405  * the library base
406  */
find_function64(struct lib64_elfinfo * lib,const char * symname)407 static unsigned long __init find_function64(struct lib64_elfinfo *lib,
408 					    const char *symname)
409 {
410 	Elf64_Sym *sym = find_symbol64(lib, symname);
411 
412 	if (sym == NULL) {
413 		printk(KERN_WARNING "vDSO64: function %s not found !\n",
414 		       symname);
415 		return 0;
416 	}
417 #ifdef VDS64_HAS_DESCRIPTORS
418 	return *((u64 *)(vdso64_kbase + sym->st_value - VDSO64_LBASE)) -
419 		VDSO64_LBASE;
420 #else
421 	return sym->st_value - VDSO64_LBASE;
422 #endif
423 }
424 
vdso_do_func_patch64(struct lib32_elfinfo * v32,struct lib64_elfinfo * v64,const char * orig,const char * fix)425 static int __init vdso_do_func_patch64(struct lib32_elfinfo *v32,
426 				       struct lib64_elfinfo *v64,
427 				       const char *orig, const char *fix)
428 {
429 	Elf64_Sym *sym64_gen, *sym64_fix;
430 
431 	sym64_gen = find_symbol64(v64, orig);
432 	if (sym64_gen == NULL) {
433 		printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", orig);
434 		return -1;
435 	}
436 	if (fix == NULL) {
437 		sym64_gen->st_name = 0;
438 		return 0;
439 	}
440 	sym64_fix = find_symbol64(v64, fix);
441 	if (sym64_fix == NULL) {
442 		printk(KERN_ERR "vDSO64: Can't find symbol %s !\n", fix);
443 		return -1;
444 	}
445 	sym64_gen->st_value = sym64_fix->st_value;
446 	sym64_gen->st_size = sym64_fix->st_size;
447 	sym64_gen->st_info = sym64_fix->st_info;
448 	sym64_gen->st_other = sym64_fix->st_other;
449 	sym64_gen->st_shndx = sym64_fix->st_shndx;
450 
451 	return 0;
452 }
453 
454 #endif /* CONFIG_PPC64 */
455 
456 
vdso_do_find_sections(struct lib32_elfinfo * v32,struct lib64_elfinfo * v64)457 static __init int vdso_do_find_sections(struct lib32_elfinfo *v32,
458 					struct lib64_elfinfo *v64)
459 {
460 	void *sect;
461 
462 	/*
463 	 * Locate symbol tables & text section
464 	 */
465 
466 #ifdef CONFIG_VDSO32
467 	v32->dynsym = find_section32(v32->hdr, ".dynsym", &v32->dynsymsize);
468 	v32->dynstr = find_section32(v32->hdr, ".dynstr", NULL);
469 	if (v32->dynsym == NULL || v32->dynstr == NULL) {
470 		printk(KERN_ERR "vDSO32: required symbol section not found\n");
471 		return -1;
472 	}
473 	sect = find_section32(v32->hdr, ".text", NULL);
474 	if (sect == NULL) {
475 		printk(KERN_ERR "vDSO32: the .text section was not found\n");
476 		return -1;
477 	}
478 	v32->text = sect - vdso32_kbase;
479 #endif
480 
481 #ifdef CONFIG_PPC64
482 	v64->dynsym = find_section64(v64->hdr, ".dynsym", &v64->dynsymsize);
483 	v64->dynstr = find_section64(v64->hdr, ".dynstr", NULL);
484 	if (v64->dynsym == NULL || v64->dynstr == NULL) {
485 		printk(KERN_ERR "vDSO64: required symbol section not found\n");
486 		return -1;
487 	}
488 	sect = find_section64(v64->hdr, ".text", NULL);
489 	if (sect == NULL) {
490 		printk(KERN_ERR "vDSO64: the .text section was not found\n");
491 		return -1;
492 	}
493 	v64->text = sect - vdso64_kbase;
494 #endif /* CONFIG_PPC64 */
495 
496 	return 0;
497 }
498 
vdso_setup_trampolines(struct lib32_elfinfo * v32,struct lib64_elfinfo * v64)499 static __init void vdso_setup_trampolines(struct lib32_elfinfo *v32,
500 					  struct lib64_elfinfo *v64)
501 {
502 	/*
503 	 * Find signal trampolines
504 	 */
505 
506 #ifdef CONFIG_PPC64
507 	vdso64_rt_sigtramp = find_function64(v64, "__kernel_sigtramp_rt64");
508 #endif
509 	vdso32_sigtramp	   = find_function32(v32, "__kernel_sigtramp32");
510 	vdso32_rt_sigtramp = find_function32(v32, "__kernel_sigtramp_rt32");
511 }
512 
vdso_fixup_datapage(struct lib32_elfinfo * v32,struct lib64_elfinfo * v64)513 static __init int vdso_fixup_datapage(struct lib32_elfinfo *v32,
514 				       struct lib64_elfinfo *v64)
515 {
516 #ifdef CONFIG_VDSO32
517 	Elf32_Sym *sym32;
518 #endif
519 #ifdef CONFIG_PPC64
520 	Elf64_Sym *sym64;
521 
522        	sym64 = find_symbol64(v64, "__kernel_datapage_offset");
523 	if (sym64 == NULL) {
524 		printk(KERN_ERR "vDSO64: Can't find symbol "
525 		       "__kernel_datapage_offset !\n");
526 		return -1;
527 	}
528 	*((int *)(vdso64_kbase + sym64->st_value - VDSO64_LBASE)) =
529 		(vdso64_pages << PAGE_SHIFT) -
530 		(sym64->st_value - VDSO64_LBASE);
531 #endif /* CONFIG_PPC64 */
532 
533 #ifdef CONFIG_VDSO32
534 	sym32 = find_symbol32(v32, "__kernel_datapage_offset");
535 	if (sym32 == NULL) {
536 		printk(KERN_ERR "vDSO32: Can't find symbol "
537 		       "__kernel_datapage_offset !\n");
538 		return -1;
539 	}
540 	*((int *)(vdso32_kbase + (sym32->st_value - VDSO32_LBASE))) =
541 		(vdso32_pages << PAGE_SHIFT) -
542 		(sym32->st_value - VDSO32_LBASE);
543 #endif
544 
545 	return 0;
546 }
547 
548 
vdso_fixup_features(struct lib32_elfinfo * v32,struct lib64_elfinfo * v64)549 static __init int vdso_fixup_features(struct lib32_elfinfo *v32,
550 				      struct lib64_elfinfo *v64)
551 {
552 	unsigned long size;
553 	void *start;
554 
555 #ifdef CONFIG_PPC64
556 	start = find_section64(v64->hdr, "__ftr_fixup", &size);
557 	if (start)
558 		do_feature_fixups(cur_cpu_spec->cpu_features,
559 				  start, start + size);
560 
561 	start = find_section64(v64->hdr, "__mmu_ftr_fixup", &size);
562 	if (start)
563 		do_feature_fixups(cur_cpu_spec->mmu_features,
564 				  start, start + size);
565 
566 	start = find_section64(v64->hdr, "__fw_ftr_fixup", &size);
567 	if (start)
568 		do_feature_fixups(powerpc_firmware_features,
569 				  start, start + size);
570 
571 	start = find_section64(v64->hdr, "__lwsync_fixup", &size);
572 	if (start)
573 		do_lwsync_fixups(cur_cpu_spec->cpu_features,
574 				 start, start + size);
575 #endif /* CONFIG_PPC64 */
576 
577 #ifdef CONFIG_VDSO32
578 	start = find_section32(v32->hdr, "__ftr_fixup", &size);
579 	if (start)
580 		do_feature_fixups(cur_cpu_spec->cpu_features,
581 				  start, start + size);
582 
583 	start = find_section32(v32->hdr, "__mmu_ftr_fixup", &size);
584 	if (start)
585 		do_feature_fixups(cur_cpu_spec->mmu_features,
586 				  start, start + size);
587 
588 #ifdef CONFIG_PPC64
589 	start = find_section32(v32->hdr, "__fw_ftr_fixup", &size);
590 	if (start)
591 		do_feature_fixups(powerpc_firmware_features,
592 				  start, start + size);
593 #endif /* CONFIG_PPC64 */
594 
595 	start = find_section32(v32->hdr, "__lwsync_fixup", &size);
596 	if (start)
597 		do_lwsync_fixups(cur_cpu_spec->cpu_features,
598 				 start, start + size);
599 #endif
600 
601 	return 0;
602 }
603 
vdso_fixup_alt_funcs(struct lib32_elfinfo * v32,struct lib64_elfinfo * v64)604 static __init int vdso_fixup_alt_funcs(struct lib32_elfinfo *v32,
605 				       struct lib64_elfinfo *v64)
606 {
607 	int i;
608 
609 	for (i = 0; i < ARRAY_SIZE(vdso_patches); i++) {
610 		struct vdso_patch_def *patch = &vdso_patches[i];
611 		int match = (cur_cpu_spec->cpu_features & patch->ftr_mask)
612 			== patch->ftr_value;
613 		if (!match)
614 			continue;
615 
616 		DBG("replacing %s with %s...\n", patch->gen_name,
617 		    patch->fix_name ? "NONE" : patch->fix_name);
618 
619 		/*
620 		 * Patch the 32 bits and 64 bits symbols. Note that we do not
621 		 * patch the "." symbol on 64 bits.
622 		 * It would be easy to do, but doesn't seem to be necessary,
623 		 * patching the OPD symbol is enough.
624 		 */
625 		vdso_do_func_patch32(v32, v64, patch->gen_name,
626 				     patch->fix_name);
627 #ifdef CONFIG_PPC64
628 		vdso_do_func_patch64(v32, v64, patch->gen_name,
629 				     patch->fix_name);
630 #endif /* CONFIG_PPC64 */
631 	}
632 
633 	return 0;
634 }
635 
636 
vdso_setup(void)637 static __init int vdso_setup(void)
638 {
639 	struct lib32_elfinfo	v32;
640 	struct lib64_elfinfo	v64;
641 
642 	v32.hdr = vdso32_kbase;
643 #ifdef CONFIG_PPC64
644 	v64.hdr = vdso64_kbase;
645 #endif
646 	if (vdso_do_find_sections(&v32, &v64))
647 		return -1;
648 
649 	if (vdso_fixup_datapage(&v32, &v64))
650 		return -1;
651 
652 	if (vdso_fixup_features(&v32, &v64))
653 		return -1;
654 
655 	if (vdso_fixup_alt_funcs(&v32, &v64))
656 		return -1;
657 
658 	vdso_setup_trampolines(&v32, &v64);
659 
660 	return 0;
661 }
662 
663 /*
664  * Called from setup_arch to initialize the bitmap of available
665  * syscalls in the systemcfg page
666  */
vdso_setup_syscall_map(void)667 static void __init vdso_setup_syscall_map(void)
668 {
669 	unsigned int i;
670 	extern unsigned long *sys_call_table;
671 	extern unsigned long sys_ni_syscall;
672 
673 
674 	for (i = 0; i < __NR_syscalls; i++) {
675 #ifdef CONFIG_PPC64
676 		if (sys_call_table[i*2] != sys_ni_syscall)
677 			vdso_data->syscall_map_64[i >> 5] |=
678 				0x80000000UL >> (i & 0x1f);
679 		if (sys_call_table[i*2+1] != sys_ni_syscall)
680 			vdso_data->syscall_map_32[i >> 5] |=
681 				0x80000000UL >> (i & 0x1f);
682 #else /* CONFIG_PPC64 */
683 		if (sys_call_table[i] != sys_ni_syscall)
684 			vdso_data->syscall_map_32[i >> 5] |=
685 				0x80000000UL >> (i & 0x1f);
686 #endif /* CONFIG_PPC64 */
687 	}
688 }
689 
690 #ifdef CONFIG_PPC64
vdso_getcpu_init(void)691 int vdso_getcpu_init(void)
692 {
693 	unsigned long cpu, node, val;
694 
695 	/*
696 	 * SPRG_VDSO contains the CPU in the bottom 16 bits and the NUMA node
697 	 * in the next 16 bits.  The VDSO uses this to implement getcpu().
698 	 */
699 	cpu = get_cpu();
700 	WARN_ON_ONCE(cpu > 0xffff);
701 
702 	node = cpu_to_node(cpu);
703 	WARN_ON_ONCE(node > 0xffff);
704 
705 	val = (cpu & 0xffff) | ((node & 0xffff) << 16);
706 	mtspr(SPRN_SPRG_VDSO_WRITE, val);
707 	get_paca()->sprg_vdso = val;
708 
709 	put_cpu();
710 
711 	return 0;
712 }
713 /* We need to call this before SMP init */
714 early_initcall(vdso_getcpu_init);
715 #endif
716 
vdso_init(void)717 static int __init vdso_init(void)
718 {
719 	int i;
720 
721 #ifdef CONFIG_PPC64
722 	/*
723 	 * Fill up the "systemcfg" stuff for backward compatibility
724 	 */
725 	strcpy((char *)vdso_data->eye_catcher, "SYSTEMCFG:PPC64");
726 	vdso_data->version.major = SYSTEMCFG_MAJOR;
727 	vdso_data->version.minor = SYSTEMCFG_MINOR;
728 	vdso_data->processor = mfspr(SPRN_PVR);
729 	/*
730 	 * Fake the old platform number for pSeries and add
731 	 * in LPAR bit if necessary
732 	 */
733 	vdso_data->platform = 0x100;
734 	if (firmware_has_feature(FW_FEATURE_LPAR))
735 		vdso_data->platform |= 1;
736 	vdso_data->physicalMemorySize = memblock_phys_mem_size();
737 	vdso_data->dcache_size = ppc64_caches.dsize;
738 	vdso_data->dcache_line_size = ppc64_caches.dline_size;
739 	vdso_data->icache_size = ppc64_caches.isize;
740 	vdso_data->icache_line_size = ppc64_caches.iline_size;
741 
742 	/* XXXOJN: Blocks should be added to ppc64_caches and used instead */
743 	vdso_data->dcache_block_size = ppc64_caches.dline_size;
744 	vdso_data->icache_block_size = ppc64_caches.iline_size;
745 	vdso_data->dcache_log_block_size = ppc64_caches.log_dline_size;
746 	vdso_data->icache_log_block_size = ppc64_caches.log_iline_size;
747 
748 	/*
749 	 * Calculate the size of the 64 bits vDSO
750 	 */
751 	vdso64_pages = (&vdso64_end - &vdso64_start) >> PAGE_SHIFT;
752 	DBG("vdso64_kbase: %p, 0x%x pages\n", vdso64_kbase, vdso64_pages);
753 #else
754 	vdso_data->dcache_block_size = L1_CACHE_BYTES;
755 	vdso_data->dcache_log_block_size = L1_CACHE_SHIFT;
756 	vdso_data->icache_block_size = L1_CACHE_BYTES;
757 	vdso_data->icache_log_block_size = L1_CACHE_SHIFT;
758 #endif /* CONFIG_PPC64 */
759 
760 
761 #ifdef CONFIG_VDSO32
762 	vdso32_kbase = &vdso32_start;
763 
764 	/*
765 	 * Calculate the size of the 32 bits vDSO
766 	 */
767 	vdso32_pages = (&vdso32_end - &vdso32_start) >> PAGE_SHIFT;
768 	DBG("vdso32_kbase: %p, 0x%x pages\n", vdso32_kbase, vdso32_pages);
769 #endif
770 
771 
772 	/*
773 	 * Setup the syscall map in the vDOS
774 	 */
775 	vdso_setup_syscall_map();
776 
777 	/*
778 	 * Initialize the vDSO images in memory, that is do necessary
779 	 * fixups of vDSO symbols, locate trampolines, etc...
780 	 */
781 	if (vdso_setup()) {
782 		printk(KERN_ERR "vDSO setup failure, not enabled !\n");
783 		vdso32_pages = 0;
784 #ifdef CONFIG_PPC64
785 		vdso64_pages = 0;
786 #endif
787 		return 0;
788 	}
789 
790 #ifdef CONFIG_VDSO32
791 	/* Make sure pages are in the correct state */
792 	vdso32_pagelist = kzalloc(sizeof(struct page *) * (vdso32_pages + 2),
793 				  GFP_KERNEL);
794 	BUG_ON(vdso32_pagelist == NULL);
795 	for (i = 0; i < vdso32_pages; i++) {
796 		struct page *pg = virt_to_page(vdso32_kbase + i*PAGE_SIZE);
797 		ClearPageReserved(pg);
798 		get_page(pg);
799 		vdso32_pagelist[i] = pg;
800 	}
801 	vdso32_pagelist[i++] = virt_to_page(vdso_data);
802 	vdso32_pagelist[i] = NULL;
803 #endif
804 
805 #ifdef CONFIG_PPC64
806 	vdso64_pagelist = kzalloc(sizeof(struct page *) * (vdso64_pages + 2),
807 				  GFP_KERNEL);
808 	BUG_ON(vdso64_pagelist == NULL);
809 	for (i = 0; i < vdso64_pages; i++) {
810 		struct page *pg = virt_to_page(vdso64_kbase + i*PAGE_SIZE);
811 		ClearPageReserved(pg);
812 		get_page(pg);
813 		vdso64_pagelist[i] = pg;
814 	}
815 	vdso64_pagelist[i++] = virt_to_page(vdso_data);
816 	vdso64_pagelist[i] = NULL;
817 #endif /* CONFIG_PPC64 */
818 
819 	get_page(virt_to_page(vdso_data));
820 
821 	smp_wmb();
822 	vdso_ready = 1;
823 
824 	return 0;
825 }
826 arch_initcall(vdso_init);
827