1 /*
2 * Copyright (c) 2015 Google Inc. All rights reserved
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining
5 * a copy of this software and associated documentation files
6 * (the "Software"), to deal in the Software without restriction,
7 * including without limitation the rights to use, copy, modify, merge,
8 * publish, distribute, sublicense, and/or sell copies of the Software,
9 * and to permit persons to whom the Software is furnished to do so,
10 * subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be
13 * included in all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <arch/arm64.h>
25 #include <kernel/thread.h>
26 #include <trace.h>
27
28 #define LOCAL_TRACE 0
29
30 static struct fpstate *current_fpstate[SMP_MAX_CPUS];
31
arm64_fpu_load_state(struct thread * t)32 static void arm64_fpu_load_state(struct thread *t)
33 {
34 uint cpu = arch_curr_cpu_num();
35 struct fpstate *fpstate = &t->arch.fpstate;
36
37 if (fpstate == current_fpstate[cpu] && fpstate->current_cpu == cpu) {
38 LTRACEF("cpu %d, thread %s, fpstate already valid\n", cpu, t->name);
39 return;
40 }
41 LTRACEF("cpu %d, thread %s, load fpstate %p, last cpu %d, last fpstate %p\n",
42 cpu, t->name, fpstate, fpstate->current_cpu, current_fpstate[cpu]);
43 fpstate->current_cpu = cpu;
44 current_fpstate[cpu] = fpstate;
45
46
47 STATIC_ASSERT(sizeof(fpstate->regs) == 16 * 32);
48 __asm__ volatile("ldp q0, q1, [%0, #(0 * 32)]\n"
49 "ldp q2, q3, [%0, #(1 * 32)]\n"
50 "ldp q4, q5, [%0, #(2 * 32)]\n"
51 "ldp q6, q7, [%0, #(3 * 32)]\n"
52 "ldp q8, q9, [%0, #(4 * 32)]\n"
53 "ldp q10, q11, [%0, #(5 * 32)]\n"
54 "ldp q12, q13, [%0, #(6 * 32)]\n"
55 "ldp q14, q15, [%0, #(7 * 32)]\n"
56 "ldp q16, q17, [%0, #(8 * 32)]\n"
57 "ldp q18, q19, [%0, #(9 * 32)]\n"
58 "ldp q20, q21, [%0, #(10 * 32)]\n"
59 "ldp q22, q23, [%0, #(11 * 32)]\n"
60 "ldp q24, q25, [%0, #(12 * 32)]\n"
61 "ldp q26, q27, [%0, #(13 * 32)]\n"
62 "ldp q28, q29, [%0, #(14 * 32)]\n"
63 "ldp q30, q31, [%0, #(15 * 32)]\n"
64 "msr fpcr, %1\n"
65 "msr fpsr, %2\n"
66 :: "r"(fpstate),
67 "r"((uint64_t)fpstate->fpcr),
68 "r"((uint64_t)fpstate->fpsr));
69 }
70
arm64_fpu_save_state(struct thread * t)71 void arm64_fpu_save_state(struct thread *t)
72 {
73 uint64_t fpcr, fpsr;
74 struct fpstate *fpstate = &t->arch.fpstate;
75 __asm__ volatile("stp q0, q1, [%2, #(0 * 32)]\n"
76 "stp q2, q3, [%2, #(1 * 32)]\n"
77 "stp q4, q5, [%2, #(2 * 32)]\n"
78 "stp q6, q7, [%2, #(3 * 32)]\n"
79 "stp q8, q9, [%2, #(4 * 32)]\n"
80 "stp q10, q11, [%2, #(5 * 32)]\n"
81 "stp q12, q13, [%2, #(6 * 32)]\n"
82 "stp q14, q15, [%2, #(7 * 32)]\n"
83 "stp q16, q17, [%2, #(8 * 32)]\n"
84 "stp q18, q19, [%2, #(9 * 32)]\n"
85 "stp q20, q21, [%2, #(10 * 32)]\n"
86 "stp q22, q23, [%2, #(11 * 32)]\n"
87 "stp q24, q25, [%2, #(12 * 32)]\n"
88 "stp q26, q27, [%2, #(13 * 32)]\n"
89 "stp q28, q29, [%2, #(14 * 32)]\n"
90 "stp q30, q31, [%2, #(15 * 32)]\n"
91 "mrs %0, fpcr\n"
92 "mrs %1, fpsr\n"
93 : "=r"(fpcr), "=r"(fpsr)
94 : "r"(fpstate));
95
96 fpstate->fpcr = fpcr;
97 fpstate->fpsr = fpsr;
98
99 LTRACEF("thread %s, fpcr %x, fpsr %x\n", t->name, fpstate->fpcr, fpstate->fpsr);
100 }
101
arm64_fpu_exception(struct arm64_iframe_long * iframe)102 void arm64_fpu_exception(struct arm64_iframe_long *iframe)
103 {
104 uint64_t cpacr = ARM64_READ_SYSREG(cpacr_el1);
105 if (((cpacr >> 20) & 3) != 3) {
106 cpacr |= 3 << 20;
107 ARM64_WRITE_SYSREG(cpacr_el1, cpacr);
108 thread_t *t = get_current_thread();
109 if (likely(t))
110 arm64_fpu_load_state(t);
111 return;
112 }
113 }
114