1 /*
2 *
3 * Copyright 2013 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * cache.c: Cache maintenance routines for ARM64-A and ARM64-R
29 *
30 * Reference: ARM64 Architecture Reference Manual, ARM64-A and ARM64-R edition
31 */
32
33 #include <stdint.h>
34
35 #include <arch/cache.h>
36 #include <arch/lib_helpers.h>
37
tlb_invalidate_all(void)38 void tlb_invalidate_all(void)
39 {
40 /* TLBIALL includes dTLB and iTLB on systems that have them. */
41 tlbiall_el2();
42 dsb();
43 isb();
44 }
45
46 enum dcache_op {
47 OP_DCCSW,
48 OP_DCCISW,
49 OP_DCISW,
50 OP_DCCIVAC,
51 OP_DCCVAC,
52 OP_DCIVAC,
53 };
54
dcache_line_bytes(void)55 unsigned int dcache_line_bytes(void)
56 {
57 uint32_t ccsidr;
58 static unsigned int line_bytes = 0;
59
60 if (line_bytes)
61 return line_bytes;
62
63 ccsidr = raw_read_ccsidr_el1();
64 /* [2:0] - Indicates (Log2(number of words in cache line)) - 2 */
65 line_bytes = 1 << ((ccsidr & 0x7) + 2); /* words per line */
66 line_bytes *= sizeof(unsigned int); /* bytes per word */
67
68 return line_bytes;
69 }
70
71 /*
72 * Do a dcache operation by virtual address. This is useful for
73 * maintaining coherency in drivers which do DMA transfers and only need to
74 * perform cache maintenance on a particular memory range rather than the
75 * entire cache.
76 */
dcache_op_va(void const * addr,size_t len,enum dcache_op op)77 static void dcache_op_va(void const *addr, size_t len, enum dcache_op op)
78 {
79 unsigned long line, linesize;
80
81 linesize = dcache_line_bytes();
82 line = (uintptr_t)addr & ~(linesize - 1);
83
84 dsb();
85 while (line < (uintptr_t)addr + len) {
86 switch(op) {
87 case OP_DCCIVAC:
88 dccivac(line);
89 break;
90 case OP_DCCVAC:
91 dccvac(line);
92 break;
93 case OP_DCIVAC:
94 dcivac(line);
95 break;
96 default:
97 break;
98 }
99 line += linesize;
100 }
101 isb();
102 }
103
dcache_clean_by_mva(void const * addr,size_t len)104 void dcache_clean_by_mva(void const *addr, size_t len)
105 {
106 dcache_op_va(addr, len, OP_DCCVAC);
107 }
108
dcache_clean_invalidate_by_mva(void const * addr,size_t len)109 void dcache_clean_invalidate_by_mva(void const *addr, size_t len)
110 {
111 dcache_op_va(addr, len, OP_DCCIVAC);
112 }
113
dcache_invalidate_by_mva(void const * addr,size_t len)114 void dcache_invalidate_by_mva(void const *addr, size_t len)
115 {
116 dcache_op_va(addr, len, OP_DCIVAC);
117 }
118
cache_sync_instructions(void)119 void cache_sync_instructions(void)
120 {
121 uint32_t sctlr = raw_read_sctlr_el2();
122 if (sctlr & SCTLR_C)
123 dcache_clean_all(); /* includes trailing DSB (assembly) */
124 else if (sctlr & SCTLR_I)
125 dcache_clean_invalidate_all();
126 icache_invalidate_all(); /* includes leading DSB and trailing ISB */
127 }
128
arch_program_segment_loaded(void const * addr,size_t len)129 void arch_program_segment_loaded(void const *addr, size_t len)
130 {
131 dcache_clean_invalidate_by_mva(addr, len);
132 icache_invalidate_all();
133 }
134