1 // Copyright 2015, VIXL authors
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // * Neither the name of ARM Limited nor the names of its contributors may be
13 // used to endorse or promote products derived from this software without
14 // specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 #if defined(__aarch64__) && (defined(__ANDROID__) || defined(__linux__))
28 #include <sys/auxv.h>
29 #define VIXL_USE_LINUX_HWCAP 1
30 #endif
31
32 #include "../utils-vixl.h"
33
34 #include "cpu-aarch64.h"
35
36 namespace vixl {
37 namespace aarch64 {
38
39
40 const IDRegister::Field AA64PFR0::kFP(16, Field::kSigned);
41 const IDRegister::Field AA64PFR0::kAdvSIMD(20, Field::kSigned);
42 const IDRegister::Field AA64PFR0::kRAS(28);
43 const IDRegister::Field AA64PFR0::kSVE(32);
44 const IDRegister::Field AA64PFR0::kDIT(48);
45 const IDRegister::Field AA64PFR0::kCSV2(56);
46 const IDRegister::Field AA64PFR0::kCSV3(60);
47
48 const IDRegister::Field AA64PFR1::kBT(0);
49 const IDRegister::Field AA64PFR1::kSSBS(4);
50 const IDRegister::Field AA64PFR1::kMTE(8);
51
52 const IDRegister::Field AA64ISAR0::kAES(4);
53 const IDRegister::Field AA64ISAR0::kSHA1(8);
54 const IDRegister::Field AA64ISAR0::kSHA2(12);
55 const IDRegister::Field AA64ISAR0::kCRC32(16);
56 const IDRegister::Field AA64ISAR0::kAtomic(20);
57 const IDRegister::Field AA64ISAR0::kRDM(28);
58 const IDRegister::Field AA64ISAR0::kSHA3(32);
59 const IDRegister::Field AA64ISAR0::kSM3(36);
60 const IDRegister::Field AA64ISAR0::kSM4(40);
61 const IDRegister::Field AA64ISAR0::kDP(44);
62 const IDRegister::Field AA64ISAR0::kFHM(48);
63 const IDRegister::Field AA64ISAR0::kTS(52);
64 const IDRegister::Field AA64ISAR0::kRNDR(60);
65
66 const IDRegister::Field AA64ISAR1::kDPB(0);
67 const IDRegister::Field AA64ISAR1::kAPA(4);
68 const IDRegister::Field AA64ISAR1::kAPI(8);
69 const IDRegister::Field AA64ISAR1::kJSCVT(12);
70 const IDRegister::Field AA64ISAR1::kFCMA(16);
71 const IDRegister::Field AA64ISAR1::kLRCPC(20);
72 const IDRegister::Field AA64ISAR1::kGPA(24);
73 const IDRegister::Field AA64ISAR1::kGPI(28);
74 const IDRegister::Field AA64ISAR1::kFRINTTS(32);
75 const IDRegister::Field AA64ISAR1::kSB(36);
76 const IDRegister::Field AA64ISAR1::kSPECRES(40);
77 const IDRegister::Field AA64ISAR1::kBF16(44);
78 const IDRegister::Field AA64ISAR1::kDGH(48);
79 const IDRegister::Field AA64ISAR1::kI8MM(52);
80
81 const IDRegister::Field AA64ISAR2::kRPRES(4);
82
83 const IDRegister::Field AA64MMFR0::kECV(60);
84
85 const IDRegister::Field AA64MMFR1::kLO(16);
86 const IDRegister::Field AA64MMFR1::kAFP(44);
87
88 const IDRegister::Field AA64MMFR2::kAT(32);
89
90 const IDRegister::Field AA64ZFR0::kSVEver(0);
91 const IDRegister::Field AA64ZFR0::kAES(4);
92 const IDRegister::Field AA64ZFR0::kBitPerm(16);
93 const IDRegister::Field AA64ZFR0::kBF16(20);
94 const IDRegister::Field AA64ZFR0::kSHA3(32);
95 const IDRegister::Field AA64ZFR0::kSM4(40);
96 const IDRegister::Field AA64ZFR0::kI8MM(44);
97 const IDRegister::Field AA64ZFR0::kF32MM(52);
98 const IDRegister::Field AA64ZFR0::kF64MM(56);
99
GetCPUFeatures() const100 CPUFeatures AA64PFR0::GetCPUFeatures() const {
101 CPUFeatures f;
102 if (Get(kFP) >= 0) f.Combine(CPUFeatures::kFP);
103 if (Get(kFP) >= 1) f.Combine(CPUFeatures::kFPHalf);
104 if (Get(kAdvSIMD) >= 0) f.Combine(CPUFeatures::kNEON);
105 if (Get(kAdvSIMD) >= 1) f.Combine(CPUFeatures::kNEONHalf);
106 if (Get(kRAS) >= 1) f.Combine(CPUFeatures::kRAS);
107 if (Get(kSVE) >= 1) f.Combine(CPUFeatures::kSVE);
108 if (Get(kDIT) >= 1) f.Combine(CPUFeatures::kDIT);
109 if (Get(kCSV2) >= 1) f.Combine(CPUFeatures::kCSV2);
110 if (Get(kCSV2) >= 2) f.Combine(CPUFeatures::kSCXTNUM);
111 if (Get(kCSV3) >= 1) f.Combine(CPUFeatures::kCSV3);
112 return f;
113 }
114
GetCPUFeatures() const115 CPUFeatures AA64PFR1::GetCPUFeatures() const {
116 CPUFeatures f;
117 if (Get(kBT) >= 1) f.Combine(CPUFeatures::kBTI);
118 if (Get(kSSBS) >= 1) f.Combine(CPUFeatures::kSSBS);
119 if (Get(kSSBS) >= 2) f.Combine(CPUFeatures::kSSBSControl);
120 if (Get(kMTE) >= 1) f.Combine(CPUFeatures::kMTEInstructions);
121 if (Get(kMTE) >= 2) f.Combine(CPUFeatures::kMTE);
122 return f;
123 }
124
GetCPUFeatures() const125 CPUFeatures AA64ISAR0::GetCPUFeatures() const {
126 CPUFeatures f;
127 if (Get(kAES) >= 1) f.Combine(CPUFeatures::kAES);
128 if (Get(kAES) >= 2) f.Combine(CPUFeatures::kPmull1Q);
129 if (Get(kSHA1) >= 1) f.Combine(CPUFeatures::kSHA1);
130 if (Get(kSHA2) >= 1) f.Combine(CPUFeatures::kSHA2);
131 if (Get(kSHA2) >= 2) f.Combine(CPUFeatures::kSHA512);
132 if (Get(kCRC32) >= 1) f.Combine(CPUFeatures::kCRC32);
133 if (Get(kAtomic) >= 1) f.Combine(CPUFeatures::kAtomics);
134 if (Get(kRDM) >= 1) f.Combine(CPUFeatures::kRDM);
135 if (Get(kSHA3) >= 1) f.Combine(CPUFeatures::kSHA3);
136 if (Get(kSM3) >= 1) f.Combine(CPUFeatures::kSM3);
137 if (Get(kSM4) >= 1) f.Combine(CPUFeatures::kSM4);
138 if (Get(kDP) >= 1) f.Combine(CPUFeatures::kDotProduct);
139 if (Get(kFHM) >= 1) f.Combine(CPUFeatures::kFHM);
140 if (Get(kTS) >= 1) f.Combine(CPUFeatures::kFlagM);
141 if (Get(kTS) >= 2) f.Combine(CPUFeatures::kAXFlag);
142 if (Get(kRNDR) >= 1) f.Combine(CPUFeatures::kRNG);
143 return f;
144 }
145
GetCPUFeatures() const146 CPUFeatures AA64ISAR1::GetCPUFeatures() const {
147 CPUFeatures f;
148 if (Get(kDPB) >= 1) f.Combine(CPUFeatures::kDCPoP);
149 if (Get(kDPB) >= 2) f.Combine(CPUFeatures::kDCCVADP);
150 if (Get(kJSCVT) >= 1) f.Combine(CPUFeatures::kJSCVT);
151 if (Get(kFCMA) >= 1) f.Combine(CPUFeatures::kFcma);
152 if (Get(kLRCPC) >= 1) f.Combine(CPUFeatures::kRCpc);
153 if (Get(kLRCPC) >= 2) f.Combine(CPUFeatures::kRCpcImm);
154 if (Get(kFRINTTS) >= 1) f.Combine(CPUFeatures::kFrintToFixedSizedInt);
155 if (Get(kSB) >= 1) f.Combine(CPUFeatures::kSB);
156 if (Get(kSPECRES) >= 1) f.Combine(CPUFeatures::kSPECRES);
157 if (Get(kBF16) >= 1) f.Combine(CPUFeatures::kBF16);
158 if (Get(kDGH) >= 1) f.Combine(CPUFeatures::kDGH);
159 if (Get(kI8MM) >= 1) f.Combine(CPUFeatures::kI8MM);
160
161 // Only one of these fields should be non-zero, but they have the same
162 // encodings, so merge the logic.
163 int apx = std::max(Get(kAPI), Get(kAPA));
164 if (apx >= 1) {
165 f.Combine(CPUFeatures::kPAuth);
166 // APA (rather than API) indicates QARMA.
167 if (Get(kAPA) >= 1) f.Combine(CPUFeatures::kPAuthQARMA);
168 if (apx == 0b0010) f.Combine(CPUFeatures::kPAuthEnhancedPAC);
169 if (apx >= 0b0011) f.Combine(CPUFeatures::kPAuthEnhancedPAC2);
170 if (apx >= 0b0100) f.Combine(CPUFeatures::kPAuthFPAC);
171 if (apx >= 0b0101) f.Combine(CPUFeatures::kPAuthFPACCombined);
172 }
173
174 if (Get(kGPI) >= 1) f.Combine(CPUFeatures::kPAuthGeneric);
175 if (Get(kGPA) >= 1) {
176 f.Combine(CPUFeatures::kPAuthGeneric, CPUFeatures::kPAuthGenericQARMA);
177 }
178 return f;
179 }
180
GetCPUFeatures() const181 CPUFeatures AA64ISAR2::GetCPUFeatures() const {
182 CPUFeatures f;
183 if (Get(kRPRES) >= 1) f.Combine(CPUFeatures::kRPRES);
184 return f;
185 }
186
GetCPUFeatures() const187 CPUFeatures AA64MMFR0::GetCPUFeatures() const {
188 CPUFeatures f;
189 if (Get(kECV) >= 1) f.Combine(CPUFeatures::kECV);
190 return f;
191 }
192
GetCPUFeatures() const193 CPUFeatures AA64MMFR1::GetCPUFeatures() const {
194 CPUFeatures f;
195 if (Get(kLO) >= 1) f.Combine(CPUFeatures::kLORegions);
196 if (Get(kAFP) >= 1) f.Combine(CPUFeatures::kAFP);
197 return f;
198 }
199
GetCPUFeatures() const200 CPUFeatures AA64MMFR2::GetCPUFeatures() const {
201 CPUFeatures f;
202 if (Get(kAT) >= 1) f.Combine(CPUFeatures::kUSCAT);
203 return f;
204 }
205
GetCPUFeatures() const206 CPUFeatures AA64ZFR0::GetCPUFeatures() const {
207 // This register is only available with SVE, but reads-as-zero in its absence,
208 // so it's always safe to read it.
209 CPUFeatures f;
210 if (Get(kF64MM) >= 1) f.Combine(CPUFeatures::kSVEF64MM);
211 if (Get(kF32MM) >= 1) f.Combine(CPUFeatures::kSVEF32MM);
212 if (Get(kI8MM) >= 1) f.Combine(CPUFeatures::kSVEI8MM);
213 if (Get(kSM4) >= 1) f.Combine(CPUFeatures::kSVESM4);
214 if (Get(kSHA3) >= 1) f.Combine(CPUFeatures::kSVESHA3);
215 if (Get(kBF16) >= 1) f.Combine(CPUFeatures::kSVEBF16);
216 if (Get(kBitPerm) >= 1) f.Combine(CPUFeatures::kSVEBitPerm);
217 if (Get(kAES) >= 1) f.Combine(CPUFeatures::kSVEAES);
218 if (Get(kAES) >= 2) f.Combine(CPUFeatures::kSVEPmull128);
219 if (Get(kSVEver) >= 1) f.Combine(CPUFeatures::kSVE2);
220 return f;
221 }
222
Get(IDRegister::Field field) const223 int IDRegister::Get(IDRegister::Field field) const {
224 int msb = field.GetMsb();
225 int lsb = field.GetLsb();
226 VIXL_STATIC_ASSERT(static_cast<size_t>(Field::kMaxWidthInBits) <
227 (sizeof(int) * kBitsPerByte));
228 switch (field.GetType()) {
229 case Field::kSigned:
230 return static_cast<int>(ExtractSignedBitfield64(msb, lsb, value_));
231 case Field::kUnsigned:
232 return static_cast<int>(ExtractUnsignedBitfield64(msb, lsb, value_));
233 }
234 VIXL_UNREACHABLE();
235 return 0;
236 }
237
InferCPUFeaturesFromIDRegisters()238 CPUFeatures CPU::InferCPUFeaturesFromIDRegisters() {
239 CPUFeatures f;
240 #define VIXL_COMBINE_ID_REG(NAME, MRS_ARG) \
241 f.Combine(Read##NAME().GetCPUFeatures());
242 VIXL_AARCH64_ID_REG_LIST(VIXL_COMBINE_ID_REG)
243 #undef VIXL_COMBINE_ID_REG
244 return f;
245 }
246
InferCPUFeaturesFromOS(CPUFeatures::QueryIDRegistersOption option)247 CPUFeatures CPU::InferCPUFeaturesFromOS(
248 CPUFeatures::QueryIDRegistersOption option) {
249 CPUFeatures features;
250
251 #if VIXL_USE_LINUX_HWCAP
252 // Map each set bit onto a feature. Ideally, we'd use HWCAP_* macros rather
253 // than explicit bits, but explicit bits allow us to identify features that
254 // the toolchain doesn't know about.
255 static const CPUFeatures::Feature kFeatureBits[] =
256 {// Bits 0-7
257 CPUFeatures::kFP,
258 CPUFeatures::kNEON,
259 CPUFeatures::kNone, // "EVTSTRM", which VIXL doesn't track.
260 CPUFeatures::kAES,
261 CPUFeatures::kPmull1Q,
262 CPUFeatures::kSHA1,
263 CPUFeatures::kSHA2,
264 CPUFeatures::kCRC32,
265 // Bits 8-15
266 CPUFeatures::kAtomics,
267 CPUFeatures::kFPHalf,
268 CPUFeatures::kNEONHalf,
269 CPUFeatures::kIDRegisterEmulation,
270 CPUFeatures::kRDM,
271 CPUFeatures::kJSCVT,
272 CPUFeatures::kFcma,
273 CPUFeatures::kRCpc,
274 // Bits 16-23
275 CPUFeatures::kDCPoP,
276 CPUFeatures::kSHA3,
277 CPUFeatures::kSM3,
278 CPUFeatures::kSM4,
279 CPUFeatures::kDotProduct,
280 CPUFeatures::kSHA512,
281 CPUFeatures::kSVE,
282 CPUFeatures::kFHM,
283 // Bits 24-31
284 CPUFeatures::kDIT,
285 CPUFeatures::kUSCAT,
286 CPUFeatures::kRCpcImm,
287 CPUFeatures::kFlagM,
288 CPUFeatures::kSSBSControl,
289 CPUFeatures::kSB,
290 CPUFeatures::kPAuth,
291 CPUFeatures::kPAuthGeneric,
292 // Bits 32-39
293 CPUFeatures::kDCCVADP,
294 CPUFeatures::kSVE2,
295 CPUFeatures::kSVEAES,
296 CPUFeatures::kSVEPmull128,
297 CPUFeatures::kSVEBitPerm,
298 CPUFeatures::kSVESHA3,
299 CPUFeatures::kSVESM4,
300 CPUFeatures::kAXFlag,
301 // Bits 40-47
302 CPUFeatures::kFrintToFixedSizedInt,
303 CPUFeatures::kSVEI8MM,
304 CPUFeatures::kSVEF32MM,
305 CPUFeatures::kSVEF64MM,
306 CPUFeatures::kSVEBF16,
307 CPUFeatures::kI8MM,
308 CPUFeatures::kBF16,
309 CPUFeatures::kDGH,
310 // Bits 48+
311 CPUFeatures::kRNG,
312 CPUFeatures::kBTI,
313 CPUFeatures::kMTE,
314 CPUFeatures::kECV,
315 CPUFeatures::kAFP,
316 CPUFeatures::kRPRES};
317
318 uint64_t hwcap_low32 = getauxval(AT_HWCAP);
319 uint64_t hwcap_high32 = getauxval(AT_HWCAP2);
320 VIXL_ASSERT(IsUint32(hwcap_low32));
321 VIXL_ASSERT(IsUint32(hwcap_high32));
322 uint64_t hwcap = hwcap_low32 | (hwcap_high32 << 32);
323
324 VIXL_STATIC_ASSERT(ArrayLength(kFeatureBits) < 64);
325 for (size_t i = 0; i < ArrayLength(kFeatureBits); i++) {
326 if (hwcap & (UINT64_C(1) << i)) features.Combine(kFeatureBits[i]);
327 }
328 // MTE support from HWCAP2 signifies FEAT_MTE1 and FEAT_MTE2 support
329 if (features.Has(CPUFeatures::kMTE)) {
330 features.Combine(CPUFeatures::kMTEInstructions);
331 }
332 #endif // VIXL_USE_LINUX_HWCAP
333
334 if ((option == CPUFeatures::kQueryIDRegistersIfAvailable) &&
335 (features.Has(CPUFeatures::kIDRegisterEmulation))) {
336 features.Combine(InferCPUFeaturesFromIDRegisters());
337 }
338 return features;
339 }
340
341
342 #ifdef __aarch64__
343 #define VIXL_READ_ID_REG(NAME, MRS_ARG) \
344 NAME CPU::Read##NAME() { \
345 uint64_t value = 0; \
346 __asm__("mrs %0, " MRS_ARG : "=r"(value)); \
347 return NAME(value); \
348 }
349 #else // __aarch64__
350 #define VIXL_READ_ID_REG(NAME, MRS_ARG) \
351 NAME CPU::Read##NAME() { \
352 VIXL_UNREACHABLE(); \
353 return NAME(0); \
354 }
355 #endif // __aarch64__
356
357 VIXL_AARCH64_ID_REG_LIST(VIXL_READ_ID_REG)
358
359 #undef VIXL_READ_ID_REG
360
361
362 // Initialise to smallest possible cache size.
363 unsigned CPU::dcache_line_size_ = 1;
364 unsigned CPU::icache_line_size_ = 1;
365
366
367 // Currently computes I and D cache line size.
SetUp()368 void CPU::SetUp() {
369 uint32_t cache_type_register = GetCacheType();
370
371 // The cache type register holds information about the caches, including I
372 // D caches line size.
373 static const int kDCacheLineSizeShift = 16;
374 static const int kICacheLineSizeShift = 0;
375 static const uint32_t kDCacheLineSizeMask = 0xf << kDCacheLineSizeShift;
376 static const uint32_t kICacheLineSizeMask = 0xf << kICacheLineSizeShift;
377
378 // The cache type register holds the size of the I and D caches in words as
379 // a power of two.
380 uint32_t dcache_line_size_power_of_two =
381 (cache_type_register & kDCacheLineSizeMask) >> kDCacheLineSizeShift;
382 uint32_t icache_line_size_power_of_two =
383 (cache_type_register & kICacheLineSizeMask) >> kICacheLineSizeShift;
384
385 dcache_line_size_ = 4 << dcache_line_size_power_of_two;
386 icache_line_size_ = 4 << icache_line_size_power_of_two;
387 }
388
389
GetCacheType()390 uint32_t CPU::GetCacheType() {
391 #ifdef __aarch64__
392 uint64_t cache_type_register;
393 // Copy the content of the cache type register to a core register.
394 __asm__ __volatile__("mrs %[ctr], ctr_el0" // NOLINT(runtime/references)
395 : [ctr] "=r"(cache_type_register));
396 VIXL_ASSERT(IsUint32(cache_type_register));
397 return static_cast<uint32_t>(cache_type_register);
398 #else
399 // This will lead to a cache with 1 byte long lines, which is fine since
400 // neither EnsureIAndDCacheCoherency nor the simulator will need this
401 // information.
402 return 0;
403 #endif
404 }
405
406
407 // Query the SVE vector length. This requires CPUFeatures::kSVE.
ReadSVEVectorLengthInBits()408 int CPU::ReadSVEVectorLengthInBits() {
409 #ifdef __aarch64__
410 uint64_t vl;
411 // To support compilers that don't understand `rdvl`, encode the value
412 // directly and move it manually.
413 __asm__(
414 " .word 0x04bf5100\n" // rdvl x0, #8
415 " mov %[vl], x0\n"
416 : [vl] "=r"(vl)
417 :
418 : "x0");
419 VIXL_ASSERT(vl <= INT_MAX);
420 return static_cast<int>(vl);
421 #else
422 VIXL_UNREACHABLE();
423 return 0;
424 #endif
425 }
426
427
EnsureIAndDCacheCoherency(void * address,size_t length)428 void CPU::EnsureIAndDCacheCoherency(void *address, size_t length) {
429 #ifdef __aarch64__
430 // Implement the cache synchronisation for all targets where AArch64 is the
431 // host, even if we're building the simulator for an AAarch64 host. This
432 // allows for cases where the user wants to simulate code as well as run it
433 // natively.
434
435 if (length == 0) {
436 return;
437 }
438
439 // The code below assumes user space cache operations are allowed.
440
441 // Work out the line sizes for each cache, and use them to determine the
442 // start addresses.
443 uintptr_t start = reinterpret_cast<uintptr_t>(address);
444 uintptr_t dsize = static_cast<uintptr_t>(dcache_line_size_);
445 uintptr_t isize = static_cast<uintptr_t>(icache_line_size_);
446 uintptr_t dline = start & ~(dsize - 1);
447 uintptr_t iline = start & ~(isize - 1);
448
449 // Cache line sizes are always a power of 2.
450 VIXL_ASSERT(IsPowerOf2(dsize));
451 VIXL_ASSERT(IsPowerOf2(isize));
452 uintptr_t end = start + length;
453
454 do {
455 __asm__ __volatile__(
456 // Clean each line of the D cache containing the target data.
457 //
458 // dc : Data Cache maintenance
459 // c : Clean
460 // va : by (Virtual) Address
461 // u : to the point of Unification
462 // The point of unification for a processor is the point by which the
463 // instruction and data caches are guaranteed to see the same copy of a
464 // memory location. See ARM DDI 0406B page B2-12 for more information.
465 " dc cvau, %[dline]\n"
466 :
467 : [dline] "r"(dline)
468 // This code does not write to memory, but the "memory" dependency
469 // prevents GCC from reordering the code.
470 : "memory");
471 dline += dsize;
472 } while (dline < end);
473
474 __asm__ __volatile__(
475 // Make sure that the data cache operations (above) complete before the
476 // instruction cache operations (below).
477 //
478 // dsb : Data Synchronisation Barrier
479 // ish : Inner SHareable domain
480 //
481 // The point of unification for an Inner Shareable shareability domain is
482 // the point by which the instruction and data caches of all the
483 // processors
484 // in that Inner Shareable shareability domain are guaranteed to see the
485 // same copy of a memory location. See ARM DDI 0406B page B2-12 for more
486 // information.
487 " dsb ish\n"
488 :
489 :
490 : "memory");
491
492 do {
493 __asm__ __volatile__(
494 // Invalidate each line of the I cache containing the target data.
495 //
496 // ic : Instruction Cache maintenance
497 // i : Invalidate
498 // va : by Address
499 // u : to the point of Unification
500 " ic ivau, %[iline]\n"
501 :
502 : [iline] "r"(iline)
503 : "memory");
504 iline += isize;
505 } while (iline < end);
506
507 __asm__ __volatile__(
508 // Make sure that the instruction cache operations (above) take effect
509 // before the isb (below).
510 " dsb ish\n"
511
512 // Ensure that any instructions already in the pipeline are discarded and
513 // reloaded from the new data.
514 // isb : Instruction Synchronisation Barrier
515 " isb\n"
516 :
517 :
518 : "memory");
519 #else
520 // If the host isn't AArch64, we must be using the simulator, so this function
521 // doesn't have to do anything.
522 USE(address, length);
523 #endif
524 }
525
526 } // namespace aarch64
527 } // namespace vixl
528