1 /*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "linker/arm/relative_patcher_thumb2.h"
18
19 #include "arch/arm/instruction_set_features_arm.h"
20 #include "base/casts.h"
21 #include "driver/compiler_options.h"
22 #include "linker/relative_patcher_test.h"
23 #include "lock_word.h"
24 #include "mirror/array-inl.h"
25 #include "mirror/object.h"
26 #include "oat_quick_method_header.h"
27 #include "optimizing/code_generator_arm_vixl.h"
28 #include "optimizing/optimizing_unit_test.h"
29
30 namespace art {
31 namespace linker {
32
33 class Thumb2RelativePatcherTest : public RelativePatcherTest {
34 public:
Thumb2RelativePatcherTest()35 Thumb2RelativePatcherTest() : RelativePatcherTest(InstructionSet::kThumb2, "default") { }
36
37 protected:
38 static const uint8_t kCallRawCode[];
39 static const ArrayRef<const uint8_t> kCallCode;
40 static const uint8_t kNopRawCode[];
41 static const ArrayRef<const uint8_t> kNopCode;
42 static const uint8_t kUnpatchedPcRelativeRawCode[];
43 static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCode;
44 static const uint32_t kPcInsnOffset;
45
46 // The PC in Thumb mode is 4 bytes after the instruction location.
47 static constexpr uint32_t kPcAdjustment = 4u;
48
49 // Branches within range [-256, 256) can be created from these by adding the low 8 bits.
50 static constexpr uint32_t kBlPlus0 = 0xf000f800u;
51 static constexpr uint32_t kBlMinus256 = 0xf7ffff00u;
52
53 // Special BL values.
54 static constexpr uint32_t kBlPlusMax = 0xf3ffd7ffu;
55 static constexpr uint32_t kBlMinusMax = 0xf400d000u;
56
57 // BNE +0, 32-bit, encoding T3. Bits 0-10, 11, 13, 16-21, 26 are placeholder for target offset.
58 static constexpr uint32_t kBneWPlus0 = 0xf0408000u;
59
60 // LDR immediate, 16-bit, encoding T1. Bits 6-10 are imm5, 0-2 are Rt, 3-5 are Rn.
61 static constexpr uint32_t kLdrInsn = 0x6800u;
62
63 // LDR immediate, 32-bit, encoding T3. Bits 0-11 are offset, 12-15 are Rt, 16-20 are Rn.
64 static constexpr uint32_t kLdrWInsn = 0xf8d00000u;
65
66 // LDR immediate, negative offset, encoding T4. Bits 0-7 are the offset to subtract.
67 static constexpr uint32_t kLdrNegativeOffset = 0xf8500c00u;
68
69 // LDR register, lsl #2. Bits 4-5 are the imm2, i.e. the lsl shift.
70 static constexpr uint32_t kLdrRegLsl2 = 0xf8500020u;
71
72 // NOP instructions.
73 static constexpr uint32_t kNopInsn = 0xbf00u;
74 static constexpr uint32_t kNopWInsn = 0xf3af8000u;
75
InsertInsn(std::vector<uint8_t> * code,size_t pos,uint32_t insn)76 void InsertInsn(std::vector<uint8_t>* code, size_t pos, uint32_t insn) {
77 CHECK_LE(pos, code->size());
78 if (IsUint<16>(insn)) {
79 const uint8_t insn_code[] = {
80 static_cast<uint8_t>(insn),
81 static_cast<uint8_t>(insn >> 8),
82 };
83 static_assert(sizeof(insn_code) == 2u, "Invalid sizeof(insn_code).");
84 code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
85 } else {
86 const uint8_t insn_code[] = {
87 static_cast<uint8_t>(insn >> 16),
88 static_cast<uint8_t>(insn >> 24),
89 static_cast<uint8_t>(insn),
90 static_cast<uint8_t>(insn >> 8),
91 };
92 static_assert(sizeof(insn_code) == 4u, "Invalid sizeof(insn_code).");
93 code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
94 }
95 }
96
PushBackInsn(std::vector<uint8_t> * code,uint32_t insn)97 void PushBackInsn(std::vector<uint8_t>* code, uint32_t insn) {
98 InsertInsn(code, code->size(), insn);
99 }
100
GenNops(size_t num_nops)101 std::vector<uint8_t> GenNops(size_t num_nops) {
102 std::vector<uint8_t> result;
103 result.reserve(num_nops * 2u);
104 for (size_t i = 0; i != num_nops; ++i) {
105 PushBackInsn(&result, kNopInsn);
106 }
107 return result;
108 }
109
RawCode(std::initializer_list<uint32_t> insns)110 std::vector<uint8_t> RawCode(std::initializer_list<uint32_t> insns) {
111 std::vector<uint8_t> raw_code;
112 size_t number_of_16_bit_insns =
113 std::count_if(insns.begin(), insns.end(), [](uint32_t x) { return IsUint<16>(x); });
114 raw_code.reserve(insns.size() * 4u - number_of_16_bit_insns * 2u);
115 for (uint32_t insn : insns) {
116 PushBackInsn(&raw_code, insn);
117 }
118 return raw_code;
119 }
120
BneWWithOffset(uint32_t bne_offset,uint32_t target_offset)121 uint32_t BneWWithOffset(uint32_t bne_offset, uint32_t target_offset) {
122 if (!IsAligned<2u>(bne_offset)) {
123 LOG(ERROR) << "Unaligned bne_offset: " << bne_offset;
124 return 0xffffffffu; // Fails code diff later.
125 }
126 if (!IsAligned<2u>(target_offset)) {
127 LOG(ERROR) << "Unaligned target_offset: " << target_offset;
128 return 0xffffffffu; // Fails code diff later.
129 }
130 uint32_t diff = target_offset - bne_offset - kPcAdjustment;
131 DCHECK_ALIGNED(diff, 2u);
132 if ((diff >> 20) != 0 && (diff >> 20) != 0xfffu) {
133 LOG(ERROR) << "Target out of range: " << diff;
134 return 0xffffffffu; // Fails code diff later.
135 }
136 return kBneWPlus0 | ((diff >> 1) & 0x7ffu) // imm11
137 | (((diff >> 12) & 0x3fu) << 16) // imm6
138 | (((diff >> 18) & 1) << 13) // J1
139 | (((diff >> 19) & 1) << 11) // J2
140 | (((diff >> 20) & 1) << 26); // S
141 }
142
Create2MethodsWithGap(const ArrayRef<const uint8_t> & method1_code,const ArrayRef<const LinkerPatch> & method1_patches,const ArrayRef<const uint8_t> & last_method_code,const ArrayRef<const LinkerPatch> & last_method_patches,uint32_t distance_without_thunks)143 uint32_t Create2MethodsWithGap(const ArrayRef<const uint8_t>& method1_code,
144 const ArrayRef<const LinkerPatch>& method1_patches,
145 const ArrayRef<const uint8_t>& last_method_code,
146 const ArrayRef<const LinkerPatch>& last_method_patches,
147 uint32_t distance_without_thunks) {
148 CHECK_EQ(distance_without_thunks % kArmAlignment, 0u);
149 uint32_t method1_offset =
150 kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
151 AddCompiledMethod(MethodRef(1u), method1_code, method1_patches);
152 const uint32_t gap_start = method1_offset + method1_code.size();
153
154 // We want to put the last method at a very precise offset.
155 const uint32_t last_method_offset = method1_offset + distance_without_thunks;
156 CHECK_ALIGNED(last_method_offset, kArmAlignment);
157 const uint32_t gap_end = last_method_offset - sizeof(OatQuickMethodHeader);
158
159 // Fill the gap with intermediate methods in chunks of 2MiB and the first in [2MiB, 4MiB).
160 // (This allows deduplicating the small chunks to avoid using 32MiB of memory for +-16MiB
161 // offsets by this test. Making the first chunk bigger makes it easy to give all intermediate
162 // methods the same alignment of the end, so the thunk insertion adds a predictable size as
163 // long as it's after the first chunk.)
164 uint32_t method_idx = 2u;
165 constexpr uint32_t kSmallChunkSize = 2 * MB;
166 std::vector<uint8_t> gap_code;
167 uint32_t gap_size = gap_end - gap_start;
168 uint32_t num_small_chunks = std::max(gap_size / kSmallChunkSize, 1u) - 1u;
169 uint32_t chunk_start = gap_start;
170 uint32_t chunk_size = gap_size - num_small_chunks * kSmallChunkSize;
171 for (uint32_t i = 0; i <= num_small_chunks; ++i) { // num_small_chunks+1 iterations.
172 uint32_t chunk_code_size =
173 chunk_size - CodeAlignmentSize(chunk_start) - sizeof(OatQuickMethodHeader);
174 gap_code.resize(chunk_code_size, 0u);
175 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(gap_code));
176 method_idx += 1u;
177 chunk_start += chunk_size;
178 chunk_size = kSmallChunkSize; // For all but the first chunk.
179 DCHECK_EQ(CodeAlignmentSize(gap_end), CodeAlignmentSize(chunk_start));
180 }
181
182 // Add the last method and link
183 AddCompiledMethod(MethodRef(method_idx), last_method_code, last_method_patches);
184 Link();
185
186 // Check assumptions.
187 CHECK_EQ(GetMethodOffset(1), method1_offset);
188 auto last_result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
189 CHECK(last_result.first);
190 // There may be a thunk before method2.
191 if (last_result.second != last_method_offset + 1 /* thumb mode */) {
192 // Thunk present. Check that there's only one.
193 uint32_t thunk_end =
194 CompiledCode::AlignCode(gap_end, InstructionSet::kThumb2) + MethodCallThunkSize();
195 uint32_t header_offset = thunk_end + CodeAlignmentSize(thunk_end);
196 CHECK_EQ(last_result.second,
197 header_offset + sizeof(OatQuickMethodHeader) + 1 /* thumb mode */);
198 }
199 return method_idx;
200 }
201
GetMethodOffset(uint32_t method_idx)202 uint32_t GetMethodOffset(uint32_t method_idx) {
203 auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
204 CHECK(result.first);
205 CHECK_NE(result.second & 1u, 0u);
206 return result.second - 1 /* thumb mode */;
207 }
208
CompileThunk(const LinkerPatch & patch,std::string * debug_name=nullptr)209 std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
210 /*out*/ std::string* debug_name = nullptr) {
211 OptimizingUnitTestHelper helper;
212 HGraph* graph = helper.CreateGraph();
213 CompilerOptions compiler_options;
214 arm::CodeGeneratorARMVIXL codegen(graph, compiler_options);
215 ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
216 codegen.EmitThunkCode(patch, &code, debug_name);
217 return std::vector<uint8_t>(code.begin(), code.end());
218 }
219
AddCompiledMethod(MethodReference method_ref,const ArrayRef<const uint8_t> & code,const ArrayRef<const LinkerPatch> & patches=ArrayRef<const LinkerPatch> ())220 void AddCompiledMethod(
221 MethodReference method_ref,
222 const ArrayRef<const uint8_t>& code,
223 const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
224 RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
225
226 // Make sure the ThunkProvider has all the necessary thunks.
227 for (const LinkerPatch& patch : patches) {
228 if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
229 patch.GetType() == LinkerPatch::Type::kCallRelative) {
230 std::string debug_name;
231 std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
232 thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
233 }
234 }
235 }
236
CompileMethodCallThunk()237 std::vector<uint8_t> CompileMethodCallThunk() {
238 LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
239 /* target_dex_file*/ nullptr,
240 /* target_method_idx */ 0u);
241 return CompileThunk(patch);
242 }
243
MethodCallThunkSize()244 uint32_t MethodCallThunkSize() {
245 return CompileMethodCallThunk().size();
246 }
247
CheckThunk(uint32_t thunk_offset)248 bool CheckThunk(uint32_t thunk_offset) {
249 const std::vector<uint8_t> expected_code = CompileMethodCallThunk();
250 if (output_.size() < thunk_offset + expected_code.size()) {
251 LOG(ERROR) << "output_.size() == " << output_.size() << " < "
252 << "thunk_offset + expected_code.size() == " << (thunk_offset + expected_code.size());
253 return false;
254 }
255 ArrayRef<const uint8_t> linked_code(&output_[thunk_offset], expected_code.size());
256 if (linked_code == ArrayRef<const uint8_t>(expected_code)) {
257 return true;
258 }
259 // Log failure info.
260 DumpDiff(ArrayRef<const uint8_t>(expected_code), linked_code);
261 return false;
262 }
263
GenNopsAndBl(size_t num_nops,uint32_t bl)264 std::vector<uint8_t> GenNopsAndBl(size_t num_nops, uint32_t bl) {
265 std::vector<uint8_t> result;
266 result.reserve(num_nops * 2u + 4u);
267 for (size_t i = 0; i != num_nops; ++i) {
268 PushBackInsn(&result, kNopInsn);
269 }
270 PushBackInsn(&result, bl);
271 return result;
272 }
273
274 void TestStringBssEntry(uint32_t bss_begin, uint32_t string_entry_offset);
275 void TestStringReference(uint32_t string_offset);
276 void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset);
277
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg,bool narrow)278 static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg,
279 uint32_t holder_reg,
280 bool narrow) {
281 return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow);
282 }
283
EncodeBakerReadBarrierArrayData(uint32_t base_reg)284 static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
285 return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg);
286 }
287
EncodeBakerReadBarrierGcRootData(uint32_t root_reg,bool narrow)288 static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) {
289 return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow);
290 }
291
CompileBakerOffsetThunk(uint32_t base_reg,uint32_t holder_reg,bool narrow)292 std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg,
293 uint32_t holder_reg,
294 bool narrow) {
295 const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
296 /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow));
297 return CompileThunk(patch);
298 }
299
CompileBakerArrayThunk(uint32_t base_reg)300 std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
301 LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
302 /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
303 return CompileThunk(patch);
304 }
305
CompileBakerGcRootThunk(uint32_t root_reg,bool narrow)306 std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) {
307 LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
308 /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow));
309 return CompileThunk(patch);
310 }
311
GetOutputInsn32(uint32_t offset)312 uint32_t GetOutputInsn32(uint32_t offset) {
313 CHECK_LE(offset, output_.size());
314 CHECK_GE(output_.size() - offset, 4u);
315 return (static_cast<uint32_t>(output_[offset]) << 16) |
316 (static_cast<uint32_t>(output_[offset + 1]) << 24) |
317 (static_cast<uint32_t>(output_[offset + 2]) << 0) |
318 (static_cast<uint32_t>(output_[offset + 3]) << 8);
319 }
320
GetOutputInsn16(uint32_t offset)321 uint16_t GetOutputInsn16(uint32_t offset) {
322 CHECK_LE(offset, output_.size());
323 CHECK_GE(output_.size() - offset, 2u);
324 return (static_cast<uint32_t>(output_[offset]) << 0) |
325 (static_cast<uint32_t>(output_[offset + 1]) << 8);
326 }
327
328 void TestBakerFieldWide(uint32_t offset, uint32_t ref_reg);
329 void TestBakerFieldNarrow(uint32_t offset, uint32_t ref_reg);
330 };
331
332 const uint8_t Thumb2RelativePatcherTest::kCallRawCode[] = {
333 0x00, 0xf0, 0x00, 0xf8
334 };
335
336 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kCallCode(kCallRawCode);
337
338 const uint8_t Thumb2RelativePatcherTest::kNopRawCode[] = {
339 0x00, 0xbf
340 };
341
342 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kNopCode(kNopRawCode);
343
344 const uint8_t Thumb2RelativePatcherTest::kUnpatchedPcRelativeRawCode[] = {
345 0x40, 0xf2, 0x00, 0x00, // MOVW r0, #0 (placeholder)
346 0xc0, 0xf2, 0x00, 0x00, // MOVT r0, #0 (placeholder)
347 0x78, 0x44, // ADD r0, pc
348 };
349 const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kUnpatchedPcRelativeCode(
350 kUnpatchedPcRelativeRawCode);
351 const uint32_t Thumb2RelativePatcherTest::kPcInsnOffset = 8u;
352
TestStringBssEntry(uint32_t bss_begin,uint32_t string_entry_offset)353 void Thumb2RelativePatcherTest::TestStringBssEntry(uint32_t bss_begin,
354 uint32_t string_entry_offset) {
355 constexpr uint32_t kStringIndex = 1u;
356 string_index_to_offset_map_.Put(kStringIndex, string_entry_offset);
357 bss_begin_ = bss_begin;
358 const LinkerPatch patches[] = {
359 LinkerPatch::StringBssEntryPatch(0u, nullptr, kPcInsnOffset, kStringIndex),
360 LinkerPatch::StringBssEntryPatch(4u, nullptr, kPcInsnOffset, kStringIndex),
361 };
362 CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), bss_begin_ + string_entry_offset);
363 }
364
TestStringReference(uint32_t string_offset)365 void Thumb2RelativePatcherTest::TestStringReference(uint32_t string_offset) {
366 constexpr uint32_t kStringIndex = 1u;
367 string_index_to_offset_map_.Put(kStringIndex, string_offset);
368 const LinkerPatch patches[] = {
369 LinkerPatch::RelativeStringPatch(0u, nullptr, kPcInsnOffset, kStringIndex),
370 LinkerPatch::RelativeStringPatch(4u, nullptr, kPcInsnOffset, kStringIndex),
371 };
372 CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), string_offset);
373 }
374
CheckPcRelativePatch(const ArrayRef<const LinkerPatch> & patches,uint32_t target_offset)375 void Thumb2RelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches,
376 uint32_t target_offset) {
377 AddCompiledMethod(MethodRef(1u), kUnpatchedPcRelativeCode, ArrayRef<const LinkerPatch>(patches));
378 Link();
379
380 uint32_t method1_offset = GetMethodOffset(1u);
381 uint32_t pc_base_offset = method1_offset + kPcInsnOffset + 4u /* PC adjustment */;
382 uint32_t diff = target_offset - pc_base_offset;
383 // Distribute the bits of the diff between the MOVW and MOVT:
384 uint32_t diffw = diff & 0xffffu;
385 uint32_t difft = diff >> 16;
386 uint32_t movw = 0xf2400000u | // MOVW r0, #0 (placeholder),
387 ((diffw & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
388 ((diffw & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
389 ((diffw & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
390 ((diffw & 0x00ffu)); // keep imm8 at bits 0-7.
391 uint32_t movt = 0xf2c00000u | // MOVT r0, #0 (placeholder),
392 ((difft & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
393 ((difft & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
394 ((difft & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
395 ((difft & 0x00ffu)); // keep imm8 at bits 0-7.
396 const uint8_t expected_code[] = {
397 static_cast<uint8_t>(movw >> 16), static_cast<uint8_t>(movw >> 24),
398 static_cast<uint8_t>(movw >> 0), static_cast<uint8_t>(movw >> 8),
399 static_cast<uint8_t>(movt >> 16), static_cast<uint8_t>(movt >> 24),
400 static_cast<uint8_t>(movt >> 0), static_cast<uint8_t>(movt >> 8),
401 0x78, 0x44,
402 };
403 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
404 }
405
TEST_F(Thumb2RelativePatcherTest,CallSelf)406 TEST_F(Thumb2RelativePatcherTest, CallSelf) {
407 const LinkerPatch patches[] = {
408 LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
409 };
410 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
411 Link();
412
413 static const uint8_t expected_code[] = {
414 0xff, 0xf7, 0xfe, 0xff
415 };
416 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
417 }
418
TEST_F(Thumb2RelativePatcherTest,CallOther)419 TEST_F(Thumb2RelativePatcherTest, CallOther) {
420 const LinkerPatch method1_patches[] = {
421 LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
422 };
423 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(method1_patches));
424 const LinkerPatch method2_patches[] = {
425 LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
426 };
427 AddCompiledMethod(MethodRef(2u), kCallCode, ArrayRef<const LinkerPatch>(method2_patches));
428 Link();
429
430 uint32_t method1_offset = GetMethodOffset(1u);
431 uint32_t method2_offset = GetMethodOffset(2u);
432 uint32_t diff_after = method2_offset - (method1_offset + 4u /* PC adjustment */);
433 ASSERT_EQ(diff_after & 1u, 0u);
434 ASSERT_LT(diff_after >> 1, 1u << 8); // Simple encoding, (diff_after >> 1) fits into 8 bits.
435 static const uint8_t method1_expected_code[] = {
436 0x00, 0xf0, static_cast<uint8_t>(diff_after >> 1), 0xf8
437 };
438 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code)));
439 uint32_t diff_before = method1_offset - (method2_offset + 4u /* PC adjustment */);
440 ASSERT_EQ(diff_before & 1u, 0u);
441 ASSERT_GE(diff_before, -1u << 9); // Simple encoding, -256 <= (diff >> 1) < 0.
442 auto method2_expected_code = GenNopsAndBl(0u, kBlMinus256 | ((diff_before >> 1) & 0xffu));
443 EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code)));
444 }
445
TEST_F(Thumb2RelativePatcherTest,CallTrampoline)446 TEST_F(Thumb2RelativePatcherTest, CallTrampoline) {
447 const LinkerPatch patches[] = {
448 LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
449 };
450 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
451 Link();
452
453 uint32_t method1_offset = GetMethodOffset(1u);
454 uint32_t diff = kTrampolineOffset - (method1_offset + 4u);
455 ASSERT_EQ(diff & 1u, 0u);
456 ASSERT_GE(diff, -1u << 9); // Simple encoding, -256 <= (diff >> 1) < 0 (checked as unsigned).
457 auto expected_code = GenNopsAndBl(0u, kBlMinus256 | ((diff >> 1) & 0xffu));
458 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
459 }
460
TEST_F(Thumb2RelativePatcherTest,CallTrampolineTooFar)461 TEST_F(Thumb2RelativePatcherTest, CallTrampolineTooFar) {
462 constexpr uint32_t missing_method_index = 1024u;
463 auto last_method_raw_code = GenNopsAndBl(3u, kBlPlus0);
464 constexpr uint32_t bl_offset_in_last_method = 3u * 2u; // After NOPs.
465 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
466 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
467 const LinkerPatch last_method_patches[] = {
468 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, missing_method_index),
469 };
470
471 constexpr uint32_t just_over_max_negative_disp = 16 * MB + 2 - 4u /* PC adjustment */;
472 uint32_t last_method_idx = Create2MethodsWithGap(
473 kNopCode,
474 ArrayRef<const LinkerPatch>(),
475 last_method_code,
476 ArrayRef<const LinkerPatch>(last_method_patches),
477 just_over_max_negative_disp - bl_offset_in_last_method);
478 uint32_t method1_offset = GetMethodOffset(1u);
479 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
480 ASSERT_EQ(method1_offset,
481 last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
482 ASSERT_FALSE(method_offset_map_.FindMethodOffset(MethodRef(missing_method_index)).first);
483
484 // Check linked code.
485 uint32_t thunk_offset = CompiledCode::AlignCode(
486 last_method_offset + last_method_code.size(), InstructionSet::kThumb2);
487 uint32_t diff =
488 thunk_offset - (last_method_offset + bl_offset_in_last_method + 4u /* PC adjustment */);
489 ASSERT_TRUE(IsAligned<2u>(diff));
490 ASSERT_LT(diff >> 1, 1u << 8); // Simple encoding, (diff >> 1) fits into 8 bits.
491 auto expected_code = GenNopsAndBl(3u, kBlPlus0 | ((diff >> 1) & 0xffu));
492 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
493 ArrayRef<const uint8_t>(expected_code)));
494 EXPECT_TRUE(CheckThunk(thunk_offset));
495 }
496
TEST_F(Thumb2RelativePatcherTest,CallOtherAlmostTooFarAfter)497 TEST_F(Thumb2RelativePatcherTest, CallOtherAlmostTooFarAfter) {
498 auto method1_raw_code = GenNopsAndBl(3u, kBlPlus0);
499 constexpr uint32_t bl_offset_in_method1 = 3u * 2u; // After NOPs.
500 ArrayRef<const uint8_t> method1_code(method1_raw_code);
501 ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
502 const uint32_t kExpectedLastMethodIdx = 9u; // Based on 2MiB chunks in Create2MethodsWithGap().
503 const LinkerPatch method1_patches[] = {
504 LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
505 };
506
507 constexpr uint32_t max_positive_disp = 16 * MB - 2u + 4u /* PC adjustment */;
508 uint32_t last_method_idx = Create2MethodsWithGap(method1_code,
509 ArrayRef<const LinkerPatch>(method1_patches),
510 kNopCode,
511 ArrayRef<const LinkerPatch>(),
512 bl_offset_in_method1 + max_positive_disp);
513 ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
514
515 // Check linked code.
516 auto expected_code = GenNopsAndBl(3u, kBlPlusMax);
517 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
518 }
519
TEST_F(Thumb2RelativePatcherTest,CallOtherAlmostTooFarBefore)520 TEST_F(Thumb2RelativePatcherTest, CallOtherAlmostTooFarBefore) {
521 auto last_method_raw_code = GenNopsAndBl(2u, kBlPlus0);
522 constexpr uint32_t bl_offset_in_last_method = 2u * 2u; // After NOPs.
523 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
524 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
525 const LinkerPatch last_method_patches[] = {
526 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
527 };
528
529 constexpr uint32_t max_negative_disp = 16 * MB - 4u /* PC adjustment */;
530 uint32_t last_method_idx = Create2MethodsWithGap(kNopCode,
531 ArrayRef<const LinkerPatch>(),
532 last_method_code,
533 ArrayRef<const LinkerPatch>(last_method_patches),
534 max_negative_disp - bl_offset_in_last_method);
535 uint32_t method1_offset = GetMethodOffset(1u);
536 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
537 ASSERT_EQ(method1_offset, last_method_offset + bl_offset_in_last_method - max_negative_disp);
538
539 // Check linked code.
540 auto expected_code = GenNopsAndBl(2u, kBlMinusMax);
541 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
542 ArrayRef<const uint8_t>(expected_code)));
543 }
544
TEST_F(Thumb2RelativePatcherTest,CallOtherJustTooFarAfter)545 TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarAfter) {
546 auto method1_raw_code = GenNopsAndBl(2u, kBlPlus0);
547 constexpr uint32_t bl_offset_in_method1 = 2u * 2u; // After NOPs.
548 ArrayRef<const uint8_t> method1_code(method1_raw_code);
549 ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
550 const uint32_t kExpectedLastMethodIdx = 9u; // Based on 2MiB chunks in Create2MethodsWithGap().
551 const LinkerPatch method1_patches[] = {
552 LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
553 };
554
555 constexpr uint32_t just_over_max_positive_disp = 16 * MB + 4u /* PC adjustment */;
556 uint32_t last_method_idx = Create2MethodsWithGap(
557 method1_code,
558 ArrayRef<const LinkerPatch>(method1_patches),
559 kNopCode,
560 ArrayRef<const LinkerPatch>(),
561 bl_offset_in_method1 + just_over_max_positive_disp);
562 ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
563 uint32_t method_after_thunk_idx = last_method_idx;
564 if (sizeof(OatQuickMethodHeader) < kArmAlignment) {
565 // The thunk needs to start on a kArmAlignment-aligned address before the address where the
566 // last method would have been if there was no thunk. If the size of the OatQuickMethodHeader
567 // is at least kArmAlignment, the thunk start shall fit between the previous filler method
568 // and that address. Otherwise, it shall be inserted before that filler method.
569 method_after_thunk_idx -= 1u;
570 }
571
572 uint32_t method1_offset = GetMethodOffset(1u);
573 uint32_t method_after_thunk_offset = GetMethodOffset(method_after_thunk_idx);
574 ASSERT_TRUE(IsAligned<kArmAlignment>(method_after_thunk_offset));
575 uint32_t method_after_thunk_header_offset =
576 method_after_thunk_offset - sizeof(OatQuickMethodHeader);
577 uint32_t thunk_size = MethodCallThunkSize();
578 uint32_t thunk_offset = RoundDown(method_after_thunk_header_offset - thunk_size, kArmAlignment);
579 DCHECK_EQ(thunk_offset + thunk_size + CodeAlignmentSize(thunk_offset + thunk_size),
580 method_after_thunk_header_offset);
581 ASSERT_TRUE(IsAligned<kArmAlignment>(thunk_offset));
582 uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1 + 4u /* PC adjustment */);
583 ASSERT_TRUE(IsAligned<2u>(diff));
584 ASSERT_GE(diff, 16 * MB - (1u << 22)); // Simple encoding, unknown bits fit into imm10:imm11:0.
585 auto expected_code =
586 GenNopsAndBl(2u, 0xf000d000 | ((diff >> 1) & 0x7ffu) | (((diff >> 12) & 0x3ffu) << 16));
587 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
588 CheckThunk(thunk_offset);
589 }
590
TEST_F(Thumb2RelativePatcherTest,CallOtherJustTooFarBefore)591 TEST_F(Thumb2RelativePatcherTest, CallOtherJustTooFarBefore) {
592 auto last_method_raw_code = GenNopsAndBl(3u, kBlPlus0);
593 constexpr uint32_t bl_offset_in_last_method = 3u * 2u; // After NOPs.
594 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
595 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
596 const LinkerPatch last_method_patches[] = {
597 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
598 };
599
600 constexpr uint32_t just_over_max_negative_disp = 16 * MB + 2 - 4u /* PC adjustment */;
601 uint32_t last_method_idx = Create2MethodsWithGap(
602 kNopCode,
603 ArrayRef<const LinkerPatch>(),
604 last_method_code,
605 ArrayRef<const LinkerPatch>(last_method_patches),
606 just_over_max_negative_disp - bl_offset_in_last_method);
607 uint32_t method1_offset = GetMethodOffset(1u);
608 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
609 ASSERT_EQ(method1_offset,
610 last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
611
612 // Check linked code.
613 uint32_t thunk_offset = CompiledCode::AlignCode(
614 last_method_offset + last_method_code.size(), InstructionSet::kThumb2);
615 uint32_t diff =
616 thunk_offset - (last_method_offset + bl_offset_in_last_method + 4u /* PC adjustment */);
617 ASSERT_TRUE(IsAligned<2u>(diff));
618 ASSERT_LT(diff >> 1, 1u << 8); // Simple encoding, (diff >> 1) fits into 8 bits.
619 auto expected_code = GenNopsAndBl(3u, kBlPlus0 | ((diff >> 1) & 0xffu));
620 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
621 ArrayRef<const uint8_t>(expected_code)));
622 EXPECT_TRUE(CheckThunk(thunk_offset));
623 }
624
TEST_F(Thumb2RelativePatcherTest,StringBssEntry1)625 TEST_F(Thumb2RelativePatcherTest, StringBssEntry1) {
626 TestStringBssEntry(0x00ff0000u, 0x00fcu);
627 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
628 }
629
TEST_F(Thumb2RelativePatcherTest,StringBssEntry2)630 TEST_F(Thumb2RelativePatcherTest, StringBssEntry2) {
631 TestStringBssEntry(0x02ff0000u, 0x05fcu);
632 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
633 }
634
TEST_F(Thumb2RelativePatcherTest,StringBssEntry3)635 TEST_F(Thumb2RelativePatcherTest, StringBssEntry3) {
636 TestStringBssEntry(0x08ff0000u, 0x08fcu);
637 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
638 }
639
TEST_F(Thumb2RelativePatcherTest,StringBssEntry4)640 TEST_F(Thumb2RelativePatcherTest, StringBssEntry4) {
641 TestStringBssEntry(0xd0ff0000u, 0x60fcu);
642 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
643 }
644
TEST_F(Thumb2RelativePatcherTest,StringReference1)645 TEST_F(Thumb2RelativePatcherTest, StringReference1) {
646 TestStringReference(0x00ff00fcu);
647 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
648 }
649
TEST_F(Thumb2RelativePatcherTest,StringReference2)650 TEST_F(Thumb2RelativePatcherTest, StringReference2) {
651 TestStringReference(0x02ff05fcu);
652 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
653 }
654
TEST_F(Thumb2RelativePatcherTest,StringReference3)655 TEST_F(Thumb2RelativePatcherTest, StringReference3) {
656 TestStringReference(0x08ff08fcu);
657 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
658 }
659
TEST_F(Thumb2RelativePatcherTest,StringReference4)660 TEST_F(Thumb2RelativePatcherTest, StringReference4) {
661 TestStringReference(0xd0ff60fcu);
662 ASSERT_LT(GetMethodOffset(1u), 0xfcu);
663 }
664
665 const uint32_t kBakerValidRegs[] = {
666 0, 1, 2, 3, 4, 5, 6, 7,
667 9, 10, 11, // r8 (rMR), IP, SP, LR and PC are reserved.
668 };
669
670 const uint32_t kBakerValidRegsNarrow[] = {
671 0, 1, 2, 3, 4, 5, 6, 7,
672 };
673
TestBakerFieldWide(uint32_t offset,uint32_t ref_reg)674 void Thumb2RelativePatcherTest::TestBakerFieldWide(uint32_t offset, uint32_t ref_reg) {
675 DCHECK_ALIGNED(offset, 4u);
676 DCHECK_LT(offset, 4 * KB);
677 constexpr size_t kMethodCodeSize = 8u;
678 constexpr size_t kLiteralOffset = 0u;
679 uint32_t method_idx = 0u;
680 for (uint32_t base_reg : kBakerValidRegs) {
681 for (uint32_t holder_reg : kBakerValidRegs) {
682 uint32_t ldr = kLdrWInsn | offset | (base_reg << 16) | (ref_reg << 12);
683 const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
684 ASSERT_EQ(kMethodCodeSize, raw_code.size());
685 ArrayRef<const uint8_t> code(raw_code);
686 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
687 base_reg, holder_reg, /* narrow */ false);
688 const LinkerPatch patches[] = {
689 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
690 };
691 ++method_idx;
692 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
693 }
694 }
695 Link();
696
697 // All thunks are at the end.
698 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
699 method_idx = 0u;
700 for (uint32_t base_reg : kBakerValidRegs) {
701 for (uint32_t holder_reg : kBakerValidRegs) {
702 ++method_idx;
703 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
704 uint32_t ldr = kLdrWInsn | offset | (base_reg << 16) | (ref_reg << 12);
705 const std::vector<uint8_t> expected_code = RawCode({bne, ldr});
706 ASSERT_EQ(kMethodCodeSize, expected_code.size()) << "bne=0x" << std::hex << bne;
707 ASSERT_TRUE(
708 CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
709
710 std::vector<uint8_t> expected_thunk =
711 CompileBakerOffsetThunk(base_reg, holder_reg, /* narrow */ false);
712 ASSERT_GT(output_.size(), thunk_offset);
713 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
714 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
715 expected_thunk.size());
716 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
717 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
718 ASSERT_TRUE(false);
719 }
720
721 size_t gray_check_offset = thunk_offset;
722 if (holder_reg == base_reg) {
723 // Verify that the null-check uses the correct register, i.e. holder_reg.
724 if (holder_reg < 8) {
725 ASSERT_GE(output_.size() - gray_check_offset, 2u);
726 ASSERT_EQ(0xb100 | holder_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
727 gray_check_offset +=2u;
728 } else {
729 ASSERT_GE(output_.size() - gray_check_offset, 6u);
730 ASSERT_EQ(0xf1b00f00u | (holder_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
731 ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u); // BEQ
732 gray_check_offset += 6u;
733 }
734 }
735 // Verify that the lock word for gray bit check is loaded from the holder address.
736 ASSERT_GE(output_.size() - gray_check_offset,
737 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
738 const uint32_t load_lock_word =
739 kLdrWInsn |
740 (holder_reg << 16) |
741 (/* IP */ 12 << 12) |
742 mirror::Object::MonitorOffset().Uint32Value();
743 ASSERT_EQ(load_lock_word, GetOutputInsn32(gray_check_offset));
744 // Verify the gray bit check.
745 DCHECK_GE(LockWord::kReadBarrierStateShift, 8u); // ROR modified immediate.
746 uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
747 const uint32_t tst_gray_bit_without_offset =
748 0xf0100f00 | (/* IP */ 12 << 16)
749 | (((ror_shift >> 4) & 1) << 26) // i
750 | (((ror_shift >> 1) & 7) << 12) // imm3
751 | ((ror_shift & 1) << 7); // imm8, ROR('1':imm8<7:0>, ror_shift).
752 EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(gray_check_offset + 4u));
753 EXPECT_EQ(0xd100u, GetOutputInsn16(gray_check_offset + 8u) & 0xff00u); // BNE
754 // Verify the fake dependency (skip "ADD LR, LR, #ldr_offset").
755 const uint32_t fake_dependency =
756 0xeb000010 | // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
757 (/* IP */ 12) | // Rm = IP
758 (base_reg << 16) | // Rn = base_reg
759 (base_reg << 8); // Rd = base_reg
760 EXPECT_EQ(fake_dependency, GetOutputInsn32(gray_check_offset + 14u));
761 // Do not check the rest of the implementation.
762
763 // The next thunk follows on the next aligned offset.
764 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
765 }
766 }
767 }
768
TestBakerFieldNarrow(uint32_t offset,uint32_t ref_reg)769 void Thumb2RelativePatcherTest::TestBakerFieldNarrow(uint32_t offset, uint32_t ref_reg) {
770 DCHECK_ALIGNED(offset, 4u);
771 DCHECK_LT(offset, 32u);
772 constexpr size_t kMethodCodeSize = 6u;
773 constexpr size_t kLiteralOffset = 0u;
774 uint32_t method_idx = 0u;
775 for (uint32_t base_reg : kBakerValidRegs) {
776 if (base_reg >= 8u) {
777 continue;
778 }
779 for (uint32_t holder_reg : kBakerValidRegs) {
780 uint32_t ldr = kLdrInsn | (offset << (6 - 2)) | (base_reg << 3) | ref_reg;
781 const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr});
782 ASSERT_EQ(kMethodCodeSize, raw_code.size());
783 ArrayRef<const uint8_t> code(raw_code);
784 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
785 base_reg, holder_reg, /* narrow */ true);
786 const LinkerPatch patches[] = {
787 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
788 };
789 ++method_idx;
790 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
791 }
792 }
793 Link();
794
795 // All thunks are at the end.
796 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
797 method_idx = 0u;
798 for (uint32_t base_reg : kBakerValidRegs) {
799 if (base_reg >= 8u) {
800 continue;
801 }
802 for (uint32_t holder_reg : kBakerValidRegs) {
803 ++method_idx;
804 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
805 uint32_t ldr = kLdrInsn | (offset << (6 - 2)) | (base_reg << 3) | ref_reg;
806 const std::vector<uint8_t> expected_code = RawCode({bne, ldr});
807 ASSERT_EQ(kMethodCodeSize, expected_code.size()) << "bne=0x" << std::hex << bne;
808 ASSERT_TRUE(
809 CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
810
811 std::vector<uint8_t> expected_thunk =
812 CompileBakerOffsetThunk(base_reg, holder_reg, /* narrow */ true);
813 ASSERT_GT(output_.size(), thunk_offset);
814 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
815 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
816 expected_thunk.size());
817 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
818 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
819 ASSERT_TRUE(false);
820 }
821
822 size_t gray_check_offset = thunk_offset;
823 if (holder_reg == base_reg) {
824 // Verify that the null-check uses the correct register, i.e. holder_reg.
825 if (holder_reg < 8) {
826 ASSERT_GE(output_.size() - gray_check_offset, 2u);
827 ASSERT_EQ(0xb100 | holder_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
828 gray_check_offset +=2u;
829 } else {
830 ASSERT_GE(output_.size() - gray_check_offset, 6u);
831 ASSERT_EQ(0xf1b00f00u | (holder_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
832 ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u); // BEQ
833 gray_check_offset += 6u;
834 }
835 }
836 // Verify that the lock word for gray bit check is loaded from the holder address.
837 ASSERT_GE(output_.size() - gray_check_offset,
838 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
839 const uint32_t load_lock_word =
840 kLdrWInsn |
841 (holder_reg << 16) |
842 (/* IP */ 12 << 12) |
843 mirror::Object::MonitorOffset().Uint32Value();
844 ASSERT_EQ(load_lock_word, GetOutputInsn32(gray_check_offset));
845 // Verify the gray bit check.
846 DCHECK_GE(LockWord::kReadBarrierStateShift, 8u); // ROR modified immediate.
847 uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
848 const uint32_t tst_gray_bit_without_offset =
849 0xf0100f00 | (/* IP */ 12 << 16)
850 | (((ror_shift >> 4) & 1) << 26) // i
851 | (((ror_shift >> 1) & 7) << 12) // imm3
852 | ((ror_shift & 1) << 7); // imm8, ROR('1':imm8<7:0>, ror_shift).
853 EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(gray_check_offset + 4u));
854 EXPECT_EQ(0xd100u, GetOutputInsn16(gray_check_offset + 8u) & 0xff00u); // BNE
855 // Verify the fake dependency (skip "ADD LR, LR, #ldr_offset").
856 const uint32_t fake_dependency =
857 0xeb000010 | // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
858 (/* IP */ 12) | // Rm = IP
859 (base_reg << 16) | // Rn = base_reg
860 (base_reg << 8); // Rd = base_reg
861 EXPECT_EQ(fake_dependency, GetOutputInsn32(gray_check_offset + 14u));
862 // Do not check the rest of the implementation.
863
864 // The next thunk follows on the next aligned offset.
865 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
866 }
867 }
868 }
869
TEST_F(Thumb2RelativePatcherTest,BakerOffsetWide)870 TEST_F(Thumb2RelativePatcherTest, BakerOffsetWide) {
871 struct TestCase {
872 uint32_t offset;
873 uint32_t ref_reg;
874 };
875 static const TestCase test_cases[] = {
876 { 0u, 0u },
877 { 8u, 3u },
878 { 28u, 7u },
879 { 0xffcu, 11u },
880 };
881 for (const TestCase& test_case : test_cases) {
882 Reset();
883 TestBakerFieldWide(test_case.offset, test_case.ref_reg);
884 }
885 }
886
TEST_F(Thumb2RelativePatcherTest,BakerOffsetNarrow)887 TEST_F(Thumb2RelativePatcherTest, BakerOffsetNarrow) {
888 struct TestCase {
889 uint32_t offset;
890 uint32_t ref_reg;
891 };
892 static const TestCase test_cases[] = {
893 { 0, 0u },
894 { 8, 3u },
895 { 28, 7u },
896 };
897 for (const TestCase& test_case : test_cases) {
898 Reset();
899 TestBakerFieldNarrow(test_case.offset, test_case.ref_reg);
900 }
901 }
902
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkInTheMiddle)903 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddle) {
904 // One thunk in the middle with maximum distance branches to it from both sides.
905 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
906 constexpr uint32_t kLiteralOffset1 = 6u;
907 const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
908 ArrayRef<const uint8_t> code1(raw_code1);
909 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
910 /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
911 const LinkerPatch patches1[] = {
912 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
913 };
914 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
915
916 constexpr uint32_t expected_thunk_offset =
917 kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement */ ((1 << 20) - 2u);
918 static_assert(IsAligned<kArmAlignment>(expected_thunk_offset), "Target offset must be aligned.");
919 size_t filler1_size = expected_thunk_offset -
920 RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment);
921 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
922 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
923 AddCompiledMethod(MethodRef(2u), filler1_code);
924
925 // Enforce thunk reservation with a tiny method.
926 AddCompiledMethod(MethodRef(3u), kNopCode);
927
928 constexpr uint32_t kLiteralOffset2 = 4;
929 static_assert(IsAligned<kArmAlignment>(kLiteralOffset2 + kPcAdjustment),
930 "PC for BNE must be aligned.");
931
932 // Allow reaching the thunk from the very beginning of a method almost 1MiB away. Backward branch
933 // reaches the full 1MiB but we need to take PC adjustment into account. Things to subtract:
934 // - thunk size and method 3 pre-header, rounded up (padding in between if needed)
935 // - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
936 // - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
937 size_t thunk_size =
938 CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false).size();
939 size_t filler2_size =
940 1 * MB - (kLiteralOffset2 + kPcAdjustment)
941 - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArmAlignment)
942 - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArmAlignment)
943 - sizeof(OatQuickMethodHeader);
944 std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 2u);
945 ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
946 AddCompiledMethod(MethodRef(4u), filler2_code);
947
948 const std::vector<uint8_t> raw_code2 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn});
949 ArrayRef<const uint8_t> code2(raw_code2);
950 const LinkerPatch patches2[] = {
951 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
952 };
953 AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
954
955 Link();
956
957 uint32_t first_method_offset = GetMethodOffset(1u);
958 uint32_t last_method_offset = GetMethodOffset(5u);
959 EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
960
961 const uint32_t bne_max_forward = kBneWPlus0 | 0x003f2fff;
962 const uint32_t bne_max_backward = kBneWPlus0 | 0x04000000;
963 const std::vector<uint8_t> expected_code1 =
964 RawCode({kNopWInsn, kNopInsn, bne_max_forward, kLdrWInsn});
965 const std::vector<uint8_t> expected_code2 = RawCode({kNopWInsn, bne_max_backward, kLdrWInsn});
966 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
967 ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
968 }
969
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkBeforeFiller)970 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkBeforeFiller) {
971 // Based on the first part of BakerOffsetThunkInTheMiddle but the BNE is one instruction
972 // earlier, so the thunk is emitted before the filler.
973 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
974 constexpr uint32_t kLiteralOffset1 = 4u;
975 const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn});
976 ArrayRef<const uint8_t> code1(raw_code1);
977 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
978 /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
979 const LinkerPatch patches1[] = {
980 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
981 };
982 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
983
984 constexpr uint32_t expected_thunk_offset =
985 kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement + 2 */ (1u << 20);
986 static_assert(IsAligned<kArmAlignment>(expected_thunk_offset), "Target offset must be aligned.");
987 size_t filler1_size = expected_thunk_offset -
988 RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment);
989 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
990 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
991 AddCompiledMethod(MethodRef(2u), filler1_code);
992
993 Link();
994
995 const uint32_t bne = BneWWithOffset(kLiteralOffset1, RoundUp(raw_code1.size(), kArmAlignment));
996 const std::vector<uint8_t> expected_code1 = RawCode({kNopWInsn, bne, kLdrWInsn, kNopInsn});
997 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
998 }
999
TEST_F(Thumb2RelativePatcherTest,BakerOffsetThunkInTheMiddleUnreachableFromLast)1000 TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddleUnreachableFromLast) {
1001 // Based on the BakerOffsetThunkInTheMiddle but the BNE in the last method is preceded
1002 // by NOP and cannot reach the thunk in the middle, so we emit an extra thunk at the end.
1003 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1004 constexpr uint32_t kLiteralOffset1 = 6u;
1005 const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
1006 ArrayRef<const uint8_t> code1(raw_code1);
1007 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(
1008 /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false);
1009 const LinkerPatch patches1[] = {
1010 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1011 };
1012 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1013
1014 constexpr uint32_t expected_thunk_offset =
1015 kLiteralOffset1 + kPcAdjustment + /* kMaxBcondPositiveDisplacement */ ((1 << 20) - 2u);
1016 static_assert(IsAligned<kArmAlignment>(expected_thunk_offset), "Target offset must be aligned.");
1017 size_t filler1_size = expected_thunk_offset -
1018 RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment);
1019 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 2u);
1020 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1021 AddCompiledMethod(MethodRef(2u), filler1_code);
1022
1023 // Enforce thunk reservation with a tiny method.
1024 AddCompiledMethod(MethodRef(3u), kNopCode);
1025
1026 constexpr uint32_t kReachableFromOffset2 = 4;
1027 constexpr uint32_t kLiteralOffset2 = kReachableFromOffset2 + 2;
1028 static_assert(IsAligned<kArmAlignment>(kReachableFromOffset2 + kPcAdjustment),
1029 "PC for BNE must be aligned.");
1030
1031 // If not for the extra NOP, this would allow reaching the thunk from the BNE
1032 // of a method 1MiB away. Backward branch reaches the full 1MiB but we need to take
1033 // PC adjustment into account. Things to subtract:
1034 // - thunk size and method 3 pre-header, rounded up (padding in between if needed)
1035 // - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
1036 // - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
1037 size_t thunk_size =
1038 CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false).size();
1039 size_t filler2_size =
1040 1 * MB - (kReachableFromOffset2 + kPcAdjustment)
1041 - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArmAlignment)
1042 - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArmAlignment)
1043 - sizeof(OatQuickMethodHeader);
1044 std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 2u);
1045 ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
1046 AddCompiledMethod(MethodRef(4u), filler2_code);
1047
1048 // Extra 16-bit NOP compared to BakerOffsetThunkInTheMiddle.
1049 const std::vector<uint8_t> raw_code2 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn});
1050 ArrayRef<const uint8_t> code2(raw_code2);
1051 const LinkerPatch patches2[] = {
1052 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
1053 };
1054 AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
1055
1056 Link();
1057
1058 uint32_t first_method_offset = GetMethodOffset(1u);
1059 uint32_t last_method_offset = GetMethodOffset(5u);
1060 EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
1061
1062 const uint32_t bne_max_forward = kBneWPlus0 | 0x003f2fff;
1063 const uint32_t bne_last =
1064 BneWWithOffset(kLiteralOffset2, RoundUp(raw_code2.size(), kArmAlignment));
1065 const std::vector<uint8_t> expected_code1 =
1066 RawCode({kNopWInsn, kNopInsn, bne_max_forward, kLdrWInsn});
1067 const std::vector<uint8_t> expected_code2 =
1068 RawCode({kNopWInsn, kNopInsn, bne_last, kLdrWInsn});
1069 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1070 ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
1071 }
1072
TEST_F(Thumb2RelativePatcherTest,BakerArray)1073 TEST_F(Thumb2RelativePatcherTest, BakerArray) {
1074 auto ldr = [](uint32_t base_reg) {
1075 uint32_t index_reg = (base_reg == 0u) ? 1u : 0u;
1076 uint32_t ref_reg = (base_reg == 2) ? 3u : 2u;
1077 return kLdrRegLsl2 | index_reg | (base_reg << 16) | (ref_reg << 12);
1078 };
1079 constexpr size_t kMethodCodeSize = 8u;
1080 constexpr size_t kLiteralOffset = 0u;
1081 uint32_t method_idx = 0u;
1082 for (uint32_t base_reg : kBakerValidRegs) {
1083 ++method_idx;
1084 const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr(base_reg)});
1085 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1086 ArrayRef<const uint8_t> code(raw_code);
1087 const LinkerPatch patches[] = {
1088 LinkerPatch::BakerReadBarrierBranchPatch(
1089 kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
1090 };
1091 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1092 }
1093 Link();
1094
1095 // All thunks are at the end.
1096 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
1097 method_idx = 0u;
1098 for (uint32_t base_reg : kBakerValidRegs) {
1099 ++method_idx;
1100 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1101 const std::vector<uint8_t> expected_code = RawCode({bne, ldr(base_reg)});
1102 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1103 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1104
1105 std::vector<uint8_t> expected_thunk = CompileBakerArrayThunk(base_reg);
1106 ASSERT_GT(output_.size(), thunk_offset);
1107 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1108 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1109 expected_thunk.size());
1110 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1111 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1112 ASSERT_TRUE(false);
1113 }
1114
1115 // Verify that the lock word for gray bit check is loaded from the correct address
1116 // before the base_reg which points to the array data.
1117 ASSERT_GE(output_.size() - thunk_offset,
1118 4u * /* 32-bit instructions */ 4u + 2u * /* 16-bit instructions */ 2u);
1119 int32_t data_offset =
1120 mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
1121 int32_t offset = mirror::Object::MonitorOffset().Int32Value() - data_offset;
1122 ASSERT_LT(offset, 0);
1123 ASSERT_GT(offset, -256);
1124 const uint32_t load_lock_word =
1125 kLdrNegativeOffset |
1126 (-offset & 0xffu) |
1127 (base_reg << 16) |
1128 (/* IP */ 12 << 12);
1129 EXPECT_EQ(load_lock_word, GetOutputInsn32(thunk_offset));
1130 // Verify the gray bit check.
1131 DCHECK_GE(LockWord::kReadBarrierStateShift, 8u); // ROR modified immediate.
1132 uint32_t ror_shift = 7 + (32 - LockWord::kReadBarrierStateShift);
1133 const uint32_t tst_gray_bit_without_offset =
1134 0xf0100f00 | (/* IP */ 12 << 16)
1135 | (((ror_shift >> 4) & 1) << 26) // i
1136 | (((ror_shift >> 1) & 7) << 12) // imm3
1137 | ((ror_shift & 1) << 7); // imm8, ROR('1':imm8<7:0>, ror_shift).
1138 EXPECT_EQ(tst_gray_bit_without_offset, GetOutputInsn32(thunk_offset + 4u));
1139 EXPECT_EQ(0xd100u, GetOutputInsn16(thunk_offset + 8u) & 0xff00u); // BNE
1140 // Verify the fake dependency.
1141 const uint32_t fake_dependency =
1142 0xeb000010 | // ADD Rd, Rn, Rm, LSR 32 (type=01, imm3=000, imm2=00)
1143 (/* IP */ 12) | // Rm = IP
1144 (base_reg << 16) | // Rn = base_reg
1145 (base_reg << 8); // Rd = base_reg
1146 EXPECT_EQ(fake_dependency, GetOutputInsn32(thunk_offset + 14u));
1147 // Do not check the rest of the implementation.
1148
1149 // The next thunk follows on the next aligned offset.
1150 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
1151 }
1152 }
1153
TEST_F(Thumb2RelativePatcherTest,BakerGcRootWide)1154 TEST_F(Thumb2RelativePatcherTest, BakerGcRootWide) {
1155 constexpr size_t kMethodCodeSize = 8u;
1156 constexpr size_t kLiteralOffset = 4u;
1157 uint32_t method_idx = 0u;
1158 for (uint32_t root_reg : kBakerValidRegs) {
1159 ++method_idx;
1160 uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (root_reg << 12);
1161 const std::vector<uint8_t> raw_code = RawCode({ldr, kBneWPlus0});
1162 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1163 ArrayRef<const uint8_t> code(raw_code);
1164 const LinkerPatch patches[] = {
1165 LinkerPatch::BakerReadBarrierBranchPatch(
1166 kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)),
1167 };
1168 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1169 }
1170 Link();
1171
1172 // All thunks are at the end.
1173 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
1174 method_idx = 0u;
1175 for (uint32_t root_reg : kBakerValidRegs) {
1176 ++method_idx;
1177 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1178 uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (root_reg << 12);
1179 const std::vector<uint8_t> expected_code = RawCode({ldr, bne});
1180 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1181 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1182
1183 std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg, /* narrow */ false);
1184 ASSERT_GT(output_.size(), thunk_offset);
1185 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1186 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1187 expected_thunk.size());
1188 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1189 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1190 ASSERT_TRUE(false);
1191 }
1192
1193 // Verify that the fast-path null-check uses the correct register, i.e. root_reg.
1194 if (root_reg < 8) {
1195 ASSERT_GE(output_.size() - thunk_offset, 2u);
1196 ASSERT_EQ(0xb100 | root_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
1197 } else {
1198 ASSERT_GE(output_.size() - thunk_offset, 6u);
1199 ASSERT_EQ(0xf1b00f00u | (root_reg << 16), GetOutputInsn32(thunk_offset) & 0xfbff8f00u);
1200 ASSERT_EQ(0xd000u, GetOutputInsn16(thunk_offset + 4u) & 0xff00u); // BEQ
1201 }
1202 // Do not check the rest of the implementation.
1203
1204 // The next thunk follows on the next aligned offset.
1205 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
1206 }
1207 }
1208
TEST_F(Thumb2RelativePatcherTest,BakerGcRootNarrow)1209 TEST_F(Thumb2RelativePatcherTest, BakerGcRootNarrow) {
1210 constexpr size_t kMethodCodeSize = 6u;
1211 constexpr size_t kLiteralOffset = 2u;
1212 uint32_t method_idx = 0u;
1213 for (uint32_t root_reg : kBakerValidRegsNarrow) {
1214 ++method_idx;
1215 uint32_t ldr = kLdrInsn | (/* offset */ 8 << (6 - 2)) | (/* base_reg */ 0 << 3) | root_reg;
1216 const std::vector<uint8_t> raw_code = RawCode({ldr, kBneWPlus0});
1217 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1218 ArrayRef<const uint8_t> code(raw_code);
1219 const LinkerPatch patches[] = {
1220 LinkerPatch::BakerReadBarrierBranchPatch(
1221 kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)),
1222 };
1223 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1224 }
1225 Link();
1226
1227 // All thunks are at the end.
1228 uint32_t thunk_offset = GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArmAlignment);
1229 method_idx = 0u;
1230 for (uint32_t root_reg : kBakerValidRegsNarrow) {
1231 ++method_idx;
1232 uint32_t bne = BneWWithOffset(GetMethodOffset(method_idx) + kLiteralOffset, thunk_offset);
1233 uint32_t ldr = kLdrInsn | (/* offset */ 8 << (6 - 2)) | (/* base_reg */ 0 << 3) | root_reg;
1234 const std::vector<uint8_t> expected_code = RawCode({ldr, bne});
1235 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1236 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1237
1238 std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg, /* narrow */ true);
1239 ASSERT_GT(output_.size(), thunk_offset);
1240 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1241 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1242 expected_thunk.size());
1243 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1244 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1245 ASSERT_TRUE(false);
1246 }
1247
1248 // Verify that the fast-path null-check CBZ uses the correct register, i.e. root_reg.
1249 ASSERT_GE(output_.size() - thunk_offset, 2u);
1250 ASSERT_EQ(0xb100 | root_reg, GetOutputInsn16(thunk_offset) & 0xfd07u);
1251 // Do not check the rest of the implementation.
1252
1253 // The next thunk follows on the next aligned offset.
1254 thunk_offset += RoundUp(expected_thunk.size(), kArmAlignment);
1255 }
1256 }
1257
TEST_F(Thumb2RelativePatcherTest,BakerGcRootOffsetBits)1258 TEST_F(Thumb2RelativePatcherTest, BakerGcRootOffsetBits) {
1259 // Test 1MiB of patches to the same thunk to stress-test different large offsets.
1260 // (The low bits are not that important but the location of the high bits is easy to get wrong.)
1261 std::vector<uint8_t> code;
1262 code.reserve(1 * MB);
1263 const size_t num_patches = 1 * MB / 8u;
1264 std::vector<LinkerPatch> patches;
1265 patches.reserve(num_patches);
1266 const uint32_t ldr =
1267 kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12);
1268 uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false);
1269 for (size_t i = 0; i != num_patches; ++i) {
1270 PushBackInsn(&code, ldr);
1271 PushBackInsn(&code, kBneWPlus0);
1272 patches.push_back(LinkerPatch::BakerReadBarrierBranchPatch(8u * i + 4u, encoded_data));
1273 }
1274 ASSERT_EQ(1 * MB, code.size());
1275 ASSERT_EQ(num_patches, patches.size());
1276 AddCompiledMethod(MethodRef(1u),
1277 ArrayRef<const uint8_t>(code),
1278 ArrayRef<const LinkerPatch>(patches));
1279 Link();
1280
1281 // The thunk is right after the method code.
1282 DCHECK_ALIGNED(1 * MB, kArmAlignment);
1283 std::vector<uint8_t> expected_code;
1284 for (size_t i = 0; i != num_patches; ++i) {
1285 PushBackInsn(&expected_code, ldr);
1286 PushBackInsn(&expected_code, BneWWithOffset(8u * i + 4u, 1 * MB));
1287 patches.push_back(LinkerPatch::BakerReadBarrierBranchPatch(8u * i + 4u, encoded_data));
1288 }
1289 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
1290 }
1291
TEST_F(Thumb2RelativePatcherTest,BakerAndMethodCallInteraction)1292 TEST_F(Thumb2RelativePatcherTest, BakerAndMethodCallInteraction) {
1293 // During development, there was a `DCHECK_LE(MaxNextOffset(), next_thunk.MaxNextOffset());`
1294 // in `ArmBaseRelativePatcher::ThunkData::MakeSpaceBefore()` which does not necessarily
1295 // hold when we're reserving thunks of different sizes. This test exposes the situation
1296 // by using Baker thunks and a method call thunk.
1297
1298 // Add a method call patch that can reach to method 1 offset + 16MiB.
1299 uint32_t method_idx = 0u;
1300 constexpr size_t kMethodCallLiteralOffset = 2u;
1301 constexpr uint32_t kMissingMethodIdx = 2u;
1302 const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kBlPlus0});
1303 const LinkerPatch method1_patches[] = {
1304 LinkerPatch::RelativeCodePatch(kMethodCallLiteralOffset, nullptr, 2u),
1305 };
1306 ArrayRef<const uint8_t> code1(raw_code1);
1307 ++method_idx;
1308 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(method1_patches));
1309
1310 // Skip kMissingMethodIdx.
1311 ++method_idx;
1312 ASSERT_EQ(kMissingMethodIdx, method_idx);
1313 // Add a method with the right size that the method code for the next one starts 1MiB
1314 // after code for method 1.
1315 size_t filler_size =
1316 1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArmAlignment)
1317 - sizeof(OatQuickMethodHeader);
1318 std::vector<uint8_t> filler_code = GenNops(filler_size / 2u);
1319 ++method_idx;
1320 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1321 // Add 14 methods with 1MiB code+header, making the code for the next method start 1MiB
1322 // before the currently scheduled MaxNextOffset() for the method call thunk.
1323 for (uint32_t i = 0; i != 14; ++i) {
1324 filler_size = 1 * MB - sizeof(OatQuickMethodHeader);
1325 filler_code = GenNops(filler_size / 2u);
1326 ++method_idx;
1327 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1328 }
1329
1330 // Add 2 Baker GC root patches to the last method, one that would allow the thunk at
1331 // 1MiB + kArmAlignment, i.e. kArmAlignment after the method call thunk, and the
1332 // second that needs it kArmAlignment after that. Given the size of the GC root thunk
1333 // is more than the space required by the method call thunk plus kArmAlignment,
1334 // this pushes the first GC root thunk's pending MaxNextOffset() before the method call
1335 // thunk's pending MaxNextOffset() which needs to be adjusted.
1336 ASSERT_LT(RoundUp(CompileMethodCallThunk().size(), kArmAlignment) + kArmAlignment,
1337 CompileBakerGcRootThunk(/* root_reg */ 0, /* narrow */ false).size());
1338 static_assert(kArmAlignment == 8, "Code below assumes kArmAlignment == 8");
1339 constexpr size_t kBakerLiteralOffset1 = kArmAlignment + 2u - kPcAdjustment;
1340 constexpr size_t kBakerLiteralOffset2 = kBakerLiteralOffset1 + kArmAlignment;
1341 // Use offset = 0, base_reg = 0, the LDR is simply `kLdrWInsn | (root_reg << 12)`.
1342 const uint32_t ldr1 = kLdrWInsn | (/* root_reg */ 1 << 12);
1343 const uint32_t ldr2 = kLdrWInsn | (/* root_reg */ 2 << 12);
1344 const std::vector<uint8_t> last_method_raw_code = RawCode({
1345 kNopInsn, // Padding before first GC root read barrier.
1346 ldr1, kBneWPlus0, // First GC root LDR with read barrier.
1347 ldr2, kBneWPlus0, // Second GC root LDR with read barrier.
1348 });
1349 uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false);
1350 uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false);
1351 const LinkerPatch last_method_patches[] = {
1352 LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
1353 LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
1354 };
1355 ++method_idx;
1356 AddCompiledMethod(MethodRef(method_idx),
1357 ArrayRef<const uint8_t>(last_method_raw_code),
1358 ArrayRef<const LinkerPatch>(last_method_patches));
1359
1360 // The main purpose of the test is to check that Link() does not cause a crash.
1361 Link();
1362
1363 ASSERT_EQ(15 * MB, GetMethodOffset(method_idx) - GetMethodOffset(1u));
1364 }
1365
1366 } // namespace linker
1367 } // namespace art
1368