1 /*
2 * Copyright (c) 2023-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "frame_builder.h"
17 #include "llvm_ark_interface.h"
18
19 #include "utils/bit_field.h"
20 #include "libpandabase/utils/cframe_layout.h"
21 #include "libpandabase/utils/arch.h"
22 #include "compiler/optimizer/code_generator/target_info.h"
23
24 #include <llvm/CodeGen/GlobalISel/MachineIRBuilder.h>
25 #include <llvm/CodeGen/TargetInstrInfo.h>
26 #include <llvm/IR/InlineAsm.h>
27 #include <llvm/Target/TargetMachine.h>
28
29 struct InlineAsmBuilder {
InlineAsmBuilderInlineAsmBuilder30 InlineAsmBuilder(llvm::MachineBasicBlock *block, llvm::MachineBasicBlock::iterator iterator)
31 : mblock_ {block}, insertionPoint_ {iterator}
32 {
33 }
34 ~InlineAsmBuilder() = default;
35
36 InlineAsmBuilder(const InlineAsmBuilder &) = delete;
37 void operator=(const InlineAsmBuilder &) = delete;
38 InlineAsmBuilder(InlineAsmBuilder &&) = delete;
39 InlineAsmBuilder &operator=(InlineAsmBuilder &&) = delete;
40
CreateInlineAsmInlineAsmBuilder41 void CreateInlineAsm(std::string_view inlineAsm, llvm::ArrayRef<ssize_t> imms = {})
42 {
43 ASSERT(mblock_ != nullptr);
44
45 auto mfunc = mblock_->getParent();
46 auto &targetInstructionInfo = *mfunc->getSubtarget().getInstrInfo();
47 auto &asmInstructionDescriptor = targetInstructionInfo.get(llvm::TargetOpcode::INLINEASM);
48 auto mib = llvm::BuildMI(*mblock_, insertionPoint_, llvm::DebugLoc(), asmInstructionDescriptor);
49 mib.addExternalSymbol(inlineAsm.data());
50 mib.addImm(static_cast<unsigned>(llvm::InlineAsm::Extra_HasSideEffects));
51
52 for (auto value : imms) {
53 mib.addImm(llvm::InlineAsm::getFlagWord(llvm::InlineAsm::Kind_Imm, 1U));
54 mib.add(llvm::ArrayRef {llvm::MachineOperand::CreateImm(value)});
55 }
56 }
57
58 private:
59 llvm::MachineBasicBlock *mblock_ {nullptr};
60 llvm::MachineBasicBlock::iterator insertionPoint_;
61 };
62
63 namespace {
64
RemoveInstsIf(llvm::MachineBasicBlock & mblock,const std::function<bool (llvm::MachineInstr &)> & predicate)65 void RemoveInstsIf(llvm::MachineBasicBlock &mblock, const std::function<bool(llvm::MachineInstr &)> &predicate)
66 {
67 std::vector<llvm::MachineInstr *> markedAsRemoved;
68 for (auto &inst : mblock) {
69 if (predicate(inst)) {
70 markedAsRemoved.push_back(&inst);
71 }
72 }
73
74 for (auto inst : markedAsRemoved) {
75 inst->removeFromParent();
76 }
77 }
78
79 } // namespace
80
81 // AMD64 Frame builder implementation
82
RemovePrologue(llvm::MachineBasicBlock & mblock)83 bool AMD64FrameBuilder::RemovePrologue(llvm::MachineBasicBlock &mblock)
84 {
85 auto predicate = [](llvm::MachineInstr &inst) -> bool {
86 bool isFrameSetup = inst.getFlag(llvm::MachineInstr::FrameSetup);
87 return isFrameSetup;
88 };
89 RemoveInstsIf(mblock, predicate);
90 return mblock.getPrevNode() == nullptr;
91 }
92
RemoveEpilogue(llvm::MachineBasicBlock & mblock)93 bool AMD64FrameBuilder::RemoveEpilogue(llvm::MachineBasicBlock &mblock)
94 {
95 bool isEpilogue = false;
96 auto predicate = [&isEpilogue](llvm::MachineInstr &inst) -> bool {
97 bool isFrameDestroy = inst.getFlag(llvm::MachineInstr::FrameDestroy);
98 // Sometimes llvm may generate code, when there is no any frame-destroy instructions
99 // that contain RET. And we still need to remember such basic blocks.
100 isEpilogue |= inst.isReturn();
101 return isFrameDestroy;
102 };
103 RemoveInstsIf(mblock, predicate);
104 return isEpilogue;
105 }
106
InsertPrologue(llvm::MachineBasicBlock & mblock)107 void AMD64FrameBuilder::InsertPrologue(llvm::MachineBasicBlock &mblock)
108 {
109 auto [xregsMask, vregsMask] = frameInfo_.regMasks;
110 if (!frameInfo_.hasCalls && !frameInfo_.usesStack && xregsMask == 0 && vregsMask == 0) {
111 // Do not generate any code
112 return;
113 }
114
115 InlineAsmBuilder builder(&mblock, mblock.begin());
116
117 constexpr ark::CFrameLayout FL(ark::Arch::X86_64, 0);
118 constexpr auto SP_ORIGIN = ark::CFrameLayout::OffsetOrigin::SP;
119 constexpr auto BYTES_UNITS = ark::CFrameLayout::OffsetUnit::BYTES;
120
121 constexpr ssize_t SLOT_SIZE = ark::PointerSize(ark::Arch::X86_64);
122 constexpr ssize_t FRAME_SIZE = FL.GetFrameSize<BYTES_UNITS>();
123 constexpr ssize_t METHOD_OFFSET = FL.GetOffset<SP_ORIGIN, BYTES_UNITS>(ark::CFrameLayout::MethodSlot::Start());
124 constexpr ssize_t FLAGS_OFFSET = FL.GetOffset<SP_ORIGIN, BYTES_UNITS>(ark::CFrameLayout::FlagsSlot::Start());
125 constexpr ssize_t CALLEE_OFFSET = FL.GetOffset<SP_ORIGIN, BYTES_UNITS>(FL.GetCalleeRegsStartSlot());
126
127 auto frameFlags = constantPool_(FrameConstantDescriptor::FRAME_FLAGS);
128 auto tlsFrameOffset = constantPool_(FrameConstantDescriptor::TLS_FRAME_OFFSET);
129
130 builder.CreateInlineAsm("push %rbp");
131 builder.CreateInlineAsm("movq %rsp, %rbp");
132 builder.CreateInlineAsm("lea ${0:c}(%rsp), %rsp", {-(FRAME_SIZE - SLOT_SIZE * 2U)});
133 builder.CreateInlineAsm("movq %rdi, ${0:c}(%rsp)", {METHOD_OFFSET});
134 if (!frameInfo_.usesFloatRegs) {
135 frameFlags = 0;
136 }
137 builder.CreateInlineAsm("movq $0, ${1:c}(%rsp)", {frameFlags, FLAGS_OFFSET});
138 builder.CreateInlineAsm("movb $$0x1, ${0:c}(%r15)", {tlsFrameOffset});
139 builder.CreateInlineAsm("movq %r15, ${0:c}(%rsp)", {CALLEE_OFFSET - 0 * SLOT_SIZE});
140 builder.CreateInlineAsm("movq %r14, ${0:c}(%rsp)", {CALLEE_OFFSET - 1 * SLOT_SIZE});
141 builder.CreateInlineAsm("movq %r13, ${0:c}(%rsp)", {CALLEE_OFFSET - 2 * SLOT_SIZE});
142 builder.CreateInlineAsm("movq %r12, ${0:c}(%rsp)", {CALLEE_OFFSET - 3 * SLOT_SIZE});
143 builder.CreateInlineAsm("movq %rbx, ${0:c}(%rsp)", {CALLEE_OFFSET - 4 * SLOT_SIZE});
144 // Ignore space allocated for rbp, we already save it
145 ssize_t spaceForSpills = frameInfo_.stackSize - SLOT_SIZE;
146 if (spaceForSpills != 0) {
147 builder.CreateInlineAsm("sub $0, %rsp", {spaceForSpills});
148 }
149 // StackOverflowCheck
150 builder.CreateInlineAsm("test %rdi, ${0:c}(%rsp)", {frameInfo_.soOffset});
151 }
152
InsertEpilogue(llvm::MachineBasicBlock & mblock)153 void AMD64FrameBuilder::InsertEpilogue(llvm::MachineBasicBlock &mblock)
154 {
155 auto [xregsMask, vregsMask] = frameInfo_.regMasks;
156 if (!frameInfo_.hasCalls && !frameInfo_.usesStack && xregsMask == 0 && vregsMask == 0) {
157 // Do not generate any code
158 return;
159 }
160
161 InlineAsmBuilder builder(&mblock, mblock.getFirstTerminator());
162
163 constexpr ark::CFrameLayout FL(ark::Arch::X86_64, 0);
164
165 constexpr auto SP_ORIGIN = ark::CFrameLayout::OffsetOrigin::SP;
166 constexpr auto BYTES_UNITS = ark::CFrameLayout::OffsetUnit::BYTES;
167
168 constexpr ssize_t SLOT_SIZE = ark::PointerSize(ark::Arch::X86_64);
169 constexpr ssize_t FRAME_SIZE = FL.GetFrameSize<BYTES_UNITS>();
170 constexpr ssize_t CALLEE_OFFSET = FL.GetOffset<SP_ORIGIN, BYTES_UNITS>(FL.GetCalleeRegsStartSlot());
171
172 builder.CreateInlineAsm("movq %rbp, %rsp");
173 builder.CreateInlineAsm("lea ${0:c}(%rsp), %rsp", {-(FRAME_SIZE - SLOT_SIZE * 2U)});
174 builder.CreateInlineAsm("movq ${0:c}(%rsp), %rbx", {CALLEE_OFFSET - 4 * SLOT_SIZE});
175 builder.CreateInlineAsm("movq ${0:c}(%rsp), %r12", {CALLEE_OFFSET - 3 * SLOT_SIZE});
176 builder.CreateInlineAsm("movq ${0:c}(%rsp), %r13", {CALLEE_OFFSET - 2 * SLOT_SIZE});
177 builder.CreateInlineAsm("movq ${0:c}(%rsp), %r14", {CALLEE_OFFSET - 1 * SLOT_SIZE});
178 builder.CreateInlineAsm("movq ${0:c}(%rsp), %r15", {CALLEE_OFFSET - 0 * SLOT_SIZE});
179 builder.CreateInlineAsm("movq %rbp, %rsp");
180 builder.CreateInlineAsm("pop %rbp");
181 }
182
183 // ARM64 Frame builder implementation
184
RemovePrologue(llvm::MachineBasicBlock & mblock)185 bool ARM64FrameBuilder::RemovePrologue(llvm::MachineBasicBlock &mblock)
186 {
187 auto predicate = [](llvm::MachineInstr &inst) -> bool {
188 bool isFrameSetup = inst.getFlag(llvm::MachineInstr::FrameSetup);
189 return isFrameSetup;
190 };
191 RemoveInstsIf(mblock, predicate);
192 return mblock.getPrevNode() == nullptr;
193 }
194
RemoveEpilogue(llvm::MachineBasicBlock & mblock)195 bool ARM64FrameBuilder::RemoveEpilogue(llvm::MachineBasicBlock &mblock)
196 {
197 bool isEpilogue = false;
198 auto predicate = [&isEpilogue](llvm::MachineInstr &inst) -> bool {
199 bool isFrameDestroy = inst.getFlag(llvm::MachineInstr::FrameDestroy);
200 // Sometimes llvm may generate code, when there is no any frame-destroy instructions
201 // that contain RET. And we still need to remember such basic blocks.
202 isEpilogue |= inst.isReturn();
203 return isFrameDestroy;
204 };
205 RemoveInstsIf(mblock, predicate);
206 return isEpilogue;
207 }
208
209 namespace {
210 namespace arm_frame_helpers {
211 constexpr ark::CFrameLayout FL(ark::Arch::AARCH64, 0);
212 constexpr ssize_t SLOT_SIZE = ark::PointerSize(ark::Arch::AARCH64);
213 constexpr ssize_t DSLOT_SIZE = SLOT_SIZE * 2U;
214 constexpr auto FP_ORIGIN = ark::CFrameLayout::OffsetOrigin::FP;
215 constexpr auto BYTES_UNITS = ark::CFrameLayout::OffsetUnit::BYTES;
216 constexpr ssize_t FLAGS_OFFSET = FL.GetOffset<FP_ORIGIN, BYTES_UNITS>(ark::CFrameLayout::FlagsSlot::Start());
217 constexpr ssize_t X_CALLEE_OFFSET = FL.GetOffset<FP_ORIGIN, BYTES_UNITS>(FL.GetCalleeRegsStartSlot()) - SLOT_SIZE;
218 constexpr ssize_t V_CALLEE_OFFSET = FL.GetOffset<FP_ORIGIN, BYTES_UNITS>(FL.GetCalleeFpRegsStartSlot()) - SLOT_SIZE;
219 constexpr auto INVALID_REGISTER = 255;
220 constexpr auto MAX_OFFSET = 256;
221
222 static_assert(FL.GetOffset<FP_ORIGIN, BYTES_UNITS>(FL.GetCallerRegsStartSlot()) <= MAX_OFFSET);
223 } // namespace arm_frame_helpers
224 } // namespace
225
InsertPrologue(llvm::MachineBasicBlock & mblock)226 void ARM64FrameBuilder::InsertPrologue(llvm::MachineBasicBlock &mblock)
227 {
228 InlineAsmBuilder builder(&mblock, mblock.begin());
229
230 auto [xregsMask, vregsMask] = frameInfo_.regMasks;
231
232 // Do not generate any code
233 if (!frameInfo_.hasCalls && !frameInfo_.usesStack && xregsMask == 0 && vregsMask == 0) {
234 return;
235 }
236
237 builder.CreateInlineAsm("stp x29, x30, [sp, #-16]!");
238 builder.CreateInlineAsm("mov x29, sp");
239
240 if (frameInfo_.usesFloatRegs) {
241 auto frameFlags = constantPool_(FrameConstantDescriptor::FRAME_FLAGS);
242 builder.CreateInlineAsm("mov x30, $0", {frameFlags});
243 builder.CreateInlineAsm("stp x30, x0, [fp, ${0:n}]", {arm_frame_helpers::FLAGS_OFFSET});
244 } else {
245 builder.CreateInlineAsm("stp xzr, x0, [fp, ${0:n}]", {arm_frame_helpers::FLAGS_OFFSET});
246 }
247
248 if (frameInfo_.hasCalls) {
249 auto tlsFrameOffset = constantPool_(FrameConstantDescriptor::TLS_FRAME_OFFSET);
250 builder.CreateInlineAsm("mov x30, #1");
251 builder.CreateInlineAsm("str w30, [x28, ${0:c}]", {tlsFrameOffset});
252 }
253
254 if (xregsMask != 0) {
255 EmitCSRSaveRestoreCode(&builder, xregsMask, "stur x$0, [sp, -$1]", "stp x$0, x$1, [sp, -$2]",
256 arm_frame_helpers::X_CALLEE_OFFSET);
257 }
258
259 if (vregsMask != 0) {
260 EmitCSRSaveRestoreCode(&builder, vregsMask, "stur d$0, [sp, -$1]", "stp d$0, d$1, [sp, -$2]",
261 arm_frame_helpers::V_CALLEE_OFFSET);
262 }
263
264 constexpr uint32_t MAX_IMM_12 = 4096;
265 builder.CreateInlineAsm("sub sp, fp, ${0:c}", {frameInfo_.stackSize % MAX_IMM_12});
266 ASSERT(frameInfo_.stackSize / MAX_IMM_12 < MAX_IMM_12);
267 if (frameInfo_.stackSize / MAX_IMM_12 > 0) {
268 builder.CreateInlineAsm("sub sp, sp, ${0:c}", {MAX_IMM_12 * (frameInfo_.stackSize / MAX_IMM_12)});
269 }
270
271 // StackOverflow check
272 builder.CreateInlineAsm("add x30, sp, ${0:c}", {frameInfo_.soOffset});
273 builder.CreateInlineAsm("ldr x30, [x30]");
274 }
275
InsertEpilogue(llvm::MachineBasicBlock & mblock)276 void ARM64FrameBuilder::InsertEpilogue(llvm::MachineBasicBlock &mblock)
277 {
278 InlineAsmBuilder builder(&mblock, mblock.getFirstTerminator());
279
280 auto [xregsMask, vregsMask] = frameInfo_.regMasks;
281 if (!frameInfo_.hasCalls && !frameInfo_.usesStack && xregsMask == 0 && vregsMask == 0) {
282 // Do not generate any code
283 return;
284 }
285
286 builder.CreateInlineAsm("mov sp, fp");
287 if (xregsMask != 0) {
288 EmitCSRSaveRestoreCode(&builder, xregsMask, "ldur x$0, [sp, -$1]", "ldp x$0, x$1, [sp, -$2]",
289 arm_frame_helpers::X_CALLEE_OFFSET);
290 }
291 if (vregsMask != 0) {
292 EmitCSRSaveRestoreCode(&builder, vregsMask, "ldur d$0, [sp, -$1]", "ldp d$0, d$1, [sp, -$2]",
293 arm_frame_helpers::V_CALLEE_OFFSET);
294 }
295
296 builder.CreateInlineAsm("ldp x29, x30, [sp], #16");
297 }
298
EmitCSRSaveRestoreCode(InlineAsmBuilder * builder,uint32_t regsMask,std::string_view asmSingleReg,std::string_view asmPairRegs,ssize_t calleeOffset)299 void ARM64FrameBuilder::EmitCSRSaveRestoreCode(InlineAsmBuilder *builder, uint32_t regsMask,
300 std::string_view asmSingleReg, std::string_view asmPairRegs,
301 ssize_t calleeOffset)
302 {
303 std::vector<int> regs;
304 unsigned i = sizeof(regsMask) * 8U;
305 do {
306 i--;
307 if ((regsMask & (1U << i)) != 0) {
308 regs.push_back(i);
309 }
310 } while (i != 0);
311
312 if (regs.size() % 2U == 1) {
313 regs.push_back(arm_frame_helpers::INVALID_REGISTER);
314 }
315
316 auto off = calleeOffset;
317 for (i = 0; i < regs.size(); i += 2U) {
318 auto [reg_a, reg_b] = std::minmax(regs[i + 0U], regs[i + 1U]);
319 if (reg_b != arm_frame_helpers::INVALID_REGISTER) {
320 off += arm_frame_helpers::DSLOT_SIZE;
321 builder->CreateInlineAsm(asmPairRegs, {reg_a, reg_b, off});
322 } else {
323 off += arm_frame_helpers::SLOT_SIZE;
324 builder->CreateInlineAsm(asmSingleReg, {reg_a, off});
325 }
326 }
327 }
328