1 /**
2 * Copyright (c) 2023-2025 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #include "compiler/optimizer/code_generator/operands.h"
16 #include "compiler/optimizer/code_generator/codegen.h"
17 #include "compiler/optimizer/ir/analysis.h"
18 #include "runtime/include/coretypes/string.h"
19 #include "runtime/include/coretypes/array.h"
20
21 namespace ark::compiler {
22
23 class SbAppendArgs {
24 private:
25 Reg dst_;
26 Reg builder_;
27 Reg value_;
28
29 public:
30 SbAppendArgs() = delete;
SbAppendArgs(Reg dst,Reg builder,Reg value)31 SbAppendArgs(Reg dst, Reg builder, Reg value) : dst_(dst), builder_(builder), value_(value)
32 {
33 ASSERT(dst_ != INVALID_REGISTER);
34 ASSERT(builder_ != INVALID_REGISTER);
35 ASSERT(value_ != INVALID_REGISTER);
36 }
Dst() const37 Reg Dst() const
38 {
39 return dst_;
40 }
Builder() const41 Reg Builder() const
42 {
43 return builder_;
44 }
Value() const45 Reg Value() const
46 {
47 return value_;
48 }
DstCanBeUsedAsTemp() const49 bool DstCanBeUsedAsTemp() const
50 {
51 return (dst_.GetId() != builder_.GetId() && dst_.GetId() != value_.GetId());
52 }
SbBufferAddr() const53 MemRef SbBufferAddr() const
54 {
55 return MemRef(builder_, RuntimeInterface::GetSbBufferOffset());
56 }
SbIndexAddr() const57 MemRef SbIndexAddr() const
58 {
59 return MemRef(builder_, RuntimeInterface::GetSbIndexOffset());
60 }
SbCompressAddr() const61 MemRef SbCompressAddr() const
62 {
63 return MemRef(builder_, RuntimeInterface::GetSbCompressOffset());
64 }
SbLengthAddr() const65 MemRef SbLengthAddr() const
66 {
67 return MemRef(builder_, RuntimeInterface::GetSbLengthOffset());
68 }
69 };
70
CreateMathTrunc(IntrinsicInst * inst,Reg dst,SRCREGS src)71 void Codegen::CreateMathTrunc([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
72 {
73 GetEncoder()->EncodeTrunc(dst, src[0]);
74 }
75
CreateETSMathRound(IntrinsicInst * inst,Reg dst,SRCREGS src)76 void Codegen::CreateETSMathRound([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
77 {
78 GetEncoder()->EncodeRoundToPInfReturnFloat(dst, src[0]);
79 }
80
CreateArrayCopyTo(IntrinsicInst * inst,Reg dst,SRCREGS src)81 void Codegen::CreateArrayCopyTo(IntrinsicInst *inst, [[maybe_unused]] Reg dst, SRCREGS src)
82 {
83 auto entrypointId = EntrypointId::INVALID;
84
85 switch (inst->GetIntrinsicId()) {
86 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_BOOL_COPY_TO:
87 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_BYTE_COPY_TO:
88 entrypointId = EntrypointId::ARRAY_COPY_TO_1B;
89 break;
90
91 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_CHAR_COPY_TO:
92 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_SHORT_COPY_TO:
93 entrypointId = EntrypointId::ARRAY_COPY_TO_2B;
94 break;
95
96 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_INT_COPY_TO:
97 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_FLOAT_COPY_TO:
98 entrypointId = EntrypointId::ARRAY_COPY_TO_4B;
99 break;
100
101 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_LONG_COPY_TO:
102 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_DOUBLE_COPY_TO:
103 entrypointId = EntrypointId::ARRAY_COPY_TO_8B;
104 break;
105
106 default:
107 UNREACHABLE();
108 break;
109 }
110
111 ASSERT(entrypointId != EntrypointId::COUNT);
112
113 auto srcObj = src[FIRST_OPERAND];
114 auto dstObj = src[SECOND_OPERAND];
115 auto dstStart = src[THIRD_OPERAND];
116 auto srcStart = src[FOURTH_OPERAND];
117 auto srcEnd = src[FIFTH_OPERAND];
118 CallFastPath(inst, entrypointId, INVALID_REGISTER, RegMask::GetZeroMask(), srcObj, dstObj, dstStart, srcStart,
119 srcEnd);
120 }
121
122 // Generates a call to StringBuilder.append() for values (EtsBool/Char/Bool/Short/Int/Long),
123 // which are translated to array of utf16 chars.
GenerateSbAppendCall(Codegen * cg,IntrinsicInst * inst,SbAppendArgs args,RuntimeInterface::EntrypointId entrypoint)124 static inline void GenerateSbAppendCall(Codegen *cg, IntrinsicInst *inst, SbAppendArgs args,
125 RuntimeInterface::EntrypointId entrypoint)
126 {
127 auto *runtime = cg->GetGraph()->GetRuntime();
128 if (cg->GetGraph()->IsAotMode()) {
129 auto *enc = cg->GetEncoder();
130 ScopedTmpReg klass(enc);
131 enc->EncodeLdr(klass, false, MemRef(cg->ThreadReg(), runtime->GetArrayU16ClassPointerTlsOffset(cg->GetArch())));
132 cg->CallFastPath(inst, entrypoint, args.Dst(), {}, args.Builder(), args.Value(), klass);
133 } else {
134 auto klass = TypedImm(reinterpret_cast<uintptr_t>(runtime->GetArrayU16Class(cg->GetGraph()->GetMethod())));
135 cg->CallFastPath(inst, entrypoint, args.Dst(), {}, args.Builder(), args.Value(), klass);
136 }
137 }
138
CreateStringBuilderAppendNumber(IntrinsicInst * inst,Reg dst,SRCREGS src)139 void Codegen::CreateStringBuilderAppendNumber(IntrinsicInst *inst, Reg dst, SRCREGS src)
140 {
141 auto sb = src[FIRST_OPERAND];
142 auto num = src[SECOND_OPERAND];
143 auto type = ConvertDataType(DataType::INT64, GetArch());
144 ScopedTmpReg tmp(GetEncoder(), type);
145
146 if (num.GetType() != INT64_TYPE) {
147 ASSERT(num.GetType() == INT32_TYPE || num.GetType() == INT16_TYPE || num.GetType() == INT8_TYPE);
148 if (dst.GetId() != sb.GetId() && dst.GetId() != num.GetId()) {
149 GetEncoder()->EncodeCast(dst.As(type), true, num, true);
150 num = dst.As(type);
151 } else {
152 GetEncoder()->EncodeCast(tmp, true, num, true);
153 num = tmp.GetReg();
154 }
155 }
156 GenerateSbAppendCall(this, inst, SbAppendArgs(dst, sb, num), EntrypointId::STRING_BUILDER_APPEND_LONG);
157 }
158
CreateStringBuilderAppendChar(IntrinsicInst * inst,Reg dst,SRCREGS src)159 void Codegen::CreateStringBuilderAppendChar(IntrinsicInst *inst, Reg dst, SRCREGS src)
160 {
161 ASSERT(IsCompressedStringsEnabled());
162 auto entrypoint = EntrypointId::STRING_BUILDER_APPEND_CHAR_COMPRESSED;
163 SbAppendArgs args(dst, src[FIRST_OPERAND], src[SECOND_OPERAND]);
164 GenerateSbAppendCall(this, inst, args, entrypoint);
165 }
166
CreateStringBuilderAppendBool(IntrinsicInst * inst,Reg dst,SRCREGS src)167 void Codegen::CreateStringBuilderAppendBool(IntrinsicInst *inst, Reg dst, SRCREGS src)
168 {
169 SbAppendArgs args(dst, src[FIRST_OPERAND], src[SECOND_OPERAND]);
170 GenerateSbAppendCall(this, inst, args, EntrypointId::STRING_BUILDER_APPEND_BOOL);
171 }
172
EncodeSbAppendNullString(Codegen * cg,IntrinsicInst * inst,Reg dst,Reg builder)173 static inline void EncodeSbAppendNullString(Codegen *cg, IntrinsicInst *inst, Reg dst, Reg builder)
174 {
175 auto entrypoint = RuntimeInterface::EntrypointId::STRING_BUILDER_APPEND_NULL_STRING;
176 cg->CallRuntime(inst, entrypoint, dst, {}, builder);
177 }
178
EncodeSbInsertStringIntoSlot(Codegen * cg,IntrinsicInst * inst,Reg slot,SbAppendArgs args)179 static inline void EncodeSbInsertStringIntoSlot(Codegen *cg, IntrinsicInst *inst, Reg slot, SbAppendArgs args)
180 {
181 ASSERT(slot.IsValid());
182 auto slotMemRef = MemRef(slot.As(Codegen::ConvertDataType(DataType::REFERENCE, cg->GetArch())));
183 RegMask preserved(MakeMask(args.Builder().GetId(), args.Value().GetId(), slot.GetId()));
184 cg->CreatePreWRB(inst, slotMemRef, preserved);
185 cg->GetEncoder()->EncodeStr(args.Value(), slotMemRef);
186 preserved.Reset(slot.GetId());
187 cg->CreatePostWRB(inst, slotMemRef, args.Value(), INVALID_REGISTER, preserved);
188 }
189
EncodeSbAppendString(Codegen * cg,IntrinsicInst * inst,const SbAppendArgs & args,LabelHolder::LabelId labelReturn,LabelHolder::LabelId labelSlowPath)190 static void EncodeSbAppendString(Codegen *cg, IntrinsicInst *inst, const SbAppendArgs &args,
191 LabelHolder::LabelId labelReturn, LabelHolder::LabelId labelSlowPath)
192 {
193 auto *enc = cg->GetEncoder();
194 ScopedTmpReg tmp1(enc);
195 ScopedTmpRegLazy tmp2(enc, false);
196 auto reg0 = cg->ConvertInstTmpReg(inst, DataType::REFERENCE);
197 auto reg1 = tmp1.GetReg().As(INT32_TYPE);
198 auto reg2 = INVALID_REGISTER;
199 if (args.DstCanBeUsedAsTemp() && args.Dst().GetId() != reg0.GetId()) {
200 reg2 = args.Dst().As(INT32_TYPE);
201 } else {
202 tmp2.Acquire();
203 reg2 = tmp2.GetReg().As(INT32_TYPE);
204 }
205 auto labelInsertStringIntoSlot = enc->CreateLabel();
206 auto labelFastPathDone = enc->CreateLabel();
207 auto labelIncIndex = enc->CreateLabel();
208 // Jump to slowPath if buffer is full and needs to be reallocated
209 enc->EncodeLdr(reg0, false, args.SbBufferAddr());
210 enc->EncodeLdr(reg1, false, MemRef(reg0, coretypes::Array::GetLengthOffset()));
211 enc->EncodeLdr(reg2, false, args.SbIndexAddr());
212 enc->EncodeJump(labelSlowPath, reg2, reg1, Condition::HS);
213 // Compute an address of a free slot so as not to reload SbIndex again
214 enc->EncodeShl(reg1, reg2, Imm(compiler::DataType::ShiftByType(compiler::DataType::REFERENCE, cg->GetArch())));
215 enc->EncodeAdd(reg0, reg0, Imm(coretypes::Array::GetDataOffset()));
216 enc->EncodeAdd(reg0, reg0, reg1);
217 // Process string length and compression
218 enc->EncodeLdr(reg1, false, MemRef(args.Value(), ark::coretypes::STRING_LENGTH_OFFSET));
219 // Do nothing if length of string is equal to 0.
220 // The least significant bit indicates COMPRESSED/UNCOMPRESSED,
221 // thus if (packed length <= 1) then the actual length is equal to 0.
222 enc->EncodeJump(labelFastPathDone, reg1, Imm(1), Condition::LS);
223 // Skip setting 'compress' to false if the string is compressed.
224 enc->EncodeJumpTest(labelIncIndex, reg1, Imm(1), Condition::TST_EQ);
225 // Otherwise set 'compress' to false
226 enc->EncodeSti(0, 1, args.SbCompressAddr());
227 // Increment 'index' field
228 enc->BindLabel(labelIncIndex);
229 enc->EncodeAdd(reg2, reg2, Imm(1));
230 enc->EncodeStr(reg2, args.SbIndexAddr());
231 // Unpack length of string
232 enc->EncodeShr(reg1, reg1, Imm(1));
233 // Add length of string to the current length of StringBuilder
234 enc->EncodeLdr(reg2, false, args.SbLengthAddr());
235 enc->EncodeAdd(reg2, reg2, reg1);
236 enc->EncodeStr(reg2, args.SbLengthAddr());
237 // Insert the string into the slot:
238 // - reg0 contains an address of the slot
239 // - release temps for barriers
240 enc->BindLabel(labelInsertStringIntoSlot);
241 tmp1.Release();
242 tmp2.Release();
243 EncodeSbInsertStringIntoSlot(cg, inst, reg0, args);
244 // Return the reference to StringBuilder
245 enc->BindLabel(labelFastPathDone);
246 enc->EncodeMov(args.Dst(), args.Builder());
247 enc->EncodeJump(labelReturn);
248 }
249
CreateStringBuilderAppendString(IntrinsicInst * inst,Reg dst,SRCREGS src)250 void Codegen::CreateStringBuilderAppendString(IntrinsicInst *inst, Reg dst, SRCREGS src)
251 {
252 using StringLengthType = std::result_of<decltype (&coretypes::String::GetLength)(coretypes::String)>::type;
253 static_assert(TypeInfo::GetScalarTypeBySize(sizeof(ark::ArraySizeT) * CHAR_BIT) == INT32_TYPE);
254 static_assert(TypeInfo::GetScalarTypeBySize(sizeof(StringLengthType) * CHAR_BIT) == INT32_TYPE);
255 ASSERT(GetArch() != Arch::AARCH32);
256 ASSERT(IsCompressedStringsEnabled());
257
258 auto *enc = GetEncoder();
259 auto builder = src[FIRST_OPERAND];
260 auto *strInst = inst->GetInput(1).GetInst();
261 if (strInst->IsNullPtr()) {
262 EncodeSbAppendNullString(this, inst, dst, builder);
263 return;
264 }
265 auto labelReturn = enc->CreateLabel();
266 auto labelSlowPath = enc->CreateLabel();
267 auto str = src[SECOND_OPERAND];
268 if (IsInstNotNull(strInst)) {
269 EncodeSbAppendString(this, inst, SbAppendArgs(dst, builder, str), labelReturn, labelSlowPath);
270 } else {
271 auto labelStrNotNull = enc->CreateLabel();
272 enc->EncodeJump(labelStrNotNull, str, Condition::NE);
273 EncodeSbAppendNullString(this, inst, dst, builder);
274 enc->EncodeJump(labelReturn);
275 enc->BindLabel(labelStrNotNull);
276 EncodeSbAppendString(this, inst, SbAppendArgs(dst, builder, str), labelReturn, labelSlowPath);
277 }
278 // Slow path
279 static constexpr auto ENTRYPOINT_ID = RuntimeInterface::EntrypointId::STRING_BUILDER_APPEND_STRING;
280 enc->BindLabel(labelSlowPath);
281 CallRuntime(inst, ENTRYPOINT_ID, dst, {}, builder, str);
282 // Return
283 enc->BindLabel(labelReturn);
284 }
285
GetStringBuilderAppendStringsEntrypointId(uint32_t numArgs,mem::BarrierType barrierType)286 RuntimeInterface::EntrypointId GetStringBuilderAppendStringsEntrypointId(uint32_t numArgs, mem::BarrierType barrierType)
287 {
288 using EntrypointId = RuntimeInterface::EntrypointId;
289 switch (barrierType) {
290 case mem::BarrierType::POST_INTERGENERATIONAL_BARRIER: { // Gen GC
291 std::array<EntrypointId, 5U> entrypoints {
292 EntrypointId::INVALID, // numArgs = 0
293 EntrypointId::INVALID, // numArgs = 1
294 EntrypointId::STRING_BUILDER_APPEND_STRING2_ASYNC_MANUAL,
295 EntrypointId::STRING_BUILDER_APPEND_STRING3_ASYNC_MANUAL,
296 EntrypointId::STRING_BUILDER_APPEND_STRING4_ASYNC_MANUAL,
297 };
298 return entrypoints[numArgs];
299 }
300 case mem::BarrierType::POST_INTERREGION_BARRIER: { // G1 GC
301 std::array<EntrypointId, 5U> entrypoints {
302 EntrypointId::INVALID, // numArgs = 0
303 EntrypointId::INVALID, // numArgs = 1
304 EntrypointId::STRING_BUILDER_APPEND_STRING2_ASYNC,
305 EntrypointId::STRING_BUILDER_APPEND_STRING3_ASYNC,
306 EntrypointId::STRING_BUILDER_APPEND_STRING4_ASYNC,
307 };
308 return entrypoints[numArgs];
309 }
310 default: { // STW GC
311 std::array<EntrypointId, 5U> entrypoints {
312 EntrypointId::INVALID, // numArgs = 0
313 EntrypointId::INVALID, // numArgs = 1
314 EntrypointId::STRING_BUILDER_APPEND_STRING2_SYNC,
315 EntrypointId::STRING_BUILDER_APPEND_STRING3_SYNC,
316 EntrypointId::STRING_BUILDER_APPEND_STRING4_SYNC,
317 };
318 return entrypoints[numArgs];
319 }
320 }
321 }
322
CreateStringBuilderAppendStrings(IntrinsicInst * inst,Reg dst,SRCREGS src)323 void Codegen::CreateStringBuilderAppendStrings(IntrinsicInst *inst, Reg dst, SRCREGS src)
324 {
325 ASSERT(IsCompressedStringsEnabled());
326 auto builder = src[FIRST_OPERAND];
327 auto str0 = src[SECOND_OPERAND];
328 auto str1 = src[THIRD_OPERAND];
329 switch (inst->GetIntrinsicId()) {
330 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_SB_APPEND_STRING2: {
331 auto entrypoint = GetStringBuilderAppendStringsEntrypointId(2U, GetGraph()->GetRuntime()->GetPostType());
332 CallFastPath(inst, entrypoint, dst, {}, builder, str0, str1);
333 break;
334 }
335
336 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_SB_APPEND_STRING3: {
337 auto str2 = src[FOURTH_OPERAND];
338 auto entrypoint = GetStringBuilderAppendStringsEntrypointId(3U, GetGraph()->GetRuntime()->GetPostType());
339 CallFastPath(inst, entrypoint, dst, {}, builder, str0, str1, str2);
340 break;
341 }
342
343 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_SB_APPEND_STRING4: {
344 auto str2 = src[FOURTH_OPERAND];
345 auto str3 = src[FIFTH_OPERAND];
346 auto entrypoint = GetStringBuilderAppendStringsEntrypointId(4U, GetGraph()->GetRuntime()->GetPostType());
347 CallFastPath(inst, entrypoint, dst, {}, builder, str0, str1, str2, str3);
348 break;
349 }
350
351 default:
352 UNREACHABLE();
353 break;
354 }
355 }
356
CreateStringConcat(IntrinsicInst * inst,Reg dst,SRCREGS src)357 void Codegen::CreateStringConcat([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
358 {
359 ASSERT(IsCompressedStringsEnabled());
360 switch (inst->GetIntrinsicId()) {
361 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_STRING_CONCAT2: {
362 auto str1 = src[FIRST_OPERAND];
363 auto str2 = src[SECOND_OPERAND];
364 CallFastPath(inst, EntrypointId::STRING_CONCAT2_TLAB, dst, {}, str1, str2);
365 break;
366 }
367
368 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_STRING_CONCAT3: {
369 auto str1 = src[FIRST_OPERAND];
370 auto str2 = src[SECOND_OPERAND];
371 auto str3 = src[THIRD_OPERAND];
372 CallFastPath(inst, EntrypointId::STRING_CONCAT3_TLAB, dst, {}, str1, str2, str3);
373 break;
374 }
375
376 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_STRING_CONCAT4: {
377 auto str1 = src[FIRST_OPERAND];
378 auto str2 = src[SECOND_OPERAND];
379 auto str3 = src[THIRD_OPERAND];
380 auto str4 = src[FOURTH_OPERAND];
381 CallFastPath(inst, EntrypointId::STRING_CONCAT4_TLAB, dst, {}, str1, str2, str3, str4);
382 break;
383 }
384
385 default:
386 UNREACHABLE();
387 break;
388 }
389 }
390
CreateStringBuilderToString(IntrinsicInst * inst,Reg dst,SRCREGS src)391 void Codegen::CreateStringBuilderToString(IntrinsicInst *inst, Reg dst, SRCREGS src)
392 {
393 ASSERT(GetArch() != Arch::AARCH32);
394 ASSERT(IsCompressedStringsEnabled());
395
396 auto *enc = GetEncoder();
397 auto entrypoint = EntrypointId::STRING_BUILDER_TO_STRING;
398 auto sb = src[FIRST_OPERAND];
399 if (GetGraph()->IsAotMode()) {
400 ScopedTmpReg klass(enc);
401 enc->EncodeLdr(klass, false, MemRef(ThreadReg(), GetRuntime()->GetStringClassPointerTlsOffset(GetArch())));
402 CallFastPath(inst, entrypoint, dst, {}, sb, klass);
403 } else {
404 auto klass =
405 TypedImm(reinterpret_cast<uintptr_t>(GetRuntime()->GetStringClass(GetGraph()->GetMethod(), nullptr)));
406 CallFastPath(inst, entrypoint, dst, {}, sb, klass);
407 }
408 }
409
CreateDoubleToStringDecimal(IntrinsicInst * inst,Reg dst,SRCREGS src)410 void Codegen::CreateDoubleToStringDecimal(IntrinsicInst *inst, Reg dst, SRCREGS src)
411 {
412 ASSERT(GetArch() != Arch::AARCH32);
413 ASSERT(inst->GetInputsCount() == 4U && inst->RequireState());
414 auto cache = src[FIRST_OPERAND];
415 auto numAsInt = src[SECOND_OPERAND];
416 auto unused = src[THIRD_OPERAND];
417 auto entrypoint = EntrypointId::DOUBLE_TO_STRING_DECIMAL;
418 CallFastPath(inst, entrypoint, dst, {}, cache, numAsInt, unused);
419 }
420
CreateFloatIsInteger(IntrinsicInst * inst,Reg dst,SRCREGS src)421 void Codegen::CreateFloatIsInteger([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
422 {
423 GetEncoder()->EncodeIsInteger(dst, src[0]);
424 }
425
CreateFloatIsSafeInteger(IntrinsicInst * inst,Reg dst,SRCREGS src)426 void Codegen::CreateFloatIsSafeInteger([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
427 {
428 GetEncoder()->EncodeIsSafeInteger(dst, src[0]);
429 }
430
431 /* See utf::IsWhiteSpaceChar() for the details */
CreateCharIsWhiteSpace(IntrinsicInst * inst,Reg dst,SRCREGS src)432 void Codegen::CreateCharIsWhiteSpace([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
433 {
434 ASSERT(GetArch() != Arch::AARCH32);
435 auto entrypoint = RuntimeInterface::EntrypointId::CHAR_IS_WHITE_SPACE;
436 auto ch = src[FIRST_OPERAND];
437 CallFastPath(inst, entrypoint, dst, {}, ch.As(INT16_TYPE));
438 }
439
CreateStringTrimLeft(IntrinsicInst * inst,Reg dst,SRCREGS src)440 void Codegen::CreateStringTrimLeft(IntrinsicInst *inst, Reg dst, SRCREGS src)
441 {
442 ASSERT(IsCompressedStringsEnabled());
443 auto str = src[FIRST_OPERAND];
444 auto unused = TypedImm(0);
445 // llvm backend needs unused args to call 3-args slow_path from 1-arg fast_path.
446 CallFastPath(inst, RuntimeInterface::EntrypointId::STRING_TRIM_LEFT, dst, {}, str, unused, unused);
447 }
448
CreateStringTrimRight(IntrinsicInst * inst,Reg dst,SRCREGS src)449 void Codegen::CreateStringTrimRight(IntrinsicInst *inst, Reg dst, SRCREGS src)
450 {
451 ASSERT(IsCompressedStringsEnabled());
452 auto str = src[FIRST_OPERAND];
453 auto unused = TypedImm(0);
454 // llvm backend needs unused args to call 3-args slow_path from 1-arg fast_path.
455 CallFastPath(inst, RuntimeInterface::EntrypointId::STRING_TRIM_RIGHT, dst, {}, str, unused, unused);
456 }
457
CreateStringTrim(IntrinsicInst * inst,Reg dst,SRCREGS src)458 void Codegen::CreateStringTrim(IntrinsicInst *inst, Reg dst, SRCREGS src)
459 {
460 ASSERT(IsCompressedStringsEnabled());
461 auto str = src[FIRST_OPERAND];
462 auto unused = TypedImm(0);
463 // llvm backend needs unused args to call 3-args slow_path from 1-arg fast_path.
464 CallFastPath(inst, RuntimeInterface::EntrypointId::STRING_TRIM, dst, {}, str, unused, unused);
465 }
466
CreateStringStartsWith(IntrinsicInst * inst,Reg dst,SRCREGS src)467 void Codegen::CreateStringStartsWith(IntrinsicInst *inst, Reg dst, SRCREGS src)
468 {
469 ASSERT(IsCompressedStringsEnabled());
470 auto str = src[FIRST_OPERAND];
471 auto pfx = src[SECOND_OPERAND];
472 auto idx = src[THIRD_OPERAND];
473 CallFastPath(inst, RuntimeInterface::EntrypointId::STRING_STARTS_WITH, dst, {}, str, pfx, idx);
474 }
475
CreateStringEndsWith(IntrinsicInst * inst,Reg dst,SRCREGS src)476 void Codegen::CreateStringEndsWith(IntrinsicInst *inst, Reg dst, SRCREGS src)
477 {
478 ASSERT(IsCompressedStringsEnabled());
479 auto str = src[FIRST_OPERAND];
480 auto sfx = src[SECOND_OPERAND];
481 auto idx = src[THIRD_OPERAND];
482 CallFastPath(inst, RuntimeInterface::EntrypointId::STRING_ENDS_WITH, dst, {}, str, sfx, idx);
483 }
484
CreateStringGetBytesTlab(IntrinsicInst * inst,Reg dst,SRCREGS src)485 void Codegen::CreateStringGetBytesTlab([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
486 {
487 ASSERT(IsCompressedStringsEnabled());
488 auto entrypointId = EntrypointId::STRING_GET_BYTES_TLAB;
489 auto runtime = GetGraph()->GetRuntime();
490 if (GetGraph()->IsAotMode()) {
491 ScopedTmpReg klassReg(GetEncoder());
492 GetEncoder()->EncodeLdr(klassReg, false,
493 MemRef(ThreadReg(), runtime->GetArrayU8ClassPointerTlsOffset(GetArch())));
494 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
495 klassReg);
496 } else {
497 auto klassImm = TypedImm(reinterpret_cast<uintptr_t>(runtime->GetArrayU8Class(GetGraph()->GetMethod())));
498 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
499 klassImm);
500 }
501 }
502
CreateStringIndexOf(IntrinsicInst * inst,Reg dst,SRCREGS src)503 void Codegen::CreateStringIndexOf(IntrinsicInst *inst, Reg dst, SRCREGS src)
504 {
505 ASSERT(IsCompressedStringsEnabled());
506 auto str = src[FIRST_OPERAND];
507 auto ch = src[SECOND_OPERAND];
508 CallFastPath(inst, RuntimeInterface::EntrypointId::STRING_INDEX_OF, dst, {}, str, ch, GetRegfile()->GetZeroReg());
509 }
510
CreateStringIndexOfAfter(IntrinsicInst * inst,Reg dst,SRCREGS src)511 void Codegen::CreateStringIndexOfAfter(IntrinsicInst *inst, Reg dst, SRCREGS src)
512 {
513 ASSERT(IsCompressedStringsEnabled());
514 ASSERT(IntrinsicNeedsParamLocations(inst->GetIntrinsicId()));
515 auto *enc = GetEncoder();
516 ScopedTmpReg tmpReg(enc);
517 auto str = src[FIRST_OPERAND];
518 auto ch = src[SECOND_OPERAND];
519 auto startIndex = src[THIRD_OPERAND];
520 auto tmp = tmpReg.GetReg().As(startIndex.GetType());
521 auto zreg = GetRegfile()->GetZeroReg();
522 // If 'startIndex' is negative then make it zero.
523 // We must not overwrite 'startIndex' register so use a temp register instead.
524 enc->EncodeSelect({tmp, startIndex, zreg, startIndex, Imm(0), Condition::GE});
525 RegMask preserved(MakeMask(startIndex.GetId()));
526 CallFastPath(inst, RuntimeInterface::EntrypointId::STRING_INDEX_OF_AFTER, dst, {preserved}, str, ch, tmp);
527 // Irtoc implementation of INDEX_OF_AFTER calls the corresponding INDEX_OF_ variant.
528 // That is done for the optimization purpose.
529 // So here we must add 'startIndex' to the 'dst' (if it's not equal to '-1')
530 auto charNotFoundLabel = enc->CreateLabel();
531 enc->EncodeJump(charNotFoundLabel, dst, Imm(-1), Condition::EQ);
532 enc->EncodeSelect({tmp, startIndex, zreg, startIndex, Imm(0), Condition::GE});
533 enc->EncodeAdd(dst, dst, tmp);
534 enc->BindLabel(charNotFoundLabel);
535 }
536
CreateStringFromCharCode(IntrinsicInst * inst,Reg dst,SRCREGS src)537 void Codegen::CreateStringFromCharCode(IntrinsicInst *inst, Reg dst, SRCREGS src)
538 {
539 ASSERT(GetArch() != Arch::AARCH32);
540 ASSERT(inst->GetInputsCount() == 2U && inst->RequireState());
541 auto array = src[FIRST_OPERAND];
542 auto getEntryId = [this, inst]() {
543 switch (inst->GetIntrinsicId()) {
544 case RuntimeInterface::IntrinsicId::INTRINSIC_STD_CORE_STRING_FROM_CHAR_CODE:
545 return GetRuntime()->IsCompressedStringsEnabled()
546 ? EntrypointId::CREATE_STRING_FROM_CHAR_CODE_TLAB_COMPRESSED
547 : EntrypointId::CREATE_STRING_FROM_CHAR_CODE_TLAB;
548 case RuntimeInterface::IntrinsicId::INTRINSIC_COMPILER_ETS_STRING_FROM_CHAR_CODE_SINGLE:
549 return GetRuntime()->IsCompressedStringsEnabled()
550 ? EntrypointId::CREATE_STRING_FROM_CHAR_CODE_SINGLE_TLAB_COMPRESSED
551 : EntrypointId::CREATE_STRING_FROM_CHAR_CODE_SINGLE_TLAB;
552 default:
553 UNREACHABLE();
554 }
555 };
556 if (GetGraph()->IsAotMode()) {
557 ScopedTmpReg klassReg(GetEncoder());
558 GetEncoder()->EncodeLdr(
559 klassReg, false,
560 MemRef(ThreadReg(), static_cast<ssize_t>(GetRuntime()->GetStringClassPointerTlsOffset(GetArch()))));
561 CallFastPath(inst, getEntryId(), dst, RegMask::GetZeroMask(), array, klassReg);
562 } else {
563 auto klassImm =
564 TypedImm(reinterpret_cast<uintptr_t>(GetRuntime()->GetStringClass(GetGraph()->GetMethod(), nullptr)));
565 CallFastPath(inst, getEntryId(), dst, RegMask::GetZeroMask(), array, klassImm);
566 }
567 }
568
CreateStringRepeat(IntrinsicInst * inst,Reg dst,SRCREGS src)569 void Codegen::CreateStringRepeat([[maybe_unused]] IntrinsicInst *inst, Reg dst, SRCREGS src)
570 {
571 ASSERT(IsCompressedStringsEnabled());
572 auto entrypointId = EntrypointId::STRING_REPEAT;
573 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND]);
574 }
575
CreateInt8ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)576 void Codegen::CreateInt8ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
577 {
578 ASSERT(GetArch() != Arch::AARCH32);
579 auto entrypoint = EntrypointId::INT8_ARRAY_FILL_INTERNAL_FAST_PATH;
580 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
581 src[FOURTH_OPERAND]);
582 }
583
CreateInt16ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)584 void Codegen::CreateInt16ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
585 {
586 ASSERT(GetArch() != Arch::AARCH32);
587 auto entrypoint = EntrypointId::INT16_ARRAY_FILL_INTERNAL_FAST_PATH;
588 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
589 src[FOURTH_OPERAND]);
590 }
591
CreateInt32ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)592 void Codegen::CreateInt32ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
593 {
594 ASSERT(GetArch() != Arch::AARCH32);
595 auto entrypoint = EntrypointId::INT32_ARRAY_FILL_INTERNAL_FAST_PATH;
596 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
597 src[FOURTH_OPERAND]);
598 }
599
CreateBigInt64ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)600 void Codegen::CreateBigInt64ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
601 {
602 ASSERT(GetArch() != Arch::AARCH32);
603 auto entrypoint = EntrypointId::BIG_INT64_ARRAY_FILL_INTERNAL_FAST_PATH;
604 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
605 src[FOURTH_OPERAND]);
606 }
607
CreateFloat32ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)608 void Codegen::CreateFloat32ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
609 {
610 ASSERT(GetArch() != Arch::AARCH32);
611 auto valType = inst->GetInputType(SECOND_OPERAND);
612 ASSERT(valType == DataType::FLOAT32);
613 auto entrypoint = EntrypointId::FLOAT32_ARRAY_FILL_INTERNAL_FAST_PATH;
614 ScopedTmpReg tmp(GetEncoder(), ConvertDataType(DataType::INT32, GetArch()));
615 auto val = ConvertRegister(inst->GetSrcReg(SECOND_OPERAND), valType);
616 GetEncoder()->EncodeMov(tmp, val);
617 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], tmp, src[THIRD_OPERAND], src[FOURTH_OPERAND]);
618 }
619
CreateFloat64ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)620 void Codegen::CreateFloat64ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
621 {
622 ASSERT(GetArch() != Arch::AARCH32);
623 auto valType = inst->GetInputType(SECOND_OPERAND);
624 ASSERT(valType == DataType::FLOAT64);
625 auto entrypoint = EntrypointId::FLOAT64_ARRAY_FILL_INTERNAL_FAST_PATH;
626 ScopedTmpReg tmp(GetEncoder(), ConvertDataType(DataType::INT64, GetArch()));
627 auto val = ConvertRegister(inst->GetSrcReg(SECOND_OPERAND), valType);
628 GetEncoder()->EncodeMov(tmp, val);
629 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], tmp, src[THIRD_OPERAND], src[FOURTH_OPERAND]);
630 }
631
CreateUInt8ClampedArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)632 void Codegen::CreateUInt8ClampedArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
633 {
634 ASSERT(GetArch() != Arch::AARCH32);
635 auto entrypoint = EntrypointId::U_INT8_CLAMPED_ARRAY_FILL_INTERNAL_FAST_PATH;
636 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
637 src[FOURTH_OPERAND]);
638 }
639
CreateUInt8ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)640 void Codegen::CreateUInt8ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
641 {
642 ASSERT(GetArch() != Arch::AARCH32);
643 auto entrypoint = EntrypointId::U_INT8_ARRAY_FILL_INTERNAL_FAST_PATH;
644 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
645 src[FOURTH_OPERAND]);
646 }
647
CreateUInt16ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)648 void Codegen::CreateUInt16ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
649 {
650 ASSERT(GetArch() != Arch::AARCH32);
651 auto entrypoint = EntrypointId::U_INT16_ARRAY_FILL_INTERNAL_FAST_PATH;
652 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
653 src[FOURTH_OPERAND]);
654 }
655
CreateUInt32ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)656 void Codegen::CreateUInt32ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
657 {
658 ASSERT(GetArch() != Arch::AARCH32);
659 auto entrypoint = EntrypointId::U_INT32_ARRAY_FILL_INTERNAL_FAST_PATH;
660 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
661 src[FOURTH_OPERAND]);
662 }
663
CreateBigUInt64ArrayFillInternal(IntrinsicInst * inst,Reg dst,SRCREGS src)664 void Codegen::CreateBigUInt64ArrayFillInternal(IntrinsicInst *inst, Reg dst, SRCREGS src)
665 {
666 ASSERT(GetArch() != Arch::AARCH32);
667 auto entrypoint = EntrypointId::BIG_U_INT64_ARRAY_FILL_INTERNAL_FAST_PATH;
668 CallFastPath(inst, entrypoint, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND], src[THIRD_OPERAND],
669 src[FOURTH_OPERAND]);
670 }
671
672 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
673 #define CODEGEN_TYPED_ARRAY_TO_REVERSED(Name, MacroType) \
674 /* CC-OFFNXT(G.PRE.02) name part */ \
675 void Codegen::Create##Name##ArrayToReversed(IntrinsicInst *inst, Reg dst, SRCREGS src) \
676 { \
677 ASSERT(GetArch() != Arch::AARCH32); \
678 auto array = src[FIRST_OPERAND]; \
679 auto eid = RuntimeInterface::EntrypointId::MacroType##_ARRAY_TO_REVERSED; \
680 /* CC-OFFNXT(G.PRE.05) function gen */ \
681 CallFastPath(inst, eid, dst, {}, array); \
682 }
683
CODEGEN_TYPED_ARRAY_TO_REVERSED(Int8,INT8)684 CODEGEN_TYPED_ARRAY_TO_REVERSED(Int8, INT8)
685 CODEGEN_TYPED_ARRAY_TO_REVERSED(Int16, INT16)
686 CODEGEN_TYPED_ARRAY_TO_REVERSED(Int32, INT32)
687 CODEGEN_TYPED_ARRAY_TO_REVERSED(BigInt64, BIG_INT64)
688 CODEGEN_TYPED_ARRAY_TO_REVERSED(Float32, FLOAT32)
689 CODEGEN_TYPED_ARRAY_TO_REVERSED(Float64, FLOAT64)
690 CODEGEN_TYPED_ARRAY_TO_REVERSED(Uint8, UINT8)
691 CODEGEN_TYPED_ARRAY_TO_REVERSED(Uint16, UINT16)
692 CODEGEN_TYPED_ARRAY_TO_REVERSED(Uint32, UINT32)
693 CODEGEN_TYPED_ARRAY_TO_REVERSED(BigUint64, BIG_UINT64)
694
695 #undef CODEGEN_TYPED_ARRAY_TO_REVERSED
696
697 void Codegen::CreateWriteString(IntrinsicInst *inst, Reg dst, SRCREGS src)
698 {
699 ASSERT(IsCompressedStringsEnabled());
700 auto entrypointId = EntrypointId::WRITE_STRING_TO_MEM;
701 CallFastPath(inst, entrypointId, dst, {}, src[FIRST_OPERAND], src[SECOND_OPERAND]);
702 }
703
CreateReadString(IntrinsicInst * inst,Reg dst,SRCREGS src)704 void Codegen::CreateReadString(IntrinsicInst *inst, Reg dst, SRCREGS src)
705 {
706 ASSERT(IsCompressedStringsEnabled());
707 auto entrypointId = EntrypointId::CREATE_STRING_FROM_MEM;
708 auto buf = src[FIRST_OPERAND];
709 auto len = src[SECOND_OPERAND];
710 if (GetGraph()->IsAotMode()) {
711 auto *enc = GetEncoder();
712 auto offset = GetRuntime()->GetStringClassPointerTlsOffset(GetArch());
713 ScopedTmpReg klass(enc);
714 enc->EncodeLdr(klass, false, MemRef(ThreadReg(), offset));
715 CallFastPath(inst, entrypointId, dst, {}, buf, len, klass);
716 } else {
717 auto klass = GetRuntime()->GetStringClass(GetGraph()->GetMethod(), nullptr);
718 auto klassImm = TypedImm(reinterpret_cast<uintptr_t>(klass));
719 CallFastPath(inst, entrypointId, dst, {}, buf, len, klassImm);
720 }
721 }
722
723 } // namespace ark::compiler
724