1# plugin ets_string 2# Copyright (c) 2024-2025 Huawei Device Co., Ltd. 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15include_relative 'string_helpers.irt' 16include_relative '../../plugins/ets/irtoc_scripts/common.irt' 17 18module Constants 19 MEM_BLOCK_8_BYTES = "8" 20 MEM_BLOCK_16_BYTES = "16" 21 MEM_BLOCK_32_BYTES = "32" 22 MEM_BLOCK_32_ALIGN_MASK = "~31ULL" 23 LOG2_BITS_PER_U8 = "3" 24 LOG2_BITS_PER_U16 = "4" 25 LOG2_BYTES_PER_U16 = "1" 26 LOG2_BYTES_PER_OBJ_PTR = "2" # object pointer has a size of 4 bytes on each supported platform 27 XOR_SUB_U8_MASK = "0x0101010101010101ULL" 28 XOR_AND_U8_MASK = "0x8080808080808080ULL" 29 MAX_U8_VALUE = "255" 30 U8_SIZE = "1" 31 U16_SIZE = "2" 32 OBJ_PTR_SIZE = "ark::OBJECT_POINTER_SIZE" 33 WRONG_CHAR_FLAG_MASK = "0x10000UL" 34end 35 36# It is assumed that _begin_index and _end_index are safe and does not check/normalize them. 37# The range is [_begin_index, _end_index). 38# Note, a caller of this macro must provide a corresponding 'SlowPathEntrypoint' 39# for the case when 'allocate_string_tlab' fails (see StringTrim as an example) 40# Now TLAB implementation initializes memory with zero, so the hashcode field 41# is not initialized with zero explicitly. 42macro(:fast_substring) do |_str, _str_len, _begin_index, _end_index, _not_compressed| 43 _char_count := Sub(_end_index, _begin_index).u32 44 If(_char_count, Cast(_str_len).u32).EQ.Unlikely.b { 45 # Return the string itself 46 _same_str := Cast(_str).SrcType(Constants::COMPILER_REFERENCE).ptr 47 Goto(:_Fast_Substring_Result_No_Barrier) 48 } 49 _klass := load_class(_str) 50 If(_char_count, 0).EQ.Unlikely.b { 51 # Allocate and return an empty string 52 _empty_str := allocate_string_tlab(_klass, 0) 53 Goto(:_Fast_Substring_Result) 54 } 55 # Allocate a new normal string 56 _offset := Shl(_begin_index, _not_compressed).u32 57 _src_str_data := Add(Cast(_str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 58 _src_str_data := Add(_src_str_data, Cast(_offset).u64).ptr 59 If(_not_compressed, 1).EQ.Unlikely.b { 60 _compressable := is_array_of_compressable_chars(_src_str_data, Cast(_char_count).u64) 61 If(_compressable, 1).EQ.Likely.b { 62 _data_size1 := Cast(_char_count).word 63 Goto(:_L1) 64 } 65 _data_size2 := Cast(ShlI(_char_count).Imm(1).u32).word 66Label(:_L1) 67 _data_size := Phi(_data_size1, _data_size2).word 68 _new_str1 := allocate_string_tlab(_klass, _data_size) 69 _new_str_data := Add(_new_str1, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 70 If(_compressable, 1).EQ.Likely.b { 71 compress_u16_to_u8_chars(_src_str_data, _new_str_data, Cast(_char_count).u64) 72 StoreI(_new_str1, ShlI(_char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 73 Goto(:_Fast_Substring_Result) 74 } 75 copy_u16_chars(_src_str_data, _new_str_data, Cast(_char_count).u64) 76 StoreI(_new_str1, OrI(ShlI(_char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 77 Goto(:_Fast_Substring_Result) 78 } 79 # Source string is already compressed 80 _new_str2 := allocate_string_tlab(_klass, Cast(_char_count).word) 81 _new_str_data2 := Add(_new_str2, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 82 copy_u8_chars(_src_str_data, _new_str_data2, Cast(_char_count).u64) 83 StoreI(_new_str2, ShlI(_char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 84Label(:_Fast_Substring_Result) 85 # String is supposed to be a constant object, so all its data should be visible by all threads 86 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 87 _substring := Phi(_empty_str, _new_str1, _new_str1, _new_str2).ptr 88Label(:_Fast_Substring_Result_No_Barrier) 89 _result := Phi(_same_str, _substring).ptr 90end 91 92 93# 94# Test if u16 char is a white space 95# 96scoped_macro(:is_white_space_u16) do |ch| 97 IfImm(Compare(ch, 0x0020).EQ.b).Imm(0).NE.b { 98 Goto(:LabelWhiteSpaceChar) 99 } 100 # 0x000E..0x009F -- common non-whitespace chars 101 IfImm(Compare(ch, 0x000E).AE.b).Imm(0).NE.b { 102 IfImm(Compare(ch, 0x00A0).B.b).Imm(0).NE.b { 103 Goto(:LabelNotWhiteSpaceChar) 104 } 105 } 106 # 0x0009 -- horizontal tab 107 # 0x000A -- line feed or new line 108 # 0x000B -- vertical tab 109 # 0x000C -- formfeed 110 # 0x000D -- carriage return 111 IfImm(Compare(ch, 0x0009).B.b).Imm(0).NE.b { 112 Goto(:LabelNotWhiteSpaceChar) 113 } 114 IfImm(Compare(ch, 0x000D).BE.b).Imm(0).NE.b { 115 Goto(:LabelWhiteSpaceChar) 116 } 117 # 0x00A0 -- non-breaking space 118 IfImm(Compare(ch, 0x00A0).EQ.b).Imm(0).NE.b { 119 Goto(:LabelWhiteSpaceChar) 120 } 121 # 0x1680 -- Ogham space mark 122 If(ch, 0x1680).EQ.Unlikely.b { 123 Goto(:LabelWhiteSpaceChar) 124 } 125 # 0x2000 -- en quad 126 # 0x2001 -- em quad 127 # 0x2002 -- en space 128 # 0x2003 -- em space 129 # 0x2004 -- three-per-em space 130 # 0x2005 -- four-per-em space 131 # 0x2006 -- six-per-em space 132 # 0x2007 -- figure space 133 # 0x2008 -- punctuation space 134 # 0x2009 -- thin space 135 # 0x200A -- hair space 136 If(ch, 0x2000).B.Unlikely.b { 137 Goto(:LabelNotWhiteSpaceChar) 138 } 139 If(ch, 0x200A).BE.Unlikely.b { 140 Goto(:LabelWhiteSpaceChar) 141 } 142 # 0x2028 -- line separator 143 If(ch, 0x2028).EQ.Unlikely.b { 144 Goto(:LabelWhiteSpaceChar) 145 } 146 # 0x2029 -- paragraph separator 147 If(ch, 0x2029).EQ.Unlikely.b { 148 Goto(:LabelWhiteSpaceChar) 149 } 150 # 0x202F -- narrow no-break space 151 If(ch, 0x202F).EQ.Unlikely.b { 152 Goto(:LabelWhiteSpaceChar) 153 } 154 # 0x205F -- medium mathematical space 155 If(ch, 0x205F).EQ.Unlikely.b { 156 Goto(:LabelWhiteSpaceChar) 157 } 158 # 0xFEFF -- byte order mark 159 If(ch, 0xFEFF).EQ.Unlikely.b { 160 Goto(:LabelWhiteSpaceChar) 161 } 162 # 0x3000 -- ideographic space 163 If(ch, 0x3000).EQ.Unlikely.b { 164 Goto(:LabelWhiteSpaceChar) 165 } 166Label(:LabelNotWhiteSpaceChar) 167 whiteSpace0 := 0 168 Goto(:LabelReturn) 169Label(:LabelWhiteSpaceChar) 170 whiteSpace1 := 1 171Label(:LabelReturn) 172 result := Phi(whiteSpace0, whiteSpace1).b 173end 174 175# 176# Test if u8 char is a white space 177# 178scoped_macro(:is_white_space_u8) do |ch| 179 IfImm(Compare(ch, 0x20).EQ.b).Imm(0).NE.b { 180 Goto(:LabelWhiteSpaceChar) 181 } 182 # 0x0E..0x9F -- common non-whitespace chars 183 IfImm(Compare(ch, 0x0E).AE.b).Imm(0).NE.b { 184 IfImm(Compare(ch, 0xA0).B.b).Imm(0).NE.b { 185 Goto(:LabelNotWhiteSpaceChar) 186 } 187 } 188 # 0x09 -- horizontal tab 189 # 0x0A -- line feed or new line 190 # 0x0B -- vertical tab 191 # 0x0C -- formfeed 192 # 0x0D -- carriage return 193 IfImm(Compare(ch, 0x09).B.b).Imm(0).NE.b { 194 Goto(:LabelNotWhiteSpaceChar) 195 } 196 IfImm(Compare(ch, 0x0D).BE.b).Imm(0).NE.b { 197 Goto(:LabelWhiteSpaceChar) 198 } 199 # 0xA0 -- non-breaking space 200 IfImm(Compare(ch, 0xA0).EQ.b).Imm(0).NE.b { 201 Goto(:LabelWhiteSpaceChar) 202 } 203Label(:LabelNotWhiteSpaceChar) 204 whiteSpace0 := 0 205 Goto(:LabelReturn) 206Label(:LabelWhiteSpaceChar) 207 whiteSpace1 := 1 208Label(:LabelReturn) 209 result := Phi(whiteSpace0, whiteSpace1).b 210end 211 212 213function(:CharIsWhiteSpace, 214 params: {ch: 'u16'}, 215 regmap: $full_regmap, 216 regalloc_set: $panda_mask, 217 mode: [:FastPath]) { 218 219 if Options.arch == :arm32 220 Intrinsic(:UNREACHABLE).Terminator.void 221 ReturnVoid().void 222 next 223 end 224 Return(is_white_space_u16(ch)).b 225} 226 227 228function(:StringEmpty, 229 params: {str: 'ref'}, 230 regmap: $full_regmap, 231 regalloc_set: $panda_mask, 232 mode: [:FastPath]) { 233 234 if Options.arch == :arm32 235 Intrinsic(:UNREACHABLE).Terminator.void 236 ReturnVoid().void 237 next 238 end 239 240 klass := load_class(str) 241 empty_str := allocate_string_tlab(klass, 0) 242 # String is supposed to be a constant object, so all its data should be visible by all threads 243 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 244 Return(empty_str).ptr 245Label(:SlowPathEntrypoint) 246 entrypoint = get_entrypoint_offset("CREATE_EMPTY_STRING_SLOW_PATH") 247 Intrinsic(:SLOW_PATH_ENTRY).AddImm(entrypoint).MethodAsImm("CreateEmptyString1ArgBridge").Terminator.ptr 248 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 249} 250 251 252if Options.arch == :arm64 253 trim_left_regs = $temps_mask + :callee0 + :callee1 254 trim_right_regs = $temps_mask + :callee0 + :callee1 255else 256 trim_left_regs = $temps_mask + :callee0 + :caller0 + :caller1 257 trim_right_regs = $temps_mask + :callee0 + :caller0 + :caller1 258end 259 260 261function(:StringTrimLeftBase, 262 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 263 regmap: $full_regmap, 264 regalloc_set: $panda_mask, 265 mode: [:FastPath]) { 266 267 if Options.arch == :arm32 268 Intrinsic(:UNREACHABLE).Terminator.void 269 ReturnVoid().void 270 next 271 end 272 273 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 274 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 275 not_compressed := AndI(length_packed).Imm(1).i32 276 length := ShrI(length_packed).Imm(1).i32 277 start_index := Cast(1).i32 278 If(not_compressed, 0).EQ.Likely.b { 279 # String contains 8-bit chars 280Label(:Loop1) 281 i := Phi(start_index, i1).i32 282 ws1 := is_white_space_u8(Load(str_data, i).u8) 283 If(ws1, 0).NE.Likely.b { 284 i1 := AddI(i).Imm(1).i32 285 If(i1, length).LT.Likely.b { 286 Goto(:Loop1) 287 } 288 } 289 index1 := Phi(i, i1).i32 290 Goto(:TrimLeft) 291 } 292 # String contains 16-bit chars 293Label(:Loop2) 294 j := Phi(start_index, j1).i32 295 ws2 := is_white_space_u16(Load(str_data, ShlI(j).Imm(1).i32).u16) 296 If(ws2, 0).NE.Likely.b { 297 j1 := AddI(j).Imm(1).i32 298 If(j1, length).LT.Likely.b { 299 Goto(:Loop2) 300 } 301 } 302 index2 := Phi(j, j1).i32 303Label(:TrimLeft) 304 index := Phi(index1, index2).i32 305 trimmed := fast_substring(str, length, index, length, not_compressed) 306 Return(trimmed).ptr 307Label(:SlowPathEntrypoint) 308 entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH") 309 Intrinsic(:SLOW_PATH_ENTRY, str, index, length).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr 310 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 311} 312 313 314function(:StringTrimLeft, 315 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 316 regmap: $full_regmap, 317 regalloc_set: $trim_left_regs, 318 mode: [:FastPath]) { 319 320 if Options.arch == :arm32 321 Intrinsic(:UNREACHABLE).Terminator.void 322 ReturnVoid().void 323 next 324 end 325 326 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 327 If(length_packed, 1).LE.Unlikely.b { 328 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 329 } 330 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 331 not_compressed := AndI(length_packed).Imm(1).i32 332 If(not_compressed, 0).EQ.Likely.b { 333 ws1 := is_white_space_u8(Load(str_data, 0).u8) 334 Goto(:L1) 335 } 336 ws2 := is_white_space_u16(Load(str_data, 0).u16) 337Label(:L1) 338 ws := Phi(ws1, ws2).b 339 If(ws, 0).EQ.Likely.b { 340 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 341 } 342 If(ShrI(length_packed).Imm(1).i32, 1).EQ.Unlikely.b { 343 LiveOut(str).DstReg(regmap[:arg0]).ref 344 entrypoint1 = get_entrypoint_offset("STRING_EMPTY") 345 Intrinsic(:TAIL_CALL).AddImm(entrypoint1).MethodAsImm("StringEmpty").Terminator.ptr 346 } 347 LiveOut(str).DstReg(regmap[:arg0]).ref 348 LiveOut(unused1).DstReg(regmap[:arg1]).i32 349 LiveOut(unused2).DstReg(regmap[:arg2]).i32 350 entrypoint2 = get_entrypoint_offset("STRING_TRIM_LEFT_BASE") 351 Intrinsic(:TAIL_CALL).AddImm(entrypoint2).MethodAsImm("StringTrimLeftBase").Terminator.ptr 352} 353 354 355function(:StringTrimRightBase, 356 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 357 regmap: $full_regmap, 358 regalloc_set: $panda_mask, 359 mode: [:FastPath]) { 360 361 if Options.arch == :arm32 362 Intrinsic(:UNREACHABLE).Terminator.void 363 ReturnVoid().void 364 next 365 end 366 367 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 368 length := ShrI(length_packed).Imm(1).i32 369 start_index := SubI(length).Imm(2).i32 370 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 371 not_compressed := AndI(length_packed).Imm(1).i32 372 If(not_compressed, 0).EQ.Likely.b { 373 # String contains 8-bit chars 374Label(:Loop1) 375 i := Phi(start_index, i1).i32 376 ws1 := is_white_space_u8(Load(str_data, i).u8) 377 If(ws1, 0).NE.Likely.b { 378 i1 := SubI(i).Imm(1).i32 379 If(i1, 0).GE.Likely.b { 380 Goto(:Loop1) 381 } 382 } 383 index1 := Phi(i, i1).i32 384 Goto(:TrimRight) 385 } 386 # String contains 16-bit chars 387Label(:Loop2) 388 j := Phi(start_index, j1).i32 389 ws2 := is_white_space_u16(Load(str_data, ShlI(j).Imm(1).i32).u16) 390 If(ws2, 0).NE.Likely.b { 391 j1 := SubI(j).Imm(1).i32 392 If(j1, 0).GE.Likely.b { 393 Goto(:Loop2) 394 } 395 } 396 index2 := Phi(j, j1).i32 397Label(:TrimRight) 398 index := Phi(index1, index2).i32 399 index := AddI(index).Imm(1).i32 400 trimmed := fast_substring(str, length, 0, index, not_compressed) 401 Return(trimmed).ptr 402Label(:SlowPathEntrypoint) 403 entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH") 404 Intrinsic(:SLOW_PATH_ENTRY, str, Cast(0).i32, index).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr 405 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 406} 407 408 409function(:StringTrimRight, 410 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 411 regmap: $full_regmap, 412 regalloc_set: $trim_right_regs, 413 mode: [:FastPath]) { 414 415 if Options.arch == :arm32 416 Intrinsic(:UNREACHABLE).Terminator.void 417 ReturnVoid().void 418 next 419 end 420 421 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 422 If(length_packed, 1).LE.Unlikely.b { 423 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 424 } 425 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 426 not_compressed := AndI(length_packed).Imm(1).i32 427 length := ShrI(length_packed).Imm(1).i32 428 last_char_index := SubI(length).Imm(1).i32 429 If(not_compressed, 0).EQ.Likely.b { 430 ws1 := is_white_space_u8(Load(str_data, last_char_index).u8) 431 Goto(:L1) 432 } 433 ws2 := is_white_space_u16(Load(str_data, ShlI(last_char_index).Imm(1).i32).u16) 434Label(:L1) 435 ws := Phi(ws1, ws2).b 436 If(ws, 0).EQ.Likely.b { 437 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 438 } 439 If(length, 1).EQ.Unlikely.b { 440 LiveOut(str).DstReg(regmap[:arg0]).ref 441 entrypoint1 = get_entrypoint_offset("STRING_EMPTY") 442 Intrinsic(:TAIL_CALL).AddImm(entrypoint1).MethodAsImm("StringEmpty").Terminator.ptr 443 } 444 LiveOut(str).DstReg(regmap[:arg0]).ref 445 LiveOut(unused1).DstReg(regmap[:arg1]).i32 446 LiveOut(unused2).DstReg(regmap[:arg2]).i32 447 entrypoint2 = get_entrypoint_offset("STRING_TRIM_RIGHT_BASE") 448 Intrinsic(:TAIL_CALL).AddImm(entrypoint2).MethodAsImm("StringTrimRightBase").Terminator.ptr 449} 450 451 452function(:StringTrimBase, 453 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 454 regmap: $full_regmap, 455 regalloc_set: $panda_mask, 456 mode: [:FastPath]) { 457 458 if Options.arch == :arm32 459 Intrinsic(:UNREACHABLE).Terminator.void 460 ReturnVoid().void 461 next 462 end 463 464 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 465 length := ShrI(length_packed).Imm(1).i32 466 left := 0 467 right := SubI(length).Imm(2).i32 468 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 469 not_compressed := AndI(length_packed).Imm(1).i32 470 If(not_compressed, 0).EQ.Likely.b { 471 # String contains 8-bit chars 472Label(:Loop1) # while (utf::IsWhiteSpaceChar(str->At(right))) 473 right1 := Phi(right, right2).i32 474 If(is_white_space_u8(Load(str_data, right1).u8), 0).NE.Likely.b { 475 If(right1, 0).EQ.Unlikely.b { 476 Goto(:Trim) 477 } 478 right2 := SubI(right1).Imm(1).i32 479 Goto(:Loop1) 480 } 481Label(:Loop2) # while (left < right && utf::IsWhiteSpaceChar(str->At(left))) 482 left1 := Phi(left, left2).i32 483 If(left1, right1).LT.Unlikely.b { 484 If(is_white_space_u8(Load(str_data, left1).u8), 0).NE.Likely.b { 485 left2 := AddI(left1).Imm(1).i32 486 Goto(:Loop2) 487 } 488 } 489 right3 := AddI(right1).Imm(1).i32 490 Goto(:Trim) 491 } 492 # String contains 16-bit chars 493Label(:Loop3) # while (utf::IsWhiteSpaceChar(str->At(right))) 494 right11 := Phi(right, right22).i32 495 If(is_white_space_u16(Load(str_data, ShlI(right11).Imm(1).i32).u16), 0).NE.Likely.b { 496 If(right11, 0).EQ.Unlikely.b { 497 Goto(:Trim) 498 } 499 right22 := SubI(right11).Imm(1).i32 500 Goto(:Loop3) 501 } 502Label(:Loop4) # while (left < right && utf::IsWhiteSpaceChar(str->At(left))) 503 left11 := Phi(left, left22).i32 504 If(left11, right11).LT.Unlikely.b { 505 If(is_white_space_u16(Load(str_data, ShlI(left11).Imm(1).i32).u16), 0).NE.Likely.b { 506 left22 := AddI(left11).Imm(1).i32 507 Goto(:Loop4) 508 } 509 } 510 right33 := AddI(right11).Imm(1).i32 511Label(:Trim) 512 l := Phi(left, left1, left, left11).i32 513 r := Phi(right1, right3, right11, right33).i32 514 trimmed := fast_substring(str, length, l, r, not_compressed) 515 Return(trimmed).ptr 516Label(:SlowPathEntrypoint) 517 entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH") 518 Intrinsic(:SLOW_PATH_ENTRY, str, l, r).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr 519 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 520} 521 522 523function(:StringTrim, 524 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 525 regmap: $full_regmap, 526 regalloc_set: $panda_mask, 527 mode: [:FastPath]) { 528 529 if Options.arch == :arm32 530 Intrinsic(:UNREACHABLE).Terminator.void 531 ReturnVoid().void 532 next 533 end 534 535 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 536 # length == 0 537 If(length_packed, 1).LE.Unlikely.b { 538 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 539 } 540 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 541 not_compressed := AndI(length_packed).Imm(1).i32 542 length := ShrI(length_packed).Imm(1).i32 543 # length == 1 544 If(length, 1).EQ.b { 545 If(not_compressed, 0).EQ.Likely.b { 546 ws1 := is_white_space_u8(Load(str_data, 0).u8) 547 Goto(:L1) 548 } 549 ws2 := is_white_space_u16(Load(str_data, 0).u16) 550Label(:L1) 551 ws3 := Phi(ws1, ws2).b 552 If(ws3, 0).EQ.Likely.b { 553 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 554 } 555 LiveOut(str).DstReg(regmap[:arg0]).ref 556 entrypoint1 = get_entrypoint_offset("STRING_EMPTY") 557 Intrinsic(:TAIL_CALL).AddImm(entrypoint1).MethodAsImm("StringEmpty").Terminator.ptr 558 } 559 # length > 1 560 last_char_index := SubI(length).Imm(1).i32 561 If(not_compressed, 0).EQ.Likely.b { 562 ws4 := is_white_space_u8(Load(str_data, last_char_index).u8) 563 Goto(:L2) 564 } 565 ws5 := is_white_space_u16(Load(str_data, ShlI(last_char_index).Imm(1).i32).u16) 566Label(:L2) 567 ws6 := Phi(ws4, ws5).b 568 If(ws6, 0).EQ.Likely.b { 569 # last char is not whitespace, so check the first char 570 If(not_compressed, 0).EQ.Likely.b { 571 ws7 := is_white_space_u8(Load(str_data, 0).u8) 572 Goto(:L3) 573 } 574 ws8 := is_white_space_u16(Load(str_data, 0).u16) 575Label(:L3) 576 ws9 := Phi(ws7, ws8).b 577 If(ws9, 0).EQ.Likely.b { 578 # first char is not white space, so return 'str' 579 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 580 } 581 Goto(:FirstCharWhitespace) 582 } 583 # last char is whitespace, so call StringTrimBase 584 LiveOut(str).DstReg(regmap[:arg0]).ref 585 LiveOut(unused1).DstReg(regmap[:arg1]).i32 586 LiveOut(unused2).DstReg(regmap[:arg2]).i32 587 entrypoint2 = get_entrypoint_offset("STRING_TRIM_BASE") 588 Intrinsic(:TAIL_CALL).AddImm(entrypoint2).MethodAsImm("StringTrimBase").Terminator.ptr 589Label(:FirstCharWhitespace) 590 LiveOut(str).DstReg(regmap[:arg0]).ref 591 LiveOut(unused1).DstReg(regmap[:arg1]).i32 592 LiveOut(unused2).DstReg(regmap[:arg2]).i32 593 entrypoint3 = get_entrypoint_offset("STRING_TRIM_LEFT_BASE") 594 Intrinsic(:TAIL_CALL).AddImm(entrypoint3).MethodAsImm("StringTrimLeftBase").Terminator.ptr 595} 596 597 598scoped_macro(:at) do |str_data, index, not_compressed| 599 If(not_compressed, 0).EQ.Likely.b { 600 # String contains 8-bit chars 601 c8 := Cast(Load(str_data, index).u8).u16 602 Goto(:Done) 603 } 604 # String contains 16-bit chars 605 c16 := Load(str_data, ShlI(index).Imm(1).i32).u16 606Label(:Done) 607 c := Phi(c8, c16).u16 608end 609 610 611function(:StringStartsWithBase, 612 params: {str: 'ref', pfx: 'ref', from_index: 'i32'}, 613 regmap: $full_regmap, 614 regalloc_set: $panda_mask, 615 mode: [:FastPath]) { 616 617 if Options.arch == :arm32 618 Intrinsic(:UNREACHABLE).Terminator.void 619 ReturnVoid().void 620 next 621 end 622 623 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 624 str_not_compressed := AndI(str_len_packed).Imm(1).i32 625 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 626 pfx_len_packed := LoadI(pfx).Imm(Constants::STRING_LENGTH_OFFSET).u32 627 pfx_len := ShrI(pfx_len_packed).Imm(1).i32 628 pfx_not_compressed := AndI(pfx_len_packed).Imm(1).i32 629 pfx_data := Add(Cast(pfx).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 630 631 pfx_i1 := 0 632Label(:Loop) 633 str_i := Phi(from_index, str_i1).i32 634 pfx_i := Phi(pfx_i1, pfx_i2).i32 635 If(pfx_i, pfx_len).GE.Unlikely.b { 636 Goto(:Done) 637 } 638 s := at(str_data, str_i, str_not_compressed) 639 p := at(pfx_data, pfx_i, pfx_not_compressed) 640 If(s, p).NE.Likely.b { 641 Return(0).b 642 } 643 pfx_i2 := AddI(pfx_i).Imm(1).i32 644 str_i1 := AddI(str_i).Imm(1).i32 645 Goto(:Loop) 646Label(:Done) 647 Return(1).b 648} 649 650 651function(:StringStartsWith, 652 params: {str: 'ref', pfx: 'ref', from_index: 'i32'}, 653 regmap: $full_regmap, 654 regalloc_set: $panda_mask, 655 mode: [:FastPath]) { 656 657 if Options.arch == :arm32 658 Intrinsic(:UNREACHABLE).Terminator.void 659 ReturnVoid().void 660 next 661 end 662 663 pfx_len_packed := LoadI(pfx).Imm(Constants::STRING_LENGTH_OFFSET).u32 664 # Return 'true' if prefix is empty 665 # The least significant bit indicates COMPRESSED/UNCOMPRESSED, 666 # thus if (packed length <= 1) then the actual length is equal to 0. 667 If(pfx_len_packed, 1).LE.Unlikely.b { 668 Return(1).b 669 } 670 671 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 672 # Return 'false' if 'str' is empty as 'prefix' is not empty. 673 If(str_len_packed, 1).LE.Unlikely.b { 674 Return(0).b 675 } 676 677 # If 'from_index' is less than zero then make it zero. 678 IfImm(Compare(from_index, 0).LT.b).Imm(0).NE.Unlikely.b { 679 from_index1 := Cast(0).i32 680 } 681 from_index2 := Phi(from_index, from_index1).i32 682 683 str_len := ShrI(str_len_packed).Imm(1).i32 684 pfx_len := ShrI(pfx_len_packed).Imm(1).i32 685 686 If(from_index2, Sub(str_len, pfx_len).i32).GT.Unlikely.b { 687 # Return 'false' in this case, as we know that 'pfx' is not empty 688 # and it is longer than the part of 'str' to be checked. 689 Return(0).b 690 } 691 692 LiveOut(str).DstReg(regmap[:arg0]).ref 693 LiveOut(pfx).DstReg(regmap[:arg1]).ref 694 LiveOut(from_index2).DstReg(regmap[:arg2]).i32 695 entrypoint = get_entrypoint_offset("STRING_STARTS_WITH_BASE") 696 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringStartsWithBase").Terminator.b 697} 698 699 700function(:StringEndsWithBase, 701 params: {str: 'ref', sfx: 'ref', end_index: 'i32'}, 702 regmap: $full_regmap, 703 regalloc_set: $panda_mask, 704 mode: [:FastPath]) { 705 706 if Options.arch == :arm32 707 Intrinsic(:UNREACHABLE).Terminator.void 708 ReturnVoid().void 709 next 710 end 711 712 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 713 str_not_compressed := AndI(str_len_packed).Imm(1).i32 714 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 715 sfx_len_packed := LoadI(sfx).Imm(Constants::STRING_LENGTH_OFFSET).u32 716 sfx_not_compressed := AndI(sfx_len_packed).Imm(1).i32 717 sfx_data := Add(Cast(sfx).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 718 sfx_len := ShrI(sfx_len_packed).Imm(1).i32 719 from_index := Sub(end_index, sfx_len).i32; 720 721 sfx_i1 := 0 722Label(:Loop) 723 str_i := Phi(from_index, str_i1).i32 724 sfx_i := Phi(sfx_i1, sfx_i2).i32 725 If(sfx_i, sfx_len).GE.Unlikely.b { 726 Goto(:Done) 727 } 728 s := at(str_data, str_i, str_not_compressed) 729 p := at(sfx_data, sfx_i, sfx_not_compressed) 730 If(s, p).NE.Likely.b { 731 Return(0).b 732 } 733 sfx_i2 := AddI(sfx_i).Imm(1).i32 734 str_i1 := AddI(str_i).Imm(1).i32 735 Goto(:Loop) 736 737Label(:Done) 738 Return(1).b 739} 740 741 742function(:StringEndsWith, 743 params: {str: 'ref', sfx: 'ref', end_index: 'i32'}, 744 regmap: $full_regmap, 745 regalloc_set: $panda_mask, 746 mode: [:FastPath]) { 747 748 if Options.arch == :arm32 749 Intrinsic(:UNREACHABLE).Terminator.void 750 ReturnVoid().void 751 next 752 end 753 754 sfx_len_packed := LoadI(sfx).Imm(Constants::STRING_LENGTH_OFFSET).u32 755 # Return 'true' if suffix is empty 756 # The least significant bit indicates COMPRESSED/UNCOMPRESSED, 757 # thus if (packed length <= 1) then the actual length is equal to 0. 758 If(sfx_len_packed, 1).LE.Unlikely.b { 759 Return(1).b 760 } 761 762 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 763 # Return 'false' if 'str' is empty as 'suffix' is not empty. 764 If(str_len_packed, 1).LE.Unlikely.b { 765 Return(0).b 766 } 767 # If 'end_index' is less or equal to zero then return false. 768 IfImm(Compare(end_index, 0).LE.b).Imm(0).NE.Unlikely.b { 769 Return(0).b 770 } 771 772 str_len := ShrI(str_len_packed).Imm(1).i32 773 # If 'end_index' is greater than length of 'str' make it equal to length of 'str'. 774 If(end_index, str_len).GT.Unlikely.b { 775 end_index1 := str_len 776 } 777 end_index2 := Phi(end_index, end_index1).i32 778 779 sfx_len := ShrI(sfx_len_packed).Imm(1).i32 780 from_index := Sub(end_index2, sfx_len).i32; 781 IfImm(Compare(from_index, 0).LT.b).Imm(0).NE.Unlikely.b { 782 # Return 'false' in this case, as 'sfx' length is greater than 'end_index'. 783 Return(0).b 784 } 785 786 LiveOut(str).DstReg(regmap[:arg0]).ref 787 LiveOut(sfx).DstReg(regmap[:arg1]).ref 788 LiveOut(end_index2).DstReg(regmap[:arg2]).i32 789 entrypoint = get_entrypoint_offset("STRING_ENDS_WITH_BASE") 790 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringEndsWithBase").Terminator.b 791} 792 793function(:StringGetBytesTlab, 794 params: {str: 'ref', begin_index: 'i32', end_index: 'i32', array_klass: 'ref'}, 795 regmap: $full_regmap, 796 regalloc_set: $panda_mask, 797 mode: [:FastPath]) { 798 799 if Options.arch == :arm32 800 Intrinsic(:UNREACHABLE).Terminator.void 801 ReturnVoid().void 802 next 803 end 804 805 If(begin_index, end_index).GT.Unlikely.b { 806 Goto(:SlowPathEntrypoint) # Out of range 807 } 808 809 If(begin_index, Cast(0).i32).LT.Unlikely.b { 810 Goto(:SlowPathEntrypoint) # Out of range 811 } 812 813 # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded) 814 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32; 815 uncompressed := AndI(length).Imm(1).u32; 816 length := ShrI(length).Imm(1).u32; 817 818 If(Cast(end_index).u32, length).A.Unlikely.b { 819 Goto(:SlowPathEntrypoint) # Out of range 820 } 821 offset := Shl(begin_index, uncompressed).u32 822 823 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 824 src_str_data := Add(src_str_data, Cast(offset).u64).ptr 825 826# Allocate a new array of u8 (bytes) 827 count := Sub(Cast(end_index).u32, Cast(begin_index).u32).u64 828 new_arr := allocate_array_of_bytes_tlab(array_klass, Cast(count).word) 829 new_arr_data := Add(new_arr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr 830 If(uncompressed, Cast(0).u32).EQ.Likely.b { 831 copy_u8_chars(src_str_data, new_arr_data, count) 832 Goto(:End) 833 } 834 compress_u16_to_u8_chars(src_str_data, new_arr_data, count) 835 836 Label(:End) 837 Return(new_arr).ptr 838 839 Label(:SlowPathEntrypoint) 840 entrypoint = get_entrypoint_offset("STRING_GET_BYTES_SLOW_PATH") 841 Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("StringGetBytes4ArgBridge").Terminator.ptr 842 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 843} 844 845### 846# Char codes in an instance of 'escompat.Array' are represented as an array of object pointers to instances of the 847# boxing class 'std.core.Double'. Runtime stores such objects as instances of the 'ark::ets::EtsBoxPrimitive' class 848# and every instance has a field of type 'double', the value. 849# 850# The macro reads the boxed value by an object pointer and applies unboxing. 851# 852scoped_macro(:load_char_code) do |char_codes, offset| 853 _boxed_ptr := get_object_pointer(char_codes, offset) 854 _char_core := LoadI(_boxed_ptr).Imm(EtsConstants::BOX_PRIMITIVE_VALUE_OFFSET).f64 855end # load_char_code 856 857### 858# Tries to convert flag_char_pair, the result of calling the JS_CAST_DOUBLE_TO_CHAR intrinsic, into a Utf16 char (u16). 859# The macro checks whether the conversion was successful: whether an overflow or underflow takes place during the 860# conversion of a denormalized number and if so goes to the slow path. 861# 862scoped_macro(:validate_flag_char_pair) do |flag_char_pair| 863 _convertible1 := Cast(1).b 864 If(AndI(flag_char_pair).Imm(Constants::WRONG_CHAR_FLAG_MASK).u32, 0).NE.Unlikely { 865 _convertible2 := Cast(0).b 866 } 867 _convertible := Phi(_convertible1, _convertible2).b 868end # validate_flag_char_pair 869 870### 871# Tries to convert a char_code into a Utf16 char (u16). 872# The macro checks whether the conversion was successful: whether an overflow or underflow takes place during the 873# conversion of a denormalized number and if so goes to the slow path. 874# 875scoped_macro(:validate_char_code) do |char_code| 876 _flag_char_pair := Intrinsic(:JS_CAST_DOUBLE_TO_CHAR, char_code).u32 877 _convertible := validate_flag_char_pair(_flag_char_pair) 878end # validate_char_code 879 880### 881# Checks if starting from codes_data the specified number of codes (codes_count) are convertible: whenever 882# an overflow or underflow takes place during the conversion of a denormalized number we have to go to the 883# slow path. 884# 885scoped_macro(:is_array_of_convertible_char_codes) do |codes_data, codes_count| 886 if Options.jscvt_feature_enabled? 887 _convertible := Cast(1).b 888 else 889 _offset1 := Cast(0).u64 890 _codes_len := ShlI(codes_count).Imm(Constants::LOG2_BYTES_PER_OBJ_PTR).u64 891 _convertible1 := Cast(1).b 892Label(:Loop) 893 _offset := Phi(_offset1, _offset2).u64 894 If(_offset, _codes_len).AE.Unlikely { 895 Goto(:LoopDone) 896 } 897 _convertible2 := validate_char_code(load_char_code(codes_data, _offset).f64) 898 If(_convertible2, 0).EQ.Unlikely.b { 899 Goto(:LoopDone) 900 } 901 _offset2 := AddI(_offset).Imm(Constants::OBJ_PTR_SIZE).u64 902 Goto(:Loop) 903Label(:LoopDone) 904 _convertible := Phi(_convertible1, _convertible2).b 905 end 906end # is_array_of_convertible_char_codes 907 908### 909# Converts a flag_char_pair, the result of calling the JS_CAST_DOUBLE_TO_CHAR intrinsic, into a Utf16 char (u16). 910# Note, a caller of this macro must be sure the conversion will always be successful: neither overflow nor 911# underflow will take place during the conversion of a denormalized number. 912# 913scoped_macro(:get_char_from_flag_char_pair) do |flag_char_pair| 914 if !Options.jscvt_feature_enabled? and defines.DEBUG 915 If(AndI(flag_char_pair).Imm(Constants::WRONG_CHAR_FLAG_MASK).u32, 0).NE.Unlikely { 916 Intrinsic(:UNREACHABLE).Terminator.void 917 } 918 end 919 _char := Cast(flag_char_pair).u16 920end # get_char_from_flag_char_pair 921 922### 923# Converts a char_code (f64) into a Utf16 char (u16). 924# Note, a caller of this macro must be sure the conversion will always be successful: neither overflow nor 925# underflow will take place during the conversion of a denormalized number. 926# 927scoped_macro(:get_char_from_code) do |char_code| 928 _flag_char_pair := Intrinsic(:JS_CAST_DOUBLE_TO_CHAR, char_code).u32 929 _char := get_char_from_flag_char_pair(_flag_char_pair) 930end # get_char_from_code 931 932### 933# Checks if starting from codes_data the specified number of codes (codes_count) represent 934# an array of compressible chars 935# 936# Utf16 char is ASCII if (utf16_char - 1U < utf::UTF8_1B_MAX) 937# See runtime/include/coretypes/string.h - IsASCIICharacter 938# 939scoped_macro(:is_array_of_compressible_char_codes) do |codes_data, codes_count| 940 _offset1 := Cast(0).u64 941 _codes_len := ShlI(codes_count).Imm(Constants::LOG2_BYTES_PER_OBJ_PTR).u64 942 _compressible1 := Cast(1).b 943Label(:Loop) 944 _offset := Phi(_offset1, _offset2).u64 945 If(_offset, _codes_len).AE.Unlikely { 946 Goto(:LoopDone) 947 } 948 _char := get_char_from_code(load_char_code(codes_data, _offset).f64) 949 If(SubI(_char).Imm(Constants::STRING_MUTF8_1B_MIN).u16, Cast(Constants::STRING_MUTF8_1B_MAX).u16).AE.Unlikely { 950 _compressible2 := Cast(0).b 951 Goto(:LoopDone) 952 } 953 _offset2 := AddI(_offset).Imm(Constants::OBJ_PTR_SIZE).u64 954 Goto(:Loop) 955 956Label(:LoopDone) 957 _compressible := Phi(_compressible1, _compressible2).b 958end # is_array_of_compressible_char_codes 959 960### 961# Converts char codes (numbers) to chars with type equals either to "u8" or "u16". For type equals to "u8", it is 962# assumed that all char codes represent compressible chars. 963# 964["u8", "u16"].each do |type| 965 scoped_macro("convert_char_codes_to_#{type}_chars".to_sym) do |src, dst, count| 966 compressed_string = type == "u8" 967 _i1 := Cast(0).u64 968Label(:Loop) 969 _i := Phi(_i1, _i2).u64 970 If(_i, count).AE.Unlikely { 971 Goto(:LoopDone) 972 } 973 _code_offset := ShlI(_i).Imm(Constants::LOG2_BYTES_PER_OBJ_PTR).u64 974 _char := get_char_from_code(load_char_code(src, _code_offset).f64) 975 if compressed_string 976 _char_offset := Cast(_i).u64 977 Store(dst, _char_offset, _char).u8 978 else 979 _char_offset := ShlI(_i).Imm(Constants::LOG2_BYTES_PER_U16).u64 980 Store(dst, _char_offset, _char).u16 981 end 982 _i2 := AddI(_i).Imm(1).u64 983 Goto(:Loop) 984 985Label(:LoopDone) 986 end # convert_char_codes_to_#{type}_chars 987end 988 989def GenerateCreateStringFromCharCodeTlab(string_compression_enabled) 990 suffix = (string_compression_enabled ? "Compressed" : "") 991 available_regs = $panda_mask 992 function("CreateStringFromCharCodeTlab#{suffix}".to_sym, 993 params: {char_codes: 'ref', string_klass: 'ref'}, 994 regmap: $full_regmap, 995 regalloc_set: available_regs, 996 mode: [:FastPath]) { 997 998 if Options.arch == :arm32 999 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 1000 ReturnVoid().void 1001 next 1002 end 1003 1004 # There is no check of the arguments against NullPointer as 1005 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) for non-empty arrays 1006 # and we suppose that the `NewArray (size=0)` instruction never returns `null`. 1007 if defines.DEBUG 1008 If(char_codes, 0).EQ { 1009 Intrinsic(:UNREACHABLE).Terminator.void 1010 } 1011 end 1012 1013 char_codes_array := get_object_pointer_imm(char_codes, EtsConstants::ESCOMPAT_ARRAY_DATA_OFFSET) 1014 codes_data := AddI(char_codes_array).Imm(Constants::ARRAY_DATA_OFFSET).ptr 1015 codes_count := LoadI(char_codes_array).Imm(Constants::ARRAY_LENGTH_OFFSET).u32 1016 1017 # Validate if all the char codes are convertible: whenever an overflow or underflow takes place during the 1018 # conversion of a denormalized number we have to go to the slow path. The validation must take place before 1019 # any attempts to allocate the memory in TLAB. 1020 convertible := is_array_of_convertible_char_codes(codes_data, Cast(codes_count).u64) 1021 If(convertible, 0).EQ.Unlikely { 1022 Goto(:SlowPathEntrypoint) 1023 } 1024 1025 # Allocate a new string 1026 if string_compression_enabled 1027 compressible := is_array_of_compressible_char_codes(codes_data, Cast(codes_count).u64) 1028 If(compressible, 1).EQ.Likely { 1029 data_size1 := Cast(codes_count).word 1030 } Else { 1031 data_size2 := Cast(ShlI(codes_count).Imm(Constants::LOG2_BYTES_PER_U16).u32).word 1032 } 1033 data_size := Phi(data_size1, data_size2).word 1034 else 1035 data_size := Cast(ShlI(codes_count).Imm(Constants::LOG2_BYTES_PER_U16).u32).word 1036 end 1037 new_str := allocate_string_tlab(string_klass, data_size) 1038 str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 1039 1040 # Copy data from char_codes to the new string with the required preprocessing 1041 # String length field is set according to SetLength() from runtime/include/coretypes/string.h 1042 if string_compression_enabled 1043 If(compressible, 1).EQ.Likely { 1044 convert_char_codes_to_u8_chars(codes_data, str_data, Cast(codes_count).u64) 1045 StoreI(new_str, ShlI(codes_count).Imm(Constants::LOG2_BYTES_PER_U16).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 1046 } Else { 1047 convert_char_codes_to_u16_chars(codes_data, str_data, Cast(codes_count).u64) 1048 StoreI(new_str, OrI(ShlI(codes_count).Imm(Constants::LOG2_BYTES_PER_U16).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 1049 } 1050 else 1051 convert_char_codes_to_u16_chars(codes_data, str_data, Cast(codes_count).u64) 1052 StoreI(new_str, codes_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 1053 end 1054 # String is supposed to be a constant object, so all its data should be visible by all threads 1055 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 1056 Return(new_str).ptr 1057 1058 Label(:SlowPathEntrypoint) 1059 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_CHAR_CODE_SLOW_PATH") 1060 Intrinsic(:SLOW_PATH_ENTRY, char_codes).AddImm(entrypoint).MethodAsImm("CreateStringFromCharCode2ArgBridge").Terminator.ptr 1061 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 1062 } 1063end # def GenerateCreateStringFromCharCodeTlab 1064 1065GenerateCreateStringFromCharCodeTlab(string_compression_enabled=true) 1066GenerateCreateStringFromCharCodeTlab(string_compression_enabled=false) 1067 1068def GenerateCreateStringFromCharCodeSingleTlab(string_compression_enabled) 1069 suffix = (string_compression_enabled ? "Compressed" : "") 1070 available_regs = $panda_mask 1071 function("CreateStringFromCharCodeSingleTlab#{suffix}".to_sym, 1072 params: {char_code: 'u64', string_klass: 'ref'}, 1073 regmap: $full_regmap, 1074 regalloc_set: available_regs, 1075 mode: [:FastPath]) { 1076 1077 if Options.arch == :arm32 1078 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 1079 ReturnVoid().void 1080 next 1081 end 1082 1083 char_code_num := Bitcast(char_code).f64 1084 flag_char_pair := Intrinsic(:JS_CAST_DOUBLE_TO_CHAR, char_code_num).u32 1085 if !Options.jscvt_feature_enabled? 1086 # Validate if the char code is convertible: whenever an overflow or underflow takes place during the 1087 # conversion of a denormalized number we have to go to the slow path. The validation must take place before 1088 # any attempts to allocate the memory in TLAB. 1089 convertible := validate_flag_char_pair(flag_char_pair) 1090 If(convertible, 0).EQ.Unlikely { 1091 Goto(:SlowPathEntrypoint) 1092 } 1093 end 1094 1095 # Allocate a new string 1096 char := get_char_from_flag_char_pair(flag_char_pair) 1097 if string_compression_enabled 1098 If(SubI(char).Imm(Constants::STRING_MUTF8_1B_MIN).u16, Cast(Constants::STRING_MUTF8_1B_MAX).u16).AE.Unlikely { 1099 compressible1 := Cast(0).b 1100 data_size1 := Cast(2).word 1101 } Else { 1102 compressible2 := Cast(1).b 1103 data_size2 := Cast(1).word 1104 } 1105 compressible := Phi(compressible1, compressible2).b 1106 data_size := Phi(data_size1, data_size2).word 1107 else 1108 data_size := Cast(2).word 1109 end 1110 1111 new_str := allocate_string_tlab(string_klass, data_size) 1112 str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 1113 1114 # Copy the char to the new string with the required preprocessing 1115 # String length field is set according to SetLength() from runtime/include/coretypes/string.h 1116 if string_compression_enabled 1117 If(compressible, 1).EQ.Likely { 1118 Store(str_data, Cast(0).u64, char).u8 1119 StoreI(new_str, Cast(2).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 1120 } Else { 1121 Store(str_data, Cast(0).u64, char).u16 1122 StoreI(new_str, OrI(Cast(2).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 1123 } 1124 else 1125 Store(str_data, Cast(0).u64, char).u16 1126 StoreI(new_str, Cast(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 1127 end 1128 # String is supposed to be a constant object, so all its data should be visible by all threads 1129 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 1130 Return(new_str).ptr 1131 1132 Label(:SlowPathEntrypoint) 1133 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_CHAR_CODE_SINGLE_SLOW_PATH") 1134 Intrinsic(:SLOW_PATH_ENTRY, char_code).AddImm(entrypoint).MethodAsImm("CreateStringFromCharCodeSingle2ArgBridge").Terminator.ptr 1135 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 1136 } 1137end # def GenerateCreateStringFromCharCodeSingleTlab 1138 1139GenerateCreateStringFromCharCodeSingleTlab(string_compression_enabled=true) 1140GenerateCreateStringFromCharCodeSingleTlab(string_compression_enabled=false) 1141 1142# String contains 8-bit chars 1143# 0 < str_data_size < 8 1144function(:StringIndexOfCompressedSmall, 1145 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u8'}, 1146 regmap: $full_regmap, 1147 regalloc_set: $panda_mask, 1148 mode: [:FastPath]) { 1149 1150 if Options.arch == :arm32 || Options.arch == :x86_64 1151 Intrinsic(:UNREACHABLE).Terminator.void 1152 ReturnVoid().void 1153 next 1154 end 1155 i1 := 0 1156Label(:Loop) 1157 i := Phi(i1, i2).u32 1158 If(Load(str_data, i).u8, ch).EQ.Unlikely.b { 1159 Return(Cast(i).i32).i32 1160 } 1161 i2 := AddI(i).Imm(Constants::U8_SIZE).u32 1162 If(i2, str_data_size).LT.Likely.b { 1163 Goto(:Loop) 1164 } 1165 Return(-1).i32 1166} 1167 1168 1169# String contains 8-bit chars 1170# 8 <= str_data_size < 16 1171function(:StringIndexOfCompressedMedium, 1172 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u8'}, 1173 regmap: $full_regmap, 1174 regalloc_set: $panda_mask, 1175 mode: [:FastPath]) { 1176 1177 if Options.arch == :arm32 || Options.arch == :x86_64 1178 Intrinsic(:UNREACHABLE).Terminator.void 1179 ReturnVoid().void 1180 next 1181 end 1182 1183 pattern := Cast(ch).u64 1184 pattern := Or(ShlI(pattern).Imm("ark::BITS_PER_BYTE").u64, pattern).u64 1185 pattern := Or(ShlI(pattern).Imm("ark::BITS_PER_UINT16").u64, pattern).u64 1186 pattern := Or(ShlI(pattern).Imm("ark::BITS_PER_UINT32").u64, pattern).u64 1187 value := LoadI(str_data).Imm(0).u64 1188 x := Xor(value, pattern).u64 1189 found := AndI(And(SubI(x).Imm(Constants::XOR_SUB_U8_MASK).u64, Not(x).u64).u64).Imm(Constants::XOR_AND_U8_MASK).u64 1190 If(found, 0).NE.Likely.b { 1191 rev := Intrinsic(:REVERSE_BYTES_U64, found).u64 1192 pos := Cast(ShrI(Intrinsic(:COUNT_LEADING_ZERO_BITS_U64, rev).u64).Imm(Constants::LOG2_BITS_PER_U8).u64).i32 1193 Return(pos).i32 1194 } 1195 i1 := Constants::MEM_BLOCK_8_BYTES 1196Label(:Loop) 1197 i := Phi(i1, i2).u32 1198 If(i, str_data_size).LT.Likely.b { 1199 If(Load(str_data, i).u8, ch).EQ.Unlikely.b { 1200 Return(Cast(i).i32).i32 1201 } 1202 i2 := AddI(i).Imm(Constants::U8_SIZE).u32 1203 Goto(:Loop) 1204 } 1205 Return(-1).i32 1206} 1207 1208 1209# 16 <= str_data_size < 32 1210function(:StringIndexOfCompressedLarge, 1211 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u8'}, 1212 regmap: $full_regmap, 1213 regalloc_set: $panda_mask, 1214 mode: [:FastPath]) { 1215 1216 if Options.arch == :arm32 || Options.arch == :x86_64 1217 Intrinsic(:UNREACHABLE).Terminator.void 1218 ReturnVoid().void 1219 next 1220 end 1221 1222 # String contains 8-bit chars 1223 p := Intrinsic(:MEM_CHAR_U8_X16_USING_SIMD, ch, str_data).ptr 1224 If(p, 0).NE.Likely.b { 1225 Return(Cast(Sub(p, str_data).word).i32).i32 1226 } 1227 i1 := Constants::MEM_BLOCK_16_BYTES 1228Label(:Loop) 1229 i := Phi(i1, i2).u32 1230 If(i, str_data_size).LT.Likely.b { 1231 If(Load(str_data, i).u8, ch).EQ.Unlikely.b { 1232 Return(Cast(i).i32).i32 1233 } 1234 i2 := AddI(i).Imm(Constants::U8_SIZE).u32 1235 Goto(:Loop) 1236 } 1237 Return(-1).i32 1238} 1239 1240 1241# 32 <= data size 1242function(:StringIndexOfCompressed, 1243 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u8'}, 1244 regmap: $full_regmap, 1245 regalloc_set: $panda_mask, 1246 mode: [:FastPath]) { 1247 1248 if Options.arch == :arm32 || Options.arch == :x86_64 1249 Intrinsic(:UNREACHABLE).Terminator.void 1250 ReturnVoid().void 1251 next 1252 end 1253 1254 # String contains 8-bit chars 1255 end_ptr := Add(str_data, Cast(str_data_size).word).ptr 1256 1257 IfImm(Compare(str_data_size, Constants::MEM_BLOCK_32_BYTES).GE.b).Imm(0).NE.Likely.b { 1258 end_ptr_aligned := Bitcast(AndI(Bitcast(end_ptr).word).Imm(Constants::MEM_BLOCK_32_ALIGN_MASK).word).ptr 1259Label(:LoopSimd) 1260 curr_ptr := Phi(str_data, curr_ptr1).ptr; 1261 p := Intrinsic(:MEM_CHAR_U8_X32_USING_SIMD, ch, curr_ptr).ptr 1262 If(p, 0).NE.Likely.b { 1263 Return(Cast(Sub(p, str_data).word).i32).i32 1264 } 1265 curr_ptr1 := Add(curr_ptr, Constants::MEM_BLOCK_32_BYTES).ptr 1266 If(curr_ptr1, end_ptr_aligned).LT.Likely.b { 1267 Goto(:LoopSimd) 1268 } 1269 If(end_ptr_aligned, end_ptr).EQ.Unlikely.b { 1270 Return(-1).i32 1271 } 1272 } 1273 curr_ptr2 := Phi(str_data, curr_ptr1).ptr 1274 1275 IfImm(Compare(Sub(end_ptr, curr_ptr2).word, Constants::MEM_BLOCK_16_BYTES).GE.b).Imm(0).NE.Likely.b { 1276 p := Intrinsic(:MEM_CHAR_U8_X16_USING_SIMD, ch, curr_ptr2).ptr 1277 If(p, 0).NE.Likely.b { 1278 Return(Cast(Sub(p, str_data).word).i32).i32 1279 } 1280 curr_ptr3 := AddI(curr_ptr2).Imm(Constants::MEM_BLOCK_16_BYTES).ptr 1281 } 1282 curr_ptr4 := Phi(curr_ptr2, curr_ptr3).ptr 1283 1284Label(:Loop) 1285 curr_ptr5 := Phi(curr_ptr4, curr_ptr6).ptr 1286 If(curr_ptr5, end_ptr).LT.Likely.b { 1287 c := LoadI(curr_ptr5).Imm(0).u8 1288 If(c, ch).EQ.Unlikely.b { 1289 Return(Cast(Sub(curr_ptr5, str_data).word).i32).i32 1290 } 1291 curr_ptr6 := AddI(curr_ptr5).Imm(Constants::U8_SIZE).ptr 1292 Goto(:Loop) 1293 } 1294 Return(-1).i32 1295} 1296 1297# 0 < str_data_size < 16 1298function(:StringIndexOfUncompressedSmall, 1299 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u16'}, 1300 regmap: $full_regmap, 1301 regalloc_set: $panda_mask, 1302 mode: [:FastPath]) { 1303 1304 if Options.arch == :arm32 || Options.arch == :x86_64 1305 Intrinsic(:UNREACHABLE).Terminator.void 1306 ReturnVoid().void 1307 next 1308 end 1309 1310 # String contains 16-bit chars 1311 i1 := 0 1312Label(:Loop) 1313 i := Phi(i1, i2).u32 1314 If(Load(str_data, i).u16, ch).EQ.Unlikely.b { 1315 Return(Cast(ShrI(i).Imm(1).u32).i32).i32 1316 } 1317 i2 := AddI(i).Imm(Constants::U16_SIZE).u32 1318 If(i2, str_data_size).LT.Likely.b { 1319 Goto(:Loop) 1320 } 1321 Return(-1).i32 1322} 1323 1324 1325# 16 <= str_data_size < 32 1326function(:StringIndexOfUncompressedMedium, 1327 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u16'}, 1328 regmap: $full_regmap, 1329 regalloc_set: $panda_mask, 1330 mode: [:FastPath]) { 1331 1332 if Options.arch == :arm32 || Options.arch == :x86_64 1333 Intrinsic(:UNREACHABLE).Terminator.void 1334 ReturnVoid().void 1335 next 1336 end 1337 1338 # String contains 16-bit chars 1339 p := Intrinsic(:MEM_CHAR_U16_X8_USING_SIMD, ch, str_data).ptr 1340 If(p, 0).NE.Likely.b { 1341 Return(Cast(ShrI(Sub(p, str_data).word).Imm(1).word).i32).i32 1342 } 1343 i1 := Constants::MEM_BLOCK_16_BYTES # u16x8 1344Label(:Loop) 1345 i := Phi(i1, i2).u32 1346 If(i, str_data_size).LT.Likely.b { 1347 If(Load(str_data, i).u16, ch).EQ.Unlikely.b { 1348 Return(Cast(ShrI(i).Imm(1).u32).i32).i32 1349 } 1350 i2 := AddI(i).Imm(Constants::U16_SIZE).u32 1351 Goto(:Loop) 1352 } 1353 Return(-1).i32 1354} 1355 1356 1357# 32 <= data size 1358function(:StringIndexOfUncompressed, 1359 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u16'}, 1360 regmap: $full_regmap, 1361 regalloc_set: $panda_mask, 1362 mode: [:FastPath]) { 1363 1364 if Options.arch == :arm32 || Options.arch == :x86_64 1365 Intrinsic(:UNREACHABLE).Terminator.void 1366 ReturnVoid().void 1367 next 1368 end 1369 1370 # String contains 16-bit chars 1371 end_ptr := Add(str_data, Cast(str_data_size).word).ptr 1372 1373 IfImm(Compare(str_data_size, Constants::MEM_BLOCK_32_BYTES).GE.b).Imm(0).NE.Likely.b { 1374 end_ptr_aligned := Bitcast(AndI(Bitcast(end_ptr).word).Imm(Constants::MEM_BLOCK_32_ALIGN_MASK).word).ptr 1375Label(:LoopSimd) 1376 curr_ptr := Phi(str_data, curr_ptr1).ptr; 1377 p := Intrinsic(:MEM_CHAR_U16_X16_USING_SIMD, ch, curr_ptr).ptr 1378 If(p, 0).NE.Likely.b { 1379 Return(Cast(ShrI(Sub(p, str_data).word).Imm(1).word).i32).i32 1380 } 1381 curr_ptr1 := Add(curr_ptr, Constants::MEM_BLOCK_32_BYTES).ptr 1382 If(curr_ptr1, end_ptr_aligned).LT.Likely.b { 1383 Goto(:LoopSimd) 1384 } 1385 If(end_ptr_aligned, end_ptr).EQ.Unlikely.b { 1386 Return(-1).i32 1387 } 1388 } 1389 curr_ptr2 := Phi(str_data, curr_ptr1).ptr 1390 IfImm(Compare(Sub(end_ptr, curr_ptr2).word, Constants::MEM_BLOCK_16_BYTES).GE.b).Imm(0).NE.Likely.b { 1391 p := Intrinsic(:MEM_CHAR_U16_X8_USING_SIMD, ch, curr_ptr2).ptr 1392 If(p, 0).NE.Likely.b { 1393 Return(Cast(ShrI(Sub(p, str_data).word).Imm(1).word).i32).i32 1394 } 1395 curr_ptr3 := AddI(curr_ptr2).Imm(Constants::MEM_BLOCK_16_BYTES).ptr 1396 } 1397 curr_ptr4 := Phi(curr_ptr2, curr_ptr3).ptr 1398 1399Label(:Loop) 1400 curr_ptr5 := Phi(curr_ptr4, curr_ptr6).ptr 1401 If(curr_ptr5, end_ptr).LT.Likely.b { 1402 If(LoadI(curr_ptr5).Imm(0).u16, ch).EQ.Unlikely.b { 1403 Return(Cast(ShrI(Sub(curr_ptr5, str_data).word).Imm(1).word).i32).i32 1404 } 1405 curr_ptr6 := AddI(curr_ptr5).Imm(Constants::U16_SIZE).ptr 1406 Goto(:Loop) 1407 } 1408 Return(-1).i32 1409} 1410 1411 1412function(:StringIndexOf, 1413 params: {str: 'ref', ch: 'u16', fake: 'i32'}, 1414 regmap: $full_regmap, 1415 regalloc_set: $panda_regs, 1416 mode: [:FastPath]) { 1417 1418 if Options.arch == :arm32 || Options.arch == :x86_64 1419 Intrinsic(:UNREACHABLE).Terminator.void 1420 ReturnVoid().void 1421 next 1422 end 1423 1424 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 1425 1426 # Return '-1' if 'str' is empty. 1427 IfImm(Compare(str_len_packed, 1).LE.b).Imm(0).NE.Unlikely.b { 1428 Return(-1).i32 1429 } 1430 1431 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 1432 1433 IfImm(Compare(AndI(str_len_packed).Imm(1).i32, 1).EQ.b).Imm(0).NE.Unlikely.b { 1434 str_data_size_u16 := AndI(str_len_packed).Imm(0xFFFFFFFE).u32 1435 # 0 < data size < 16 1436 If(str_data_size_u16, 16).LT.Unlikely.b { 1437 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1438 LiveOut(str_data_size_u16).DstReg(regmap[:arg1]).u32 1439 LiveOut(ch).DstReg(regmap[:arg2]).u16 1440 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED_SMALL") 1441 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressedSmall").Terminator.i32 1442 } 1443 # 16 <= data size < 32 1444 If(str_data_size_u16, 32).LT.Unlikely.b { 1445 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1446 LiveOut(str_data_size_u16).DstReg(regmap[:arg1]).u32 1447 LiveOut(ch).DstReg(regmap[:arg2]).u16 1448 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED_MEDIUM") 1449 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressedMedium").Terminator.i32 1450 } 1451 # 32 <= data size 1452 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1453 LiveOut(str_data_size_u16).DstReg(regmap[:arg1]).u32 1454 LiveOut(ch).DstReg(regmap[:arg2]).u16 1455 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED") 1456 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressed").Terminator.i32 1457 } 1458 1459 IfImm(Compare(ch, Constants::MAX_U8_VALUE).A.b).Imm(0).NE.Unlikely.b { 1460 Return(-1).i32 1461 } 1462 1463 str_data_size_u8 := ShrI(str_len_packed).Imm(1).u32 1464 # 0 < data size < 8 1465 If(str_data_size_u8, 8).LT.Unlikely.b { 1466 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1467 LiveOut(str_data_size_u8).DstReg(regmap[:arg1]).u32 1468 LiveOut(ch).DstReg(regmap[:arg2]).u8 1469 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_SMALL") 1470 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedSmall").Terminator.i32 1471 } 1472 # 8 <= data size < 16 1473 If(str_data_size_u8, 16).LT.Unlikely.b { 1474 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1475 LiveOut(str_data_size_u8).DstReg(regmap[:arg1]).u32 1476 LiveOut(ch).DstReg(regmap[:arg2]).u8 1477 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_MEDIUM") 1478 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedMedium").Terminator.i32 1479 } 1480 # 16 <= data size < 32 1481 If(str_data_size_u8, 32).LT.Unlikely.b { 1482 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1483 LiveOut(str_data_size_u8).DstReg(regmap[:arg1]).u32 1484 LiveOut(ch).DstReg(regmap[:arg2]).u8 1485 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_LARGE") 1486 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedLarge").Terminator.i32 1487 } 1488 # 32 <= data size 1489 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1490 LiveOut(str_data_size_u8).DstReg(regmap[:arg1]).u32 1491 LiveOut(ch).DstReg(regmap[:arg2]).u8 1492 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED") 1493 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressed").Terminator.i32 1494} 1495 1496 1497# 'IndexOfAfter' calls the corresponding functions from the 'IndexOf' family 1498# for the purpose of code efficiency and to avoid code duplication. 1499# The problem is that we have to add 'start_index' to the result, but execution flow 1500# never returns to 'StringIndexOfAfter' because of TAIL_CALL. So the codegen and libllvm 1501# take care of it and emit instructions adding 'start_index' to the result returned 1502# by 'StringIndexOfAfter'. 1503function(:StringIndexOfAfter, 1504 params: {str: 'ref', ch: 'u16', start_index: 'i32'}, 1505 regmap: $full_regmap, 1506 regalloc_set: $panda_mask, 1507 mode: [:FastPath]) { 1508 1509 if Options.arch == :arm32 || Options.arch == :x86_64 1510 Intrinsic(:UNREACHABLE).Terminator.void 1511 ReturnVoid().void 1512 next 1513 end 1514 1515 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 1516 # Return '-1' if 'str' is empty. 1517 IfImm(Compare(str_len_packed, 1).LE.b).Imm(0).NE.Unlikely.b { 1518 Return(-1).i32 1519 } 1520 1521 str_len := ShrI(str_len_packed).Imm(1).i32 1522 1523 # Return '-1' if 'start_index' is out of range 1524 If(start_index, str_len).GE.Unlikely.b { 1525 Return(-1).i32 1526 } 1527 1528 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 1529 1530 IfImm(Compare(AndI(str_len_packed).Imm(1).i32, 1).EQ.b).Imm(0).NE.Unlikely.b { 1531 offs := Cast(ShlI(start_index).Imm(1).i32).u32 1532 data_size_u16 := AndI(str_len_packed).Imm(0xFFFFFFFE).u32 1533 data_size_u16 := Sub(data_size_u16, offs).u32 1534 data_ptr_u16 := Add(str_data, Cast(offs).word).ptr 1535 # 0 < data size < 16 1536 If(data_size_u16, 16).LT.Unlikely.b { 1537 LiveOut(data_ptr_u16).DstReg(regmap[:arg0]).ptr 1538 LiveOut(data_size_u16).DstReg(regmap[:arg1]).u32 1539 LiveOut(ch).DstReg(regmap[:arg2]).u16 1540 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED_SMALL") 1541 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressedSmall").Terminator.i32 1542 } 1543 # 16 <= data size < 32 1544 If(data_size_u16, 32).LT.Unlikely.b { 1545 LiveOut(data_ptr_u16).DstReg(regmap[:arg0]).ptr 1546 LiveOut(data_size_u16).DstReg(regmap[:arg1]).u32 1547 LiveOut(ch).DstReg(regmap[:arg2]).u16 1548 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED_MEDIUM") 1549 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressedMedium").Terminator.i32 1550 } 1551 # 32 <= data size 1552 LiveOut(data_ptr_u16).DstReg(regmap[:arg0]).ptr 1553 LiveOut(data_size_u16).DstReg(regmap[:arg1]).u32 1554 LiveOut(ch).DstReg(regmap[:arg2]).u16 1555 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED") 1556 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressed").Terminator.i32 1557 } 1558 1559 IfImm(Compare(ch, Constants::MAX_U8_VALUE).A.b).Imm(0).NE.Unlikely.b { 1560 # Return '-1' as 'str' is compressed and 'ch' does not fit in 8 bits. 1561 Return(-1).i32 1562 } 1563 1564 data_size_u8 := Sub(str_len, Cast(start_index).u32).u32 1565 data_ptr_u8 := Add(str_data, Cast(start_index).word).ptr 1566 # 0 < data size < 8 1567 If(data_size_u8, 8).LT.Unlikely.b { 1568 LiveOut(data_ptr_u8).DstReg(regmap[:arg0]).ptr 1569 LiveOut(data_size_u8).DstReg(regmap[:arg1]).u32 1570 LiveOut(ch).DstReg(regmap[:arg2]).u8 1571 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_SMALL") 1572 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedSmall").Terminator.i32 1573 } 1574 # 8 <= data size < 16 1575 If(data_size_u8, 16).LT.Unlikely.b { 1576 LiveOut(data_ptr_u8).DstReg(regmap[:arg0]).ptr 1577 LiveOut(data_size_u8).DstReg(regmap[:arg1]).u32 1578 LiveOut(ch).DstReg(regmap[:arg2]).u8 1579 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_MEDIUM") 1580 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedMedium").Terminator.i32 1581 } 1582 # 16 <= data size < 32 1583 If(data_size_u8, 32).LT.Unlikely.b { 1584 LiveOut(data_ptr_u8).DstReg(regmap[:arg0]).ptr 1585 LiveOut(data_size_u8).DstReg(regmap[:arg1]).u32 1586 LiveOut(ch).DstReg(regmap[:arg2]).u8 1587 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_LARGE") 1588 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedLarge").Terminator.i32 1589 } 1590 # 32 <= data size 1591 LiveOut(data_ptr_u8).DstReg(regmap[:arg0]).ptr 1592 LiveOut(data_size_u8).DstReg(regmap[:arg1]).u32 1593 LiveOut(ch).DstReg(regmap[:arg2]).u8 1594 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED") 1595 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressed").Terminator.i32 1596} 1597 1598function(:StringRepeatTlab, 1599 params: {str: 'ref', cnt: 'i32'}, 1600 regmap: $full_regmap, 1601 regalloc_set: $panda_mask, 1602 mode: [:FastPath]) { 1603 1604 if Options.arch == :arm32 1605 Intrinsic(:UNREACHABLE).Terminator.void 1606 next 1607 end 1608 1609 count := Cast(cnt).u32 1610 IfImm(Compare(count, 0).LT.b).Imm(0).NE { 1611 Goto(:SlowPathEntrypoint) 1612 } 1613 1614 IfImm(Compare(count, 0).EQ.b).Imm(0).NE { 1615 klass := load_class(str) 1616 new_str := allocate_string_tlab(klass, Cast(0).u64); 1617 Return(new_str).ptr 1618 } 1619 1620 IfImm(Compare(count, 1).EQ.b).Imm(0).NE { 1621 Return(Cast(str).ptr).ptr 1622 } 1623 1624 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32; 1625 klass := load_class(str) 1626 old_buf := AddI(str).Imm(Constants::STRING_DATA_OFFSET).ptr 1627 uncompressed := AndI(length).Imm(1).u32 1628 length := Shl(ShrI(length).Imm(1).u32, uncompressed).u32 1629 size := Mul(length, count).u32 1630 new_str := allocate_string_tlab(klass, Cast(size).u64); 1631 new_buf := AddI(new_str).Imm(Constants::STRING_DATA_OFFSET).ptr 1632 1633 If(new_str, 0).EQ { 1634 Goto(:SlowPathEntrypoint) 1635 } 1636 1637 i0 := Cast(0).u32 1638Label(:outer) 1639 i := Phi(i0, i1).u32 1640 If(i, count).GE { 1641 Goto(:End) 1642 } 1643 offset := Mul(i, length).u32 1644 1645 j0 := Cast(0).u32 1646 Label(:inner) 1647 j := Phi(j0, j1).u32 1648 If(j, length).GE { 1649 Goto(:endInner) 1650 } 1651 Store(new_buf, Add(offset, j).u32, Load(old_buf, j).u8).u8 1652 j1 := AddI(j).Imm(1).u32 1653 Goto(:inner) 1654 Label(:endInner) 1655 i1 := AddI(i).Imm(1).u32 1656 Goto(:outer) 1657 1658Label(:End) 1659 # shift left if it was a compressed string 1660 # mark the (unset) lowest bit with uncompressed 1661 compress := Neg(SubI(uncompressed).Imm(1).u32).u32 1662 length := Or(Shl(size, compress).u32, uncompressed).u32 1663 StoreI(new_str, length).Imm(Constants::STRING_LENGTH_OFFSET).u32 1664 Return(new_str).ptr 1665 1666Label(:SlowPathEntrypoint) 1667 entrypoint = get_entrypoint_offset("STRING_REPEAT_SLOW_PATH") 1668 Intrinsic(:SLOW_PATH_ENTRY, str, count).AddImm(entrypoint).MethodAsImm("StringRepeatUsualBridge").Terminator.ptr 1669 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 1670} 1671 1672function(:WriteStringToMem, 1673 params: {mem: 'i64', str: 'ref'}, 1674 regmap: $full_regmap, 1675 regalloc_set: $panda_mask, 1676 mode: [:FastPath]) { 1677 1678 if Options.arch == :arm32 1679 Intrinsic(:UNREACHABLE).Terminator.void 1680 next 1681 end 1682 1683 buf := Bitcast(mem).ptr 1684 len := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 1685 utf16 := AndI(len).Imm(1).u32 1686 len_0 := ShrI(len).Imm(1).u32 1687 len := len_0 1688 1689 If(len, 0).EQ.Unlikely.b { 1690 Goto(:End) 1691 } 1692 1693 If(utf16, 1).EQ.Unlikely.b { 1694 mark := 0xfeff 1695 StoreI(buf, mark).Imm(0).u16 1696 len_1 := ShlI(len).Imm(1).u32 1697 buf_1 := AddI(buf).Imm(2).ptr 1698 } 1699 buf := Phi(buf, buf_1).ptr 1700 len := Phi(len, len_1).u32 1701 1702 str_data := AddI(str).Imm(Constants::STRING_DATA_OFFSET).ptr 1703 copy_u8_chars(str_data, buf, len); 1704 1705 If(utf16, 1).EQ.Unlikely.b { 1706 len_1 := AddI(len).Imm(2).u32 1707 } 1708 1709Label(:End) 1710 len := Phi(len_0, len, len_1).u32 1711 Return(len).u32 1712} 1713 1714function(:CreateStringFromMem, 1715 params: {buf: 'i64', len: 'i32', klass: 'ref'}, 1716 regmap: $full_regmap, 1717 regalloc_set: $panda_mask, 1718 mode: [:FastPath]) { 1719 1720 if Options.arch == :arm32 1721 Intrinsic(:UNREACHABLE).Terminator.void 1722 next 1723 end 1724 1725 addr := Bitcast(buf).ptr 1726 mark := LoadI(addr).Imm(0).u16 1727 1728 If(mark, Cast(0xFEFF).u16).EQ.Unlikely.b { # UTF-16 string 1729 size_1 := SubI(len).Imm(2).i32 1730 addr_1 := AddI(addr).Imm(2).ptr 1731 } 1732 1733 size := Phi(len, size_1).i32 1734 addr := Phi(addr, addr_1).ptr 1735 1736 str := allocate_string_tlab_no_debug(klass, size) 1737 str_data := AddI(str).Imm(Constants::STRING_DATA_OFFSET).ptr 1738 copy_u8_chars(addr, str_data, size); 1739 size_0 := ShlI(size).Imm(1).u32 1740 1741 If(mark, Cast(0xFEFF).u16).EQ.Unlikely.b { # UTF-16 string 1742 size_1 := OrI(ShrI(size_0).Imm(1).u32).Imm(1).u32 1743 } 1744 1745 size := Phi(size_0, size_1).u32 1746 str_len := AddI(str).Imm(Constants::STRING_LENGTH_OFFSET).ptr 1747 StoreI(str_len, size).Imm(0).u32 1748 1749 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 1750 Return(str).ptr 1751 1752 Label(:SlowPathEntrypoint) 1753 eid = get_entrypoint_offset("CREATE_STRING_FROM_MEM_SLOW_PATH") 1754 Intrinsic(:SLOW_PATH_ENTRY, buf, len).AddImm(eid).MethodAsImm("CreateStringFromMem3ArgBridge").Terminator.ptr 1755 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 1756} 1757 1758