1# plugin ets_string 2# Copyright (c) 2024 Huawei Device Co., Ltd. 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15include_relative 'string_helpers.irt' 16 17module Constants 18 MEM_BLOCK_8_BYTES = "8" 19 MEM_BLOCK_16_BYTES = "16" 20 MEM_BLOCK_32_BYTES = "32" 21 MEM_BLOCK_32_ALIGN_MASK = "~31ULL" 22 LOG2_BITS_PER_U8 = "3" 23 LOG2_BITS_PER_U16 = "4" 24 XOR_SUB_U8_MASK = "0x0101010101010101ULL" 25 XOR_AND_U8_MASK = "0x8080808080808080ULL" 26 MAX_U8_VALUE = "255" 27 U8_SIZE = "1" 28 U16_SIZE = "2" 29end 30 31# It is assumed that _begin_index and _end_index are safe and does not check/normalize them. 32# The range is [_begin_index, _end_index). 33# Note, a caller of this macro must provide a corresponding 'SlowPathEntrypoint' 34# for the case when 'allocate_string_tlab' fails (see StringTrim as an example) 35# Now TLAB implementation initializes memory with zero, so the hashcode field 36# is not initialized with zero explicitly. 37macro(:fast_substring) do |_str, _str_len, _begin_index, _end_index, _not_compressed| 38 _char_count := Sub(_end_index, _begin_index).u32 39 If(_char_count, Cast(_str_len).u32).EQ.Unlikely.b { 40 # Return the string itself 41 _same_str := Cast(_str).SrcType(Constants::COMPILER_REFERENCE).ptr 42 Goto(:_Fast_Substring_Result_No_Barrier) 43 } 44 _klass := LoadI(_str).Imm(Constants::OBJECT_CLASS_OFFSET).ref 45 If(_char_count, 0).EQ.Unlikely.b { 46 # Allocate and return an empty string 47 _empty_str := allocate_string_tlab(_klass, 0) 48 Goto(:_Fast_Substring_Result) 49 } 50 # Allocate a new normal string 51 _offset := Shl(_begin_index, _not_compressed).u32 52 _src_str_data := Add(Cast(_str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 53 _src_str_data := Add(_src_str_data, Cast(_offset).u64).ptr 54 If(_not_compressed, 1).EQ.Unlikely.b { 55 _compressable := is_array_of_compressable_chars(_src_str_data, Cast(_char_count).u64) 56 If(_compressable, 1).EQ.Likely.b { 57 _data_size1 := Cast(_char_count).word 58 Goto(:_L1) 59 } 60 _data_size2 := Cast(ShlI(_char_count).Imm(1).u32).word 61Label(:_L1) 62 _data_size := Phi(_data_size1, _data_size2).word 63 _new_str1 := allocate_string_tlab(_klass, _data_size) 64 _new_str_data := Add(_new_str1, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 65 If(_compressable, 1).EQ.Likely.b { 66 compress_u16_to_u8_chars(_src_str_data, _new_str_data, Cast(_char_count).u64) 67 StoreI(_new_str1, ShlI(_char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 68 Goto(:_Fast_Substring_Result) 69 } 70 copy_u16_chars(_src_str_data, _new_str_data, Cast(_char_count).u64) 71 StoreI(_new_str1, OrI(ShlI(_char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 72 Goto(:_Fast_Substring_Result) 73 } 74 # Source string is already compressed 75 _new_str2 := allocate_string_tlab(_klass, Cast(_char_count).word) 76 _new_str_data2 := Add(_new_str2, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 77 copy_u8_chars(_src_str_data, _new_str_data2, Cast(_char_count).u64) 78 StoreI(_new_str2, ShlI(_char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 79Label(:_Fast_Substring_Result) 80 # String is supposed to be a constant object, so all its data should be visible by all threads 81 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 82 _substring := Phi(_empty_str, _new_str1, _new_str1, _new_str2).ptr 83Label(:_Fast_Substring_Result_No_Barrier) 84 _result := Phi(_same_str, _substring).ptr 85end 86 87 88# 89# Test if u16 char is a white space 90# 91scoped_macro(:is_white_space_u16) do |ch| 92 IfImm(Compare(ch, 0x0020).EQ.b).Imm(0).NE.b { 93 Goto(:LabelWhiteSpaceChar) 94 } 95 # 0x000E..0x009F -- common non-whitespace chars 96 IfImm(Compare(ch, 0x000E).AE.b).Imm(0).NE.b { 97 IfImm(Compare(ch, 0x00A0).B.b).Imm(0).NE.b { 98 Goto(:LabelNotWhiteSpaceChar) 99 } 100 } 101 # 0x0009 -- horizontal tab 102 # 0x000A -- line feed or new line 103 # 0x000B -- vertical tab 104 # 0x000C -- formfeed 105 # 0x000D -- carriage return 106 IfImm(Compare(ch, 0x0009).B.b).Imm(0).NE.b { 107 Goto(:LabelNotWhiteSpaceChar) 108 } 109 IfImm(Compare(ch, 0x000D).BE.b).Imm(0).NE.b { 110 Goto(:LabelWhiteSpaceChar) 111 } 112 # 0x00A0 -- non-breaking space 113 IfImm(Compare(ch, 0x00A0).EQ.b).Imm(0).NE.b { 114 Goto(:LabelWhiteSpaceChar) 115 } 116 # 0x1680 -- Ogham space mark 117 If(ch, 0x1680).EQ.Unlikely.b { 118 Goto(:LabelWhiteSpaceChar) 119 } 120 # 0x2000 -- en quad 121 # 0x2001 -- em quad 122 # 0x2002 -- en space 123 # 0x2003 -- em space 124 # 0x2004 -- three-per-em space 125 # 0x2005 -- four-per-em space 126 # 0x2006 -- six-per-em space 127 # 0x2007 -- figure space 128 # 0x2008 -- punctuation space 129 # 0x2009 -- thin space 130 # 0x200A -- hair space 131 If(ch, 0x2000).B.Unlikely.b { 132 Goto(:LabelNotWhiteSpaceChar) 133 } 134 If(ch, 0x200A).BE.Unlikely.b { 135 Goto(:LabelWhiteSpaceChar) 136 } 137 # 0x2028 -- line separator 138 If(ch, 0x2028).EQ.Unlikely.b { 139 Goto(:LabelWhiteSpaceChar) 140 } 141 # 0x2029 -- paragraph separator 142 If(ch, 0x2029).EQ.Unlikely.b { 143 Goto(:LabelWhiteSpaceChar) 144 } 145 # 0x202F -- narrow no-break space 146 If(ch, 0x202F).EQ.Unlikely.b { 147 Goto(:LabelWhiteSpaceChar) 148 } 149 # 0x205F -- medium mathematical space 150 If(ch, 0x205F).EQ.Unlikely.b { 151 Goto(:LabelWhiteSpaceChar) 152 } 153 # 0xFEFF -- byte order mark 154 If(ch, 0xFEFF).EQ.Unlikely.b { 155 Goto(:LabelWhiteSpaceChar) 156 } 157 # 0x3000 -- ideographic space 158 If(ch, 0x3000).EQ.Unlikely.b { 159 Goto(:LabelWhiteSpaceChar) 160 } 161Label(:LabelNotWhiteSpaceChar) 162 whiteSpace0 := 0 163 Goto(:LabelReturn) 164Label(:LabelWhiteSpaceChar) 165 whiteSpace1 := 1 166Label(:LabelReturn) 167 result := Phi(whiteSpace0, whiteSpace1).b 168end 169 170# 171# Test if u8 char is a white space 172# 173scoped_macro(:is_white_space_u8) do |ch| 174 IfImm(Compare(ch, 0x20).EQ.b).Imm(0).NE.b { 175 Goto(:LabelWhiteSpaceChar) 176 } 177 # 0x0E..0x9F -- common non-whitespace chars 178 IfImm(Compare(ch, 0x0E).AE.b).Imm(0).NE.b { 179 IfImm(Compare(ch, 0xA0).B.b).Imm(0).NE.b { 180 Goto(:LabelNotWhiteSpaceChar) 181 } 182 } 183 # 0x09 -- horizontal tab 184 # 0x0A -- line feed or new line 185 # 0x0B -- vertical tab 186 # 0x0C -- formfeed 187 # 0x0D -- carriage return 188 IfImm(Compare(ch, 0x09).B.b).Imm(0).NE.b { 189 Goto(:LabelNotWhiteSpaceChar) 190 } 191 IfImm(Compare(ch, 0x0D).BE.b).Imm(0).NE.b { 192 Goto(:LabelWhiteSpaceChar) 193 } 194 # 0xA0 -- non-breaking space 195 IfImm(Compare(ch, 0xA0).EQ.b).Imm(0).NE.b { 196 Goto(:LabelWhiteSpaceChar) 197 } 198Label(:LabelNotWhiteSpaceChar) 199 whiteSpace0 := 0 200 Goto(:LabelReturn) 201Label(:LabelWhiteSpaceChar) 202 whiteSpace1 := 1 203Label(:LabelReturn) 204 result := Phi(whiteSpace0, whiteSpace1).b 205end 206 207 208function(:CharIsWhiteSpace, 209 params: {ch: 'u16'}, 210 regmap: $full_regmap, 211 regalloc_set: $panda_mask, 212 mode: [:FastPath]) { 213 214 if Options.arch == :arm32 215 Intrinsic(:UNREACHABLE).Terminator.void 216 ReturnVoid().void 217 next 218 end 219 Return(is_white_space_u16(ch)).b 220} 221 222 223function(:StringEmpty, 224 params: {str: 'ref'}, 225 regmap: $full_regmap, 226 regalloc_set: $panda_mask, 227 mode: [:FastPath]) { 228 229 if Options.arch == :arm32 230 Intrinsic(:UNREACHABLE).Terminator.void 231 ReturnVoid().void 232 next 233 end 234 235 klass := LoadI(str).Imm(Constants::OBJECT_CLASS_OFFSET).ref 236 empty_str := allocate_string_tlab(klass, 0) 237 # String is supposed to be a constant object, so all its data should be visible by all threads 238 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 239 Return(empty_str).ptr 240Label(:SlowPathEntrypoint) 241 entrypoint = get_entrypoint_offset("CREATE_EMPTY_STRING_SLOW_PATH") 242 Intrinsic(:SLOW_PATH_ENTRY).AddImm(entrypoint).MethodAsImm("CreateEmptyString1ArgBridge").Terminator.ptr 243 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 244} 245 246 247if Options.arch == :arm64 248 trim_left_regs = $temps_mask + :callee0 + :callee1 249 trim_right_regs = $temps_mask + :callee0 + :callee1 250else 251 trim_left_regs = $temps_mask + :callee0 + :caller0 + :caller1 252 trim_right_regs = $temps_mask + :callee0 + :caller0 + :caller1 253end 254 255 256function(:StringTrimLeftBase, 257 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 258 regmap: $full_regmap, 259 regalloc_set: $panda_mask, 260 mode: [:FastPath]) { 261 262 if Options.arch == :arm32 263 Intrinsic(:UNREACHABLE).Terminator.void 264 ReturnVoid().void 265 next 266 end 267 268 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 269 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 270 not_compressed := AndI(length_packed).Imm(1).i32 271 length := ShrI(length_packed).Imm(1).i32 272 start_index := Cast(1).i32 273 If(not_compressed, 0).EQ.Likely.b { 274 # String contains 8-bit chars 275Label(:Loop1) 276 i := Phi(start_index, i1).i32 277 ws1 := is_white_space_u8(Load(str_data, i).u8) 278 If(ws1, 0).NE.Likely.b { 279 i1 := AddI(i).Imm(1).i32 280 If(i1, length).LT.Likely.b { 281 Goto(:Loop1) 282 } 283 } 284 index1 := Phi(i, i1).i32 285 Goto(:TrimLeft) 286 } 287 # String contains 16-bit chars 288Label(:Loop2) 289 j := Phi(start_index, j1).i32 290 ws2 := is_white_space_u16(Load(str_data, ShlI(j).Imm(1).i32).u16) 291 If(ws2, 0).NE.Likely.b { 292 j1 := AddI(j).Imm(1).i32 293 If(j1, length).LT.Likely.b { 294 Goto(:Loop2) 295 } 296 } 297 index2 := Phi(j, j1).i32 298Label(:TrimLeft) 299 index := Phi(index1, index2).i32 300 trimmed := fast_substring(str, length, index, length, not_compressed) 301 Return(trimmed).ptr 302Label(:SlowPathEntrypoint) 303 entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH") 304 Intrinsic(:SLOW_PATH_ENTRY, str, index, length).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr 305 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 306} 307 308 309function(:StringTrimLeft, 310 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 311 regmap: $full_regmap, 312 regalloc_set: $trim_left_regs, 313 mode: [:FastPath]) { 314 315 if Options.arch == :arm32 316 Intrinsic(:UNREACHABLE).Terminator.void 317 ReturnVoid().void 318 next 319 end 320 321 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 322 If(length_packed, 1).LE.Unlikely.b { 323 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 324 } 325 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 326 not_compressed := AndI(length_packed).Imm(1).i32 327 If(not_compressed, 0).EQ.Likely.b { 328 ws1 := is_white_space_u8(Load(str_data, 0).u8) 329 Goto(:L1) 330 } 331 ws2 := is_white_space_u16(Load(str_data, 0).u16) 332Label(:L1) 333 ws := Phi(ws1, ws2).b 334 If(ws, 0).EQ.Likely.b { 335 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 336 } 337 If(ShrI(length_packed).Imm(1).i32, 1).EQ.Unlikely.b { 338 LiveOut(str).DstReg(regmap[:arg0]).ref 339 entrypoint1 = get_entrypoint_offset("STRING_EMPTY") 340 Intrinsic(:TAIL_CALL).AddImm(entrypoint1).MethodAsImm("StringEmpty").Terminator.ptr 341 } 342 LiveOut(str).DstReg(regmap[:arg0]).ref 343 LiveOut(unused1).DstReg(regmap[:arg1]).i32 344 LiveOut(unused2).DstReg(regmap[:arg2]).i32 345 entrypoint2 = get_entrypoint_offset("STRING_TRIM_LEFT_BASE") 346 Intrinsic(:TAIL_CALL).AddImm(entrypoint2).MethodAsImm("StringTrimLeftBase").Terminator.ptr 347} 348 349 350function(:StringTrimRightBase, 351 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 352 regmap: $full_regmap, 353 regalloc_set: $panda_mask, 354 mode: [:FastPath]) { 355 356 if Options.arch == :arm32 357 Intrinsic(:UNREACHABLE).Terminator.void 358 ReturnVoid().void 359 next 360 end 361 362 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 363 length := ShrI(length_packed).Imm(1).i32 364 start_index := SubI(length).Imm(2).i32 365 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 366 not_compressed := AndI(length_packed).Imm(1).i32 367 If(not_compressed, 0).EQ.Likely.b { 368 # String contains 8-bit chars 369Label(:Loop1) 370 i := Phi(start_index, i1).i32 371 ws1 := is_white_space_u8(Load(str_data, i).u8) 372 If(ws1, 0).NE.Likely.b { 373 i1 := SubI(i).Imm(1).i32 374 If(i1, 0).GE.Likely.b { 375 Goto(:Loop1) 376 } 377 } 378 index1 := Phi(i, i1).i32 379 Goto(:TrimRight) 380 } 381 # String contains 16-bit chars 382Label(:Loop2) 383 j := Phi(start_index, j1).i32 384 ws2 := is_white_space_u16(Load(str_data, ShlI(j).Imm(1).i32).u16) 385 If(ws2, 0).NE.Likely.b { 386 j1 := SubI(j).Imm(1).i32 387 If(j1, 0).GE.Likely.b { 388 Goto(:Loop2) 389 } 390 } 391 index2 := Phi(j, j1).i32 392Label(:TrimRight) 393 index := Phi(index1, index2).i32 394 index := AddI(index).Imm(1).i32 395 trimmed := fast_substring(str, length, 0, index, not_compressed) 396 Return(trimmed).ptr 397Label(:SlowPathEntrypoint) 398 entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH") 399 Intrinsic(:SLOW_PATH_ENTRY, str, Cast(0).i32, index).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr 400 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 401} 402 403 404function(:StringTrimRight, 405 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 406 regmap: $full_regmap, 407 regalloc_set: $trim_right_regs, 408 mode: [:FastPath]) { 409 410 if Options.arch == :arm32 411 Intrinsic(:UNREACHABLE).Terminator.void 412 ReturnVoid().void 413 next 414 end 415 416 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 417 If(length_packed, 1).LE.Unlikely.b { 418 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 419 } 420 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 421 not_compressed := AndI(length_packed).Imm(1).i32 422 length := ShrI(length_packed).Imm(1).i32 423 last_char_index := SubI(length).Imm(1).i32 424 If(not_compressed, 0).EQ.Likely.b { 425 ws1 := is_white_space_u8(Load(str_data, last_char_index).u8) 426 Goto(:L1) 427 } 428 ws2 := is_white_space_u16(Load(str_data, ShlI(last_char_index).Imm(1).i32).u16) 429Label(:L1) 430 ws := Phi(ws1, ws2).b 431 If(ws, 0).EQ.Likely.b { 432 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 433 } 434 If(length, 1).EQ.Unlikely.b { 435 LiveOut(str).DstReg(regmap[:arg0]).ref 436 entrypoint1 = get_entrypoint_offset("STRING_EMPTY") 437 Intrinsic(:TAIL_CALL).AddImm(entrypoint1).MethodAsImm("StringEmpty").Terminator.ptr 438 } 439 LiveOut(str).DstReg(regmap[:arg0]).ref 440 LiveOut(unused1).DstReg(regmap[:arg1]).i32 441 LiveOut(unused2).DstReg(regmap[:arg2]).i32 442 entrypoint2 = get_entrypoint_offset("STRING_TRIM_RIGHT_BASE") 443 Intrinsic(:TAIL_CALL).AddImm(entrypoint2).MethodAsImm("StringTrimRightBase").Terminator.ptr 444} 445 446 447function(:StringTrimBase, 448 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 449 regmap: $full_regmap, 450 regalloc_set: $panda_mask, 451 mode: [:FastPath]) { 452 453 if Options.arch == :arm32 454 Intrinsic(:UNREACHABLE).Terminator.void 455 ReturnVoid().void 456 next 457 end 458 459 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 460 length := ShrI(length_packed).Imm(1).i32 461 left := 0 462 right := SubI(length).Imm(2).i32 463 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 464 not_compressed := AndI(length_packed).Imm(1).i32 465 If(not_compressed, 0).EQ.Likely.b { 466 # String contains 8-bit chars 467Label(:Loop1) # while (utf::IsWhiteSpaceChar(str->At(right))) 468 right1 := Phi(right, right2).i32 469 If(is_white_space_u8(Load(str_data, right1).u8), 0).NE.Likely.b { 470 If(right1, 0).EQ.Unlikely.b { 471 Goto(:Trim) 472 } 473 right2 := SubI(right1).Imm(1).i32 474 Goto(:Loop1) 475 } 476Label(:Loop2) # while (left < right && utf::IsWhiteSpaceChar(str->At(left))) 477 left1 := Phi(left, left2).i32 478 If(left1, right1).LT.Unlikely.b { 479 If(is_white_space_u8(Load(str_data, left1).u8), 0).NE.Likely.b { 480 left2 := AddI(left1).Imm(1).i32 481 Goto(:Loop2) 482 } 483 } 484 right3 := AddI(right1).Imm(1).i32 485 Goto(:Trim) 486 } 487 # String contains 16-bit chars 488Label(:Loop3) # while (utf::IsWhiteSpaceChar(str->At(right))) 489 right11 := Phi(right, right22).i32 490 If(is_white_space_u16(Load(str_data, ShlI(right11).Imm(1).i32).u16), 0).NE.Likely.b { 491 If(right11, 0).EQ.Unlikely.b { 492 Goto(:Trim) 493 } 494 right22 := SubI(right11).Imm(1).i32 495 Goto(:Loop3) 496 } 497Label(:Loop4) # while (left < right && utf::IsWhiteSpaceChar(str->At(left))) 498 left11 := Phi(left, left22).i32 499 If(left11, right11).LT.Unlikely.b { 500 If(is_white_space_u16(Load(str_data, ShlI(left11).Imm(1).i32).u16), 0).NE.Likely.b { 501 left22 := AddI(left11).Imm(1).i32 502 Goto(:Loop4) 503 } 504 } 505 right33 := AddI(right11).Imm(1).i32 506Label(:Trim) 507 l := Phi(left, left1, left, left11).i32 508 r := Phi(right1, right3, right11, right33).i32 509 trimmed := fast_substring(str, length, l, r, not_compressed) 510 Return(trimmed).ptr 511Label(:SlowPathEntrypoint) 512 entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH") 513 Intrinsic(:SLOW_PATH_ENTRY, str, l, r).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr 514 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 515} 516 517 518function(:StringTrim, 519 params: {str: 'ref', unused1: 'i32', unused2: 'i32'}, 520 regmap: $full_regmap, 521 regalloc_set: $panda_mask, 522 mode: [:FastPath]) { 523 524 if Options.arch == :arm32 525 Intrinsic(:UNREACHABLE).Terminator.void 526 ReturnVoid().void 527 next 528 end 529 530 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 531 # length == 0 532 If(length_packed, 1).LE.Unlikely.b { 533 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 534 } 535 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 536 not_compressed := AndI(length_packed).Imm(1).i32 537 length := ShrI(length_packed).Imm(1).i32 538 # length == 1 539 If(length, 1).EQ.b { 540 If(not_compressed, 0).EQ.Likely.b { 541 ws1 := is_white_space_u8(Load(str_data, 0).u8) 542 Goto(:L1) 543 } 544 ws2 := is_white_space_u16(Load(str_data, 0).u16) 545Label(:L1) 546 ws3 := Phi(ws1, ws2).b 547 If(ws3, 0).EQ.Likely.b { 548 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 549 } 550 LiveOut(str).DstReg(regmap[:arg0]).ref 551 entrypoint1 = get_entrypoint_offset("STRING_EMPTY") 552 Intrinsic(:TAIL_CALL).AddImm(entrypoint1).MethodAsImm("StringEmpty").Terminator.ptr 553 } 554 # length > 1 555 last_char_index := SubI(length).Imm(1).i32 556 If(not_compressed, 0).EQ.Likely.b { 557 ws4 := is_white_space_u8(Load(str_data, last_char_index).u8) 558 Goto(:L2) 559 } 560 ws5 := is_white_space_u16(Load(str_data, ShlI(last_char_index).Imm(1).i32).u16) 561Label(:L2) 562 ws6 := Phi(ws4, ws5).b 563 If(ws6, 0).EQ.Likely.b { 564 # last char is not whitespace, so check the first char 565 If(not_compressed, 0).EQ.Likely.b { 566 ws7 := is_white_space_u8(Load(str_data, 0).u8) 567 Goto(:L3) 568 } 569 ws8 := is_white_space_u16(Load(str_data, 0).u16) 570Label(:L3) 571 ws9 := Phi(ws7, ws8).b 572 If(ws9, 0).EQ.Likely.b { 573 # first char is not white space, so return 'str' 574 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 575 } 576 Goto(:FirstCharWhitespace) 577 } 578 # last char is whitespace, so call StringTrimBase 579 LiveOut(str).DstReg(regmap[:arg0]).ref 580 LiveOut(unused1).DstReg(regmap[:arg1]).i32 581 LiveOut(unused2).DstReg(regmap[:arg2]).i32 582 entrypoint2 = get_entrypoint_offset("STRING_TRIM_BASE") 583 Intrinsic(:TAIL_CALL).AddImm(entrypoint2).MethodAsImm("StringTrimBase").Terminator.ptr 584Label(:FirstCharWhitespace) 585 LiveOut(str).DstReg(regmap[:arg0]).ref 586 LiveOut(unused1).DstReg(regmap[:arg1]).i32 587 LiveOut(unused2).DstReg(regmap[:arg2]).i32 588 entrypoint3 = get_entrypoint_offset("STRING_TRIM_LEFT_BASE") 589 Intrinsic(:TAIL_CALL).AddImm(entrypoint3).MethodAsImm("StringTrimLeftBase").Terminator.ptr 590} 591 592 593scoped_macro(:at) do |str_data, index, not_compressed| 594 If(not_compressed, 0).EQ.Likely.b { 595 # String contains 8-bit chars 596 c8 := Cast(Load(str_data, index).u8).u16 597 Goto(:Done) 598 } 599 # String contains 16-bit chars 600 c16 := Load(str_data, ShlI(index).Imm(1).i32).u16 601Label(:Done) 602 c := Phi(c8, c16).u16 603end 604 605 606function(:StringStartsWithBase, 607 params: {str: 'ref', pfx: 'ref', from_index: 'i32'}, 608 regmap: $full_regmap, 609 regalloc_set: $panda_mask, 610 mode: [:FastPath]) { 611 612 if Options.arch == :arm32 613 Intrinsic(:UNREACHABLE).Terminator.void 614 ReturnVoid().void 615 next 616 end 617 618 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 619 str_not_compressed := AndI(str_len_packed).Imm(1).i32 620 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 621 pfx_len_packed := LoadI(pfx).Imm(Constants::STRING_LENGTH_OFFSET).u32 622 pfx_len := ShrI(pfx_len_packed).Imm(1).i32 623 pfx_not_compressed := AndI(pfx_len_packed).Imm(1).i32 624 pfx_data := Add(Cast(pfx).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 625 626 pfx_i1 := 0 627Label(:Loop) 628 str_i := Phi(from_index, str_i1).i32 629 pfx_i := Phi(pfx_i1, pfx_i2).i32 630 If(pfx_i, pfx_len).GE.Unlikely.b { 631 Goto(:Done) 632 } 633 s := at(str_data, str_i, str_not_compressed) 634 p := at(pfx_data, pfx_i, pfx_not_compressed) 635 If(s, p).NE.Likely.b { 636 Return(0).b 637 } 638 pfx_i2 := AddI(pfx_i).Imm(1).i32 639 str_i1 := AddI(str_i).Imm(1).i32 640 Goto(:Loop) 641Label(:Done) 642 Return(1).b 643} 644 645 646function(:StringStartsWith, 647 params: {str: 'ref', pfx: 'ref', from_index: 'i32'}, 648 regmap: $full_regmap, 649 regalloc_set: $panda_mask, 650 mode: [:FastPath]) { 651 652 if Options.arch == :arm32 653 Intrinsic(:UNREACHABLE).Terminator.void 654 ReturnVoid().void 655 next 656 end 657 658 pfx_len_packed := LoadI(pfx).Imm(Constants::STRING_LENGTH_OFFSET).u32 659 # Return 'true' if prefix is empty 660 # The least significant bit indicates COMPRESSED/UNCOMPRESSED, 661 # thus if (packed length <= 1) then the actual length is equal to 0. 662 If(pfx_len_packed, 1).LE.Unlikely.b { 663 Return(1).b 664 } 665 666 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 667 # Return 'false' if 'str' is empty as 'prefix' is not empty. 668 If(str_len_packed, 1).LE.Unlikely.b { 669 Return(0).b 670 } 671 672 # If 'from_index' is less than zero then make it zero. 673 IfImm(Compare(from_index, 0).LT.b).Imm(0).NE.Unlikely.b { 674 from_index1 := Cast(0).i32 675 } 676 from_index2 := Phi(from_index, from_index1).i32 677 678 str_len := ShrI(str_len_packed).Imm(1).i32 679 pfx_len := ShrI(pfx_len_packed).Imm(1).i32 680 681 If(from_index2, Sub(str_len, pfx_len).i32).GT.Unlikely.b { 682 # Return 'false' in this case, as we know that 'pfx' is not empty 683 # and it is longer than the part of 'str' to be checked. 684 Return(0).b 685 } 686 687 LiveOut(str).DstReg(regmap[:arg0]).ref 688 LiveOut(pfx).DstReg(regmap[:arg1]).ref 689 LiveOut(from_index2).DstReg(regmap[:arg2]).i32 690 entrypoint = get_entrypoint_offset("STRING_STARTS_WITH_BASE") 691 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringStartsWithBase").Terminator.b 692} 693 694 695function(:StringEndsWithBase, 696 params: {str: 'ref', sfx: 'ref', end_index: 'i32'}, 697 regmap: $full_regmap, 698 regalloc_set: $panda_mask, 699 mode: [:FastPath]) { 700 701 if Options.arch == :arm32 702 Intrinsic(:UNREACHABLE).Terminator.void 703 ReturnVoid().void 704 next 705 end 706 707 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 708 str_not_compressed := AndI(str_len_packed).Imm(1).i32 709 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 710 sfx_len_packed := LoadI(sfx).Imm(Constants::STRING_LENGTH_OFFSET).u32 711 sfx_not_compressed := AndI(sfx_len_packed).Imm(1).i32 712 sfx_data := Add(Cast(sfx).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 713 sfx_len := ShrI(sfx_len_packed).Imm(1).i32 714 from_index := Sub(end_index, sfx_len).i32; 715 716 sfx_i1 := 0 717Label(:Loop) 718 str_i := Phi(from_index, str_i1).i32 719 sfx_i := Phi(sfx_i1, sfx_i2).i32 720 If(sfx_i, sfx_len).GE.Unlikely.b { 721 Goto(:Done) 722 } 723 s := at(str_data, str_i, str_not_compressed) 724 p := at(sfx_data, sfx_i, sfx_not_compressed) 725 If(s, p).NE.Likely.b { 726 Return(0).b 727 } 728 sfx_i2 := AddI(sfx_i).Imm(1).i32 729 str_i1 := AddI(str_i).Imm(1).i32 730 Goto(:Loop) 731 732Label(:Done) 733 Return(1).b 734} 735 736 737function(:StringEndsWith, 738 params: {str: 'ref', sfx: 'ref', end_index: 'i32'}, 739 regmap: $full_regmap, 740 regalloc_set: $panda_mask, 741 mode: [:FastPath]) { 742 743 if Options.arch == :arm32 744 Intrinsic(:UNREACHABLE).Terminator.void 745 ReturnVoid().void 746 next 747 end 748 749 sfx_len_packed := LoadI(sfx).Imm(Constants::STRING_LENGTH_OFFSET).u32 750 # Return 'true' if suffix is empty 751 # The least significant bit indicates COMPRESSED/UNCOMPRESSED, 752 # thus if (packed length <= 1) then the actual length is equal to 0. 753 If(sfx_len_packed, 1).LE.Unlikely.b { 754 Return(1).b 755 } 756 757 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 758 # Return 'false' if 'str' is empty as 'suffix' is not empty. 759 If(str_len_packed, 1).LE.Unlikely.b { 760 Return(0).b 761 } 762 # If 'end_index' is less or equal to zero then return false. 763 IfImm(Compare(end_index, 0).LE.b).Imm(0).NE.Unlikely.b { 764 Return(0).b 765 } 766 767 str_len := ShrI(str_len_packed).Imm(1).i32 768 # If 'end_index' is greater than length of 'str' make it equal to length of 'str'. 769 If(end_index, str_len).GT.Unlikely.b { 770 end_index1 := str_len 771 } 772 end_index2 := Phi(end_index, end_index1).i32 773 774 sfx_len := ShrI(sfx_len_packed).Imm(1).i32 775 from_index := Sub(end_index2, sfx_len).i32; 776 IfImm(Compare(from_index, 0).LT.b).Imm(0).NE.Unlikely.b { 777 # Return 'false' in this case, as 'sfx' length is greater than 'end_index'. 778 Return(0).b 779 } 780 781 LiveOut(str).DstReg(regmap[:arg0]).ref 782 LiveOut(sfx).DstReg(regmap[:arg1]).ref 783 LiveOut(end_index2).DstReg(regmap[:arg2]).i32 784 entrypoint = get_entrypoint_offset("STRING_ENDS_WITH_BASE") 785 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringEndsWithBase").Terminator.b 786} 787 788function(:StringGetBytesTlab, 789 params: {str: 'ref', begin_index: 'i32', end_index: 'i32', array_klass: 'ref'}, 790 regmap: $full_regmap, 791 regalloc_set: $panda_mask, 792 mode: [:FastPath]) { 793 794 if Options.arch == :arm32 795 Intrinsic(:UNREACHABLE).Terminator.void 796 ReturnVoid().void 797 next 798 end 799 800 If(begin_index, end_index).GT.Unlikely.b { 801 Goto(:SlowPathEntrypoint) # Out of range 802 } 803 804 If(begin_index, Cast(0).i32).LT.Unlikely.b { 805 Goto(:SlowPathEntrypoint) # Out of range 806 } 807 808 # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded) 809 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32; 810 uncompressed := AndI(length).Imm(1).u32; 811 length := ShrI(length).Imm(1).u32; 812 813 If(Cast(end_index).u32, length).A.Unlikely.b { 814 Goto(:SlowPathEntrypoint) # Out of range 815 } 816 offset := Shl(begin_index, uncompressed).u32 817 818 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 819 src_str_data := Add(src_str_data, Cast(offset).u64).ptr 820 821# Allocate a new array of u8 (bytes) 822 count := Sub(Cast(end_index).u32, Cast(begin_index).u32).u64 823 new_arr := allocate_array_of_bytes_tlab(array_klass, Cast(count).word) 824 new_arr_data := Add(new_arr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr 825 If(uncompressed, Cast(0).u32).EQ.Likely.b { 826 copy_u8_chars(src_str_data, new_arr_data, count) 827 Goto(:End) 828 } 829 compress_u16_to_u8_chars(src_str_data, new_arr_data, count) 830 831 Label(:End) 832 Return(new_arr).ptr 833 834 Label(:SlowPathEntrypoint) 835 entrypoint = get_entrypoint_offset("STRING_GET_BYTES_SLOW_PATH") 836 Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("StringGetBytes4ArgBridge").Terminator.ptr 837 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 838} 839 840 841# String contains 8-bit chars 842# 0 < str_data_size < 8 843function(:StringIndexOfCompressedSmall, 844 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u8'}, 845 regmap: $full_regmap, 846 regalloc_set: $panda_mask, 847 mode: [:FastPath]) { 848 849 if Options.arch == :arm32 || Options.arch == :x86_64 850 Intrinsic(:UNREACHABLE).Terminator.void 851 ReturnVoid().void 852 next 853 end 854 i1 := 0 855Label(:Loop) 856 i := Phi(i1, i2).u32 857 If(Load(str_data, i).u8, ch).EQ.Unlikely.b { 858 Return(Cast(i).i32).i32 859 } 860 i2 := AddI(i).Imm(Constants::U8_SIZE).u32 861 If(i2, str_data_size).LT.Likely.b { 862 Goto(:Loop) 863 } 864 Return(-1).i32 865} 866 867 868# String contains 8-bit chars 869# 8 <= str_data_size < 16 870function(:StringIndexOfCompressedMedium, 871 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u8'}, 872 regmap: $full_regmap, 873 regalloc_set: $panda_mask, 874 mode: [:FastPath]) { 875 876 if Options.arch == :arm32 || Options.arch == :x86_64 877 Intrinsic(:UNREACHABLE).Terminator.void 878 ReturnVoid().void 879 next 880 end 881 882 pattern := Cast(ch).u64 883 pattern := Or(ShlI(pattern).Imm("ark::BITS_PER_BYTE").u64, pattern).u64 884 pattern := Or(ShlI(pattern).Imm("ark::BITS_PER_UINT16").u64, pattern).u64 885 pattern := Or(ShlI(pattern).Imm("ark::BITS_PER_UINT32").u64, pattern).u64 886 value := LoadI(str_data).Imm(0).u64 887 x := Xor(value, pattern).u64 888 found := AndI(And(SubI(x).Imm(Constants::XOR_SUB_U8_MASK).u64, Not(x).u64).u64).Imm(Constants::XOR_AND_U8_MASK).u64 889 If(found, 0).NE.Likely.b { 890 rev := Intrinsic(:REVERSE_BYTES_U64, found).u64 891 pos := Cast(ShrI(Intrinsic(:COUNT_LEADING_ZERO_BITS_U64, rev).u64).Imm(Constants::LOG2_BITS_PER_U8).u64).i32 892 Return(pos).i32 893 } 894 i1 := Constants::MEM_BLOCK_8_BYTES 895Label(:Loop) 896 i := Phi(i1, i2).u32 897 If(i, str_data_size).LT.Likely.b { 898 If(Load(str_data, i).u8, ch).EQ.Unlikely.b { 899 Return(Cast(i).i32).i32 900 } 901 i2 := AddI(i).Imm(Constants::U8_SIZE).u32 902 Goto(:Loop) 903 } 904 Return(-1).i32 905} 906 907 908# 16 <= str_data_size < 32 909function(:StringIndexOfCompressedLarge, 910 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u8'}, 911 regmap: $full_regmap, 912 regalloc_set: $panda_mask, 913 mode: [:FastPath]) { 914 915 if Options.arch == :arm32 || Options.arch == :x86_64 916 Intrinsic(:UNREACHABLE).Terminator.void 917 ReturnVoid().void 918 next 919 end 920 921 # String contains 8-bit chars 922 p := Intrinsic(:MEM_CHAR_U8_X16_USING_SIMD, ch, str_data).ptr 923 If(p, 0).NE.Likely.b { 924 Return(Cast(Sub(p, str_data).word).i32).i32 925 } 926 i1 := Constants::MEM_BLOCK_16_BYTES 927Label(:Loop) 928 i := Phi(i1, i2).u32 929 If(i, str_data_size).LT.Likely.b { 930 If(Load(str_data, i).u8, ch).EQ.Unlikely.b { 931 Return(Cast(i).i32).i32 932 } 933 i2 := AddI(i).Imm(Constants::U8_SIZE).u32 934 Goto(:Loop) 935 } 936 Return(-1).i32 937} 938 939 940# 32 <= data size 941function(:StringIndexOfCompressed, 942 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u8'}, 943 regmap: $full_regmap, 944 regalloc_set: $panda_mask, 945 mode: [:FastPath]) { 946 947 if Options.arch == :arm32 || Options.arch == :x86_64 948 Intrinsic(:UNREACHABLE).Terminator.void 949 ReturnVoid().void 950 next 951 end 952 953 # String contains 8-bit chars 954 end_ptr := Add(str_data, Cast(str_data_size).word).ptr 955 956 IfImm(Compare(str_data_size, Constants::MEM_BLOCK_32_BYTES).GE.b).Imm(0).NE.Likely.b { 957 end_ptr_aligned := Bitcast(AndI(Bitcast(end_ptr).word).Imm(Constants::MEM_BLOCK_32_ALIGN_MASK).word).ptr 958Label(:LoopSimd) 959 curr_ptr := Phi(str_data, curr_ptr1).ptr; 960 p := Intrinsic(:MEM_CHAR_U8_X32_USING_SIMD, ch, curr_ptr).ptr 961 If(p, 0).NE.Likely.b { 962 Return(Cast(Sub(p, str_data).word).i32).i32 963 } 964 curr_ptr1 := Add(curr_ptr, Constants::MEM_BLOCK_32_BYTES).ptr 965 If(curr_ptr1, end_ptr_aligned).LT.Likely.b { 966 Goto(:LoopSimd) 967 } 968 If(end_ptr_aligned, end_ptr).EQ.Unlikely.b { 969 Return(-1).i32 970 } 971 } 972 curr_ptr2 := Phi(str_data, curr_ptr1).ptr 973 974 IfImm(Compare(Sub(end_ptr, curr_ptr2).word, Constants::MEM_BLOCK_16_BYTES).GE.b).Imm(0).NE.Likely.b { 975 p := Intrinsic(:MEM_CHAR_U8_X16_USING_SIMD, ch, curr_ptr2).ptr 976 If(p, 0).NE.Likely.b { 977 Return(Cast(Sub(p, str_data).word).i32).i32 978 } 979 curr_ptr3 := AddI(curr_ptr2).Imm(Constants::MEM_BLOCK_16_BYTES).ptr 980 } 981 curr_ptr4 := Phi(curr_ptr2, curr_ptr3).ptr 982 983Label(:Loop) 984 curr_ptr5 := Phi(curr_ptr4, curr_ptr6).ptr 985 If(curr_ptr5, end_ptr).LT.Likely.b { 986 c := LoadI(curr_ptr5).Imm(0).u8 987 If(c, ch).EQ.Unlikely.b { 988 Return(Cast(Sub(curr_ptr5, str_data).word).i32).i32 989 } 990 curr_ptr6 := AddI(curr_ptr5).Imm(Constants::U8_SIZE).ptr 991 Goto(:Loop) 992 } 993 Return(-1).i32 994} 995 996# 0 < str_data_size < 16 997function(:StringIndexOfUncompressedSmall, 998 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u16'}, 999 regmap: $full_regmap, 1000 regalloc_set: $panda_mask, 1001 mode: [:FastPath]) { 1002 1003 if Options.arch == :arm32 || Options.arch == :x86_64 1004 Intrinsic(:UNREACHABLE).Terminator.void 1005 ReturnVoid().void 1006 next 1007 end 1008 1009 # String contains 16-bit chars 1010 i1 := 0 1011Label(:Loop) 1012 i := Phi(i1, i2).u32 1013 If(Load(str_data, i).u16, ch).EQ.Unlikely.b { 1014 Return(Cast(ShrI(i).Imm(1).u32).i32).i32 1015 } 1016 i2 := AddI(i).Imm(Constants::U16_SIZE).u32 1017 If(i2, str_data_size).LT.Likely.b { 1018 Goto(:Loop) 1019 } 1020 Return(-1).i32 1021} 1022 1023 1024# 16 <= str_data_size < 32 1025function(:StringIndexOfUncompressedMedium, 1026 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u16'}, 1027 regmap: $full_regmap, 1028 regalloc_set: $panda_mask, 1029 mode: [:FastPath]) { 1030 1031 if Options.arch == :arm32 || Options.arch == :x86_64 1032 Intrinsic(:UNREACHABLE).Terminator.void 1033 ReturnVoid().void 1034 next 1035 end 1036 1037 # String contains 16-bit chars 1038 p := Intrinsic(:MEM_CHAR_U16_X8_USING_SIMD, ch, str_data).ptr 1039 If(p, 0).NE.Likely.b { 1040 Return(Cast(ShrI(Sub(p, str_data).word).Imm(1).word).i32).i32 1041 } 1042 i1 := Constants::MEM_BLOCK_16_BYTES # u16x8 1043Label(:Loop) 1044 i := Phi(i1, i2).u32 1045 If(i, str_data_size).LT.Likely.b { 1046 If(Load(str_data, i).u16, ch).EQ.Unlikely.b { 1047 Return(Cast(ShrI(i).Imm(1).u32).i32).i32 1048 } 1049 i2 := AddI(i).Imm(Constants::U16_SIZE).u32 1050 Goto(:Loop) 1051 } 1052 Return(-1).i32 1053} 1054 1055 1056# 32 <= data size 1057function(:StringIndexOfUncompressed, 1058 params: {str_data: 'ptr', str_data_size: 'u32', ch: 'u16'}, 1059 regmap: $full_regmap, 1060 regalloc_set: $panda_mask, 1061 mode: [:FastPath]) { 1062 1063 if Options.arch == :arm32 || Options.arch == :x86_64 1064 Intrinsic(:UNREACHABLE).Terminator.void 1065 ReturnVoid().void 1066 next 1067 end 1068 1069 # String contains 16-bit chars 1070 end_ptr := Add(str_data, Cast(str_data_size).word).ptr 1071 1072 IfImm(Compare(str_data_size, Constants::MEM_BLOCK_32_BYTES).GE.b).Imm(0).NE.Likely.b { 1073 end_ptr_aligned := Bitcast(AndI(Bitcast(end_ptr).word).Imm(Constants::MEM_BLOCK_32_ALIGN_MASK).word).ptr 1074Label(:LoopSimd) 1075 curr_ptr := Phi(str_data, curr_ptr1).ptr; 1076 p := Intrinsic(:MEM_CHAR_U16_X16_USING_SIMD, ch, curr_ptr).ptr 1077 If(p, 0).NE.Likely.b { 1078 Return(Cast(ShrI(Sub(p, str_data).word).Imm(1).word).i32).i32 1079 } 1080 curr_ptr1 := Add(curr_ptr, Constants::MEM_BLOCK_32_BYTES).ptr 1081 If(curr_ptr1, end_ptr_aligned).LT.Likely.b { 1082 Goto(:LoopSimd) 1083 } 1084 If(end_ptr_aligned, end_ptr).EQ.Unlikely.b { 1085 Return(-1).i32 1086 } 1087 } 1088 curr_ptr2 := Phi(str_data, curr_ptr1).ptr 1089 IfImm(Compare(Sub(end_ptr, curr_ptr2).word, Constants::MEM_BLOCK_16_BYTES).GE.b).Imm(0).NE.Likely.b { 1090 p := Intrinsic(:MEM_CHAR_U16_X8_USING_SIMD, ch, curr_ptr2).ptr 1091 If(p, 0).NE.Likely.b { 1092 Return(Cast(ShrI(Sub(p, str_data).word).Imm(1).word).i32).i32 1093 } 1094 curr_ptr3 := AddI(curr_ptr2).Imm(Constants::MEM_BLOCK_16_BYTES).ptr 1095 } 1096 curr_ptr4 := Phi(curr_ptr2, curr_ptr3).ptr 1097 1098Label(:Loop) 1099 curr_ptr5 := Phi(curr_ptr4, curr_ptr6).ptr 1100 If(curr_ptr5, end_ptr).LT.Likely.b { 1101 If(LoadI(curr_ptr5).Imm(0).u16, ch).EQ.Unlikely.b { 1102 Return(Cast(ShrI(Sub(curr_ptr5, str_data).word).Imm(1).word).i32).i32 1103 } 1104 curr_ptr6 := AddI(curr_ptr5).Imm(Constants::U16_SIZE).ptr 1105 Goto(:Loop) 1106 } 1107 Return(-1).i32 1108} 1109 1110 1111function(:StringIndexOf, 1112 params: {str: 'ref', ch: 'u16', fake: 'i32'}, 1113 regmap: $full_regmap, 1114 regalloc_set: $panda_regs, 1115 mode: [:FastPath]) { 1116 1117 if Options.arch == :arm32 || Options.arch == :x86_64 1118 Intrinsic(:UNREACHABLE).Terminator.void 1119 ReturnVoid().void 1120 next 1121 end 1122 1123 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 1124 1125 # Return '-1' if 'str' is empty. 1126 IfImm(Compare(str_len_packed, 1).LE.b).Imm(0).NE.Unlikely.b { 1127 Return(-1).i32 1128 } 1129 1130 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 1131 1132 IfImm(Compare(AndI(str_len_packed).Imm(1).i32, 1).EQ.b).Imm(0).NE.Unlikely.b { 1133 str_data_size_u16 := AndI(str_len_packed).Imm(0xFFFFFFFE).u32 1134 # 0 < data size < 16 1135 If(str_data_size_u16, 16).LT.Unlikely.b { 1136 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1137 LiveOut(str_data_size_u16).DstReg(regmap[:arg1]).u32 1138 LiveOut(ch).DstReg(regmap[:arg2]).u16 1139 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED_SMALL") 1140 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressedSmall").Terminator.i32 1141 } 1142 # 16 <= data size < 32 1143 If(str_data_size_u16, 32).LT.Unlikely.b { 1144 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1145 LiveOut(str_data_size_u16).DstReg(regmap[:arg1]).u32 1146 LiveOut(ch).DstReg(regmap[:arg2]).u16 1147 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED_MEDIUM") 1148 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressedMedium").Terminator.i32 1149 } 1150 # 32 <= data size 1151 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1152 LiveOut(str_data_size_u16).DstReg(regmap[:arg1]).u32 1153 LiveOut(ch).DstReg(regmap[:arg2]).u16 1154 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED") 1155 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressed").Terminator.i32 1156 } 1157 1158 IfImm(Compare(ch, Constants::MAX_U8_VALUE).A.b).Imm(0).NE.Unlikely.b { 1159 Return(-1).i32 1160 } 1161 1162 str_data_size_u8 := ShrI(str_len_packed).Imm(1).u32 1163 # 0 < data size < 8 1164 If(str_data_size_u8, 8).LT.Unlikely.b { 1165 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1166 LiveOut(str_data_size_u8).DstReg(regmap[:arg1]).u32 1167 LiveOut(ch).DstReg(regmap[:arg2]).u8 1168 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_SMALL") 1169 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedSmall").Terminator.i32 1170 } 1171 # 8 <= data size < 16 1172 If(str_data_size_u8, 16).LT.Unlikely.b { 1173 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1174 LiveOut(str_data_size_u8).DstReg(regmap[:arg1]).u32 1175 LiveOut(ch).DstReg(regmap[:arg2]).u8 1176 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_MEDIUM") 1177 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedMedium").Terminator.i32 1178 } 1179 # 16 <= data size < 32 1180 If(str_data_size_u8, 32).LT.Unlikely.b { 1181 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1182 LiveOut(str_data_size_u8).DstReg(regmap[:arg1]).u32 1183 LiveOut(ch).DstReg(regmap[:arg2]).u8 1184 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_LARGE") 1185 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedLarge").Terminator.i32 1186 } 1187 # 32 <= data size 1188 LiveOut(str_data).DstReg(regmap[:arg0]).ptr 1189 LiveOut(str_data_size_u8).DstReg(regmap[:arg1]).u32 1190 LiveOut(ch).DstReg(regmap[:arg2]).u8 1191 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED") 1192 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressed").Terminator.i32 1193} 1194 1195 1196# 'IndexOfAfter' calls the corresponding functions from the 'IndexOf' family 1197# for the purpose of code efficiency and to avoid code duplication. 1198# The problem is that we have to add 'start_index' to the result, but execution flow 1199# never returns to 'StringIndexOfAfter' because of TAIL_CALL. So the codegen and libllvm 1200# take care of it and emit instructions adding 'start_index' to the result returned 1201# by 'StringIndexOfAfter'. 1202function(:StringIndexOfAfter, 1203 params: {str: 'ref', ch: 'u16', start_index: 'i32'}, 1204 regmap: $full_regmap, 1205 regalloc_set: $panda_mask, 1206 mode: [:FastPath]) { 1207 1208 if Options.arch == :arm32 || Options.arch == :x86_64 1209 Intrinsic(:UNREACHABLE).Terminator.void 1210 ReturnVoid().void 1211 next 1212 end 1213 1214 str_len_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 1215 # Return '-1' if 'str' is empty. 1216 IfImm(Compare(str_len_packed, 1).LE.b).Imm(0).NE.Unlikely.b { 1217 Return(-1).i32 1218 } 1219 1220 str_len := ShrI(str_len_packed).Imm(1).i32 1221 1222 # Return '-1' if 'start_index' is out of range 1223 If(start_index, str_len).GE.Unlikely.b { 1224 Return(-1).i32 1225 } 1226 1227 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 1228 1229 IfImm(Compare(AndI(str_len_packed).Imm(1).i32, 1).EQ.b).Imm(0).NE.Unlikely.b { 1230 offs := Cast(ShlI(start_index).Imm(1).i32).u32 1231 data_size_u16 := AndI(str_len_packed).Imm(0xFFFFFFFE).u32 1232 data_size_u16 := Sub(data_size_u16, offs).u32 1233 data_ptr_u16 := Add(str_data, Cast(offs).word).ptr 1234 # 0 < data size < 16 1235 If(data_size_u16, 16).LT.Unlikely.b { 1236 LiveOut(data_ptr_u16).DstReg(regmap[:arg0]).ptr 1237 LiveOut(data_size_u16).DstReg(regmap[:arg1]).u32 1238 LiveOut(ch).DstReg(regmap[:arg2]).u16 1239 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED_SMALL") 1240 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressedSmall").Terminator.i32 1241 } 1242 # 16 <= data size < 32 1243 If(data_size_u16, 32).LT.Unlikely.b { 1244 LiveOut(data_ptr_u16).DstReg(regmap[:arg0]).ptr 1245 LiveOut(data_size_u16).DstReg(regmap[:arg1]).u32 1246 LiveOut(ch).DstReg(regmap[:arg2]).u16 1247 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED_MEDIUM") 1248 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressedMedium").Terminator.i32 1249 } 1250 # 32 <= data size 1251 LiveOut(data_ptr_u16).DstReg(regmap[:arg0]).ptr 1252 LiveOut(data_size_u16).DstReg(regmap[:arg1]).u32 1253 LiveOut(ch).DstReg(regmap[:arg2]).u16 1254 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_UNCOMPRESSED") 1255 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfUncompressed").Terminator.i32 1256 } 1257 1258 IfImm(Compare(ch, Constants::MAX_U8_VALUE).A.b).Imm(0).NE.Unlikely.b { 1259 # Return '-1' as 'str' is compressed and 'ch' does not fit in 8 bits. 1260 Return(-1).i32 1261 } 1262 1263 data_size_u8 := Sub(str_len, Cast(start_index).u32).u32 1264 data_ptr_u8 := Add(str_data, Cast(start_index).word).ptr 1265 # 0 < data size < 8 1266 If(data_size_u8, 8).LT.Unlikely.b { 1267 LiveOut(data_ptr_u8).DstReg(regmap[:arg0]).ptr 1268 LiveOut(data_size_u8).DstReg(regmap[:arg1]).u32 1269 LiveOut(ch).DstReg(regmap[:arg2]).u8 1270 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_SMALL") 1271 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedSmall").Terminator.i32 1272 } 1273 # 8 <= data size < 16 1274 If(data_size_u8, 16).LT.Unlikely.b { 1275 LiveOut(data_ptr_u8).DstReg(regmap[:arg0]).ptr 1276 LiveOut(data_size_u8).DstReg(regmap[:arg1]).u32 1277 LiveOut(ch).DstReg(regmap[:arg2]).u8 1278 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_MEDIUM") 1279 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedMedium").Terminator.i32 1280 } 1281 # 16 <= data size < 32 1282 If(data_size_u8, 32).LT.Unlikely.b { 1283 LiveOut(data_ptr_u8).DstReg(regmap[:arg0]).ptr 1284 LiveOut(data_size_u8).DstReg(regmap[:arg1]).u32 1285 LiveOut(ch).DstReg(regmap[:arg2]).u8 1286 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED_LARGE") 1287 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressedLarge").Terminator.i32 1288 } 1289 # 32 <= data size 1290 LiveOut(data_ptr_u8).DstReg(regmap[:arg0]).ptr 1291 LiveOut(data_size_u8).DstReg(regmap[:arg1]).u32 1292 LiveOut(ch).DstReg(regmap[:arg2]).u8 1293 entrypoint = get_entrypoint_offset("STRING_INDEX_OF_COMPRESSED") 1294 Intrinsic(:TAIL_CALL).AddImm(entrypoint).MethodAsImm("StringIndexOfCompressed").Terminator.i32 1295} 1296 1297function(:StringRepeatTlab, 1298 params: {str: 'ref', cnt: 'i32'}, 1299 regmap: $full_regmap, 1300 regalloc_set: $panda_mask, 1301 mode: [:FastPath]) { 1302 1303 if Options.arch == :arm32 1304 Intrinsic(:UNREACHABLE).Terminator.void 1305 next 1306 end 1307 1308 count := Cast(cnt).u32 1309 IfImm(Compare(count, 0).LT.b).Imm(0).NE { 1310 Goto(:SlowPathEntrypoint) 1311 } 1312 1313 IfImm(Compare(count, 0).EQ.b).Imm(0).NE { 1314 klass := LoadI(str).Imm(Constants::OBJECT_CLASS_OFFSET).ref 1315 new_str := allocate_string_tlab(klass, Cast(0).u64); 1316 Return(new_str).ptr 1317 } 1318 1319 IfImm(Compare(count, 1).EQ.b).Imm(0).NE { 1320 Return(Cast(str).ptr).ptr 1321 } 1322 1323 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32; 1324 klass := LoadI(str).Imm(Constants::OBJECT_CLASS_OFFSET).ref 1325 old_buf := AddI(str).Imm(Constants::STRING_DATA_OFFSET).ptr 1326 uncompressed := AndI(length).Imm(1).u32 1327 length := Shl(ShrI(length).Imm(1).u32, uncompressed).u32 1328 size := Mul(length, count).u32 1329 new_str := allocate_string_tlab(klass, Cast(size).u64); 1330 new_buf := AddI(new_str).Imm(Constants::STRING_DATA_OFFSET).ptr 1331 1332 If(new_str, 0).EQ { 1333 Goto(:SlowPathEntrypoint) 1334 } 1335 1336 i0 := Cast(0).u32 1337Label(:outer) 1338 i := Phi(i0, i1).u32 1339 If(i, count).GE { 1340 Goto(:End) 1341 } 1342 offset := Mul(i, length).u32 1343 1344 j0 := Cast(0).u32 1345 Label(:inner) 1346 j := Phi(j0, j1).u32 1347 If(j, length).GE { 1348 Goto(:endInner) 1349 } 1350 Store(new_buf, Add(offset, j).u32, Load(old_buf, j).u8).u8 1351 j1 := AddI(j).Imm(1).u32 1352 Goto(:inner) 1353 Label(:endInner) 1354 i1 := AddI(i).Imm(1).u32 1355 Goto(:outer) 1356 1357Label(:End) 1358 # shift left if it was a compressed string 1359 # mark the (unset) lowest bit with uncompressed 1360 compress := Neg(SubI(uncompressed).Imm(1).u32).u32 1361 length := Or(Shl(size, compress).u32, uncompressed).u32 1362 StoreI(new_str, length).Imm(Constants::STRING_LENGTH_OFFSET).u32 1363 Return(new_str).ptr 1364 1365Label(:SlowPathEntrypoint) 1366 entrypoint = get_entrypoint_offset("STRING_REPEAT_SLOW_PATH") 1367 Intrinsic(:SLOW_PATH_ENTRY, str, count).AddImm(entrypoint).MethodAsImm("StringRepeatUsualBridge").Terminator.ptr 1368 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 1369} 1370