1#!/usr/bin/env ruby 2 3# Copyright (c) 2021-2025 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16macro(:load_diff) do |ptr1, ptr2, offset| 17 Xor(LoadI(ptr1).Imm(Constants::STRING_DATA_OFFSET + " + " + offset.to_s).u64, 18 LoadI(ptr2).Imm(Constants::STRING_DATA_OFFSET + " + " + offset.to_s).u64).u64 19end 20 21scoped_macro(:unpack_length) do |length, compression, length_shift| 22 if compression 23 char_length := ShrI(length).Imm(length_shift).u64 24 not_compressed := AndI(length).Imm(1).u64 25 unpacked_length := Shl(char_length, not_compressed).u64 26 elsif length_shift != 0 27 unpacked_length := ShrI(length).Imm(length_shift).u64 28 end 29end 30 31def GenerateStringEquals(lang, dynamic, compression) 32 suffix = (compression ? "Compressed" : "") 33 length_shift = (dynamic ? 2 : 1) 34 mode = [:FastPath] 35 mode.push(:DynamicMethod, :DynamicStub) if dynamic 36 37 reg_mask = Options.arch == :arm64 ? 38 RegMask.new($full_regmap, :arg0, :arg1, :callee0, :callee1, :callee2, :callee3, :callee4, :callee5, :callee6, :callee7, :tmp0, :tmp1) : 39 $panda_mask + :tmp0 + :tmp1 + :tr 40 41 function("#{lang}StringEqualsUnroll#{suffix}".to_sym, 42 params: {str1_ref: 'ref', str2_ref: 'ref'}, 43 regmap: $full_regmap, 44 regalloc_set: reg_mask, 45 mode: mode, 46 lang: lang.empty? ? 'PANDA_ASSEMBLY' : lang.upcase) { 47 if Options.arch == :arm32 48 Intrinsic(:UNREACHABLE).void.Terminator 49 next 50 end 51 str1 := Cast(str1_ref).SrcType(Constants::COMPILER_REFERENCE).ptr 52 str2 := Cast(str2_ref).SrcType(Constants::COMPILER_REFERENCE).ptr 53 # Compare first 32 bytes 54 buf1 := LoadI(str1).Imm(Constants::STRING_DATA_OFFSET).u64 55 buf2 := LoadI(str2).Imm(Constants::STRING_DATA_OFFSET).u64 56 If(buf1, buf2).NE.Unlikely.b { 57 Goto(:NotEqual) 58 } 59 diff := load_diff(str1, str2, 8) 60 diff := Or(diff, load_diff(str1, str2, 16)).u64 61 diff := Or(diff, load_diff(str1, str2, 24)).u64 62 If(diff, 0).NE.Unlikely.b { 63 Goto(:NotEqual) 64 } 65 length := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32 66 length := Cast(length).SrcType("DataType::UINT32").u64 67 length := unpack_length(length, compression, length_shift) 68 last_buf_index := SubI(length).Imm(32).u64 69 last_ptr1 := Add(str1, last_buf_index).ptr 70 last_ptr2 := Add(str2, last_buf_index).ptr 71 first_ptr1 := AddI(str1).Imm(32).ptr 72 first_ptr2 := AddI(str2).Imm(32).ptr 73 Label(:Loop) 74 ptr1 := Phi(first_ptr1, next_ptr1).ptr 75 ptr2 := Phi(first_ptr2, next_ptr2).ptr 76 diff := load_diff(ptr1, ptr2, 0) 77 diff := Or(diff, load_diff(ptr1, ptr2, 8)).u64 78 diff := Or(diff, load_diff(ptr1, ptr2, 16)).u64 79 diff := Or(diff, load_diff(ptr1, ptr2, 24)).u64 80 If(diff, 0).NE.Unlikely.b { 81 Goto(:NotEqual) 82 } 83 next_ptr1 := Add(ptr1, 32).ptr 84 If(next_ptr1, last_ptr1).GE.Unlikely.b { 85 diff := load_diff(last_ptr1, last_ptr2, 0) 86 diff := Or(diff, load_diff(last_ptr1, last_ptr2, 8)).u64 87 diff := Or(diff, load_diff(last_ptr1, last_ptr2, 16)).u64 88 diff := Or(diff, load_diff(last_ptr1, last_ptr2, 24)).u64 89 If(diff, 0).NE.Unlikely.b { 90 Goto(:NotEqual) 91 } 92 Return(1).b 93 } 94 next_ptr2 := Add(ptr2, 32).ptr 95 Goto(:Loop) 96 Label(:NotEqual) 97 Return(0).b 98 } 99 100 reg_mask = Options.arch == :arm64 ? RegMask.new($full_regmap, :arg0, :arg1, :tmp0, :tmp1, :callee0, :callee2) : 101 RegMask.new($full_regmap, :arg0, :arg1, :tmp0, :tmp1, :callee0, :caller1) 102 103 function("#{lang}StringEquals#{suffix}".to_sym, 104 params: {str1: 'ref', str2: 'ref'}, 105 regmap: $full_regmap, 106 regalloc_set: reg_mask, 107 mode: mode, 108 lang: lang.empty? ? 'PANDA_ASSEMBLY' : lang.upcase) { 109 # Arm32 is not supported 110 # length shift should be from 0 to 2 111 if Options.arch == :arm32 || length_shift > 2 112 Intrinsic(:UNREACHABLE).void.Terminator 113 next 114 end 115 unless dynamic 116 If(str2, 0).EQ.Unlikely.b { 117 Goto(:NotEqual) 118 } 119 end 120 If(str1, str2).EQ.Unlikely.b { 121 Return(1).b 122 } 123 if dynamic 124 length1 := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32 125 length2 := LoadI(str2).Imm(Constants::STRING_LENGTH_OFFSET).u32 126 length1 := AndI(length1).Imm("~(2U)").u32 127 length2 := AndI(length2).Imm("~(2U)").u32 128 If(length1, length2).NE.Unlikely.b { 129 Goto(:NotEqual) 130 } 131 length := Cast(length1.u32).u64 132 else 133 class1 := load_class(str1) 134 class2 := load_class(str2) 135 If(class1, class2).NE.Unlikely.b { 136 Goto(:NotEqual) 137 } 138 length1 := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32 139 length2 := LoadI(str2).Imm(Constants::STRING_LENGTH_OFFSET).u32 140 If(length1, length2).NE.Unlikely.b { 141 Goto(:NotEqual) 142 } 143 length := Cast(length1.u32).u64 144 end 145 length := unpack_length(length, compression, length_shift) 146 If(length, 8).GT.b { 147 Goto(:Long) 148 } 149 odd_bytes := Sub(8, length).u64 150 last_idx := Sub(Constants::STRING_DATA_OFFSET, odd_bytes).u64 151 buf1 := Load(str1, last_idx).u64 152 buf2 := Load(str2, last_idx).u64 153 odd_bits := Shl(odd_bytes, 3).u64 154 diff := Shr(Xor(buf1, buf2).u64, odd_bits).u64 155 res := Compare(diff, 0).EQ.b 156 if dynamic 157 # If length is 0, odd_bits is 64 and Shr above does nothing, so we effectively compare last 8 bytes of two strings 158 # before their data (length and hash code). Hash code for empty string is always 0, but value stored in length field 159 # of equal strings can be different in dynamic implementation, so we check (length == 0) separately 160 res := Or(res, Compare(length, 0).EQ.b).b 161 end 162 Return(res).b 163 164 Label(:Long) 165 unroll := Compare(length, 64).GE.b 166 IfImm(unroll).Imm(0).SrcType("DataType::BOOL").NE.b { 167 LiveOut(str1).DstReg(regmap[:arg0]).ref 168 LiveOut(str2).DstReg(regmap[:arg1]).ref 169 entrypoint_id = "#{lang.empty? ? '' : lang.upcase + '_'}STRING_EQUALS_UNROLL" + (compression ? "_COMPRESSED" : ""); 170 entrypoint_offset = get_entrypoint_offset(entrypoint_id) 171 entrypoint_name = "#{lang.empty? ? '' : lang}StringEqualsUnroll" + (compression ? "Compressed" : ""); 172 Intrinsic(:TAIL_CALL).AddImm(entrypoint_offset).MethodAsImm(entrypoint_name).Terminator.b 173 } 174 first_idx := Constants::STRING_DATA_OFFSET 175 last_idx := Sub(Add(first_idx, length).u64, 8).u64 176 177 Label(:Loop) 178 idx := Phi(first_idx, next_idx).u64 179 buf1 := Load(str1, idx).u64 180 buf2 := Load(str2, idx).u64 181 If(buf1, buf2).NE.Unlikely.b { 182 Goto(:NotEqual) 183 } 184 next_idx := Add(idx, 8).u64 185 If(next_idx, last_idx).GE.Unlikely.b { 186 buf1 := Load(str1, last_idx).u64 187 buf2 := Load(str2, last_idx).u64 188 res := Compare(buf1, buf2).EQ.b 189 Return(res).b 190 } 191 Goto(:Loop) 192 Label(:NotEqual) 193 Return(0).b 194 } 195end 196 197# Try to allocate String in TLAB. 198# The result is either a pointer to a new string or null if there is no enough space in TLAB. 199macro(:allocate_string_tlab) do |string_klass, data_size| 200 if Options.arch == :arm32 201 Intrinsic(:UNREACHABLE).Terminator.void 202 ReturnVoid().void 203 next 204 end 205 206 # Add sizeof(String) and do align 207 _data_size := Cast(data_size).word 208 _size := AndI(AddI(_data_size).Imm(Constants::STRING_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word 209 # Load pointer to the TLAB from TLS 210 _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr 211 # Load pointer to the start address of free memory in the TLAB 212 _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr 213 # Load pointer to the end address of free memory in the TLAB 214 _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr 215 # Check if there is enough space 216 If(Sub(_end, _start).word, _size).B.Unlikely.b { 217 Goto(:SlowPathEntrypoint) 218 } 219 Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG 220 if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__ 221 call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void 222 end 223 # Store class of the object 224 store_class(_start, string_klass) 225 # Update the TLAB state 226 StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr 227 # Return a pointer to the newly allocated string 228 _allocated_string := _start 229end 230 231# Try to allocate String in TLAB. 232# The result is either a pointer to a new string or null if there is no enough space in TLAB. 233# this is a version that does not do allocation tracking due to the bug #24977 234macro(:allocate_string_tlab_no_debug) do |string_klass, data_size| 235 if Options.arch == :arm32 236 Intrinsic(:UNREACHABLE).Terminator.void 237 ReturnVoid().void 238 next 239 end 240 241 # Add sizeof(String) and do align 242 _data_size := Cast(data_size).word 243 _size := AndI(AddI(_data_size).Imm(Constants::STRING_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word 244 # Load pointer to the TLAB from TLS 245 _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr 246 # Load pointer to the start address of free memory in the TLAB 247 _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr 248 # Load pointer to the end address of free memory in the TLAB 249 _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr 250 # Check if there is enough space 251 If(Sub(_end, _start).word, _size).B.Unlikely.b { 252 Goto(:SlowPathEntrypoint) 253 } 254 if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__ 255 call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void 256 end 257 # Store class of the object 258 StoreI(_start, string_klass).Imm(Constants::OBJECT_CLASS_OFFSET).ref 259 # Update the TLAB state 260 StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr 261 # Return a pointer to the newly allocated string 262 _allocated_string := _start 263end 264 265# Try to allocate Array of i8 chars in TLAB. 266# The result is either a pointer to a new array or null if there is no enough space in TLAB. 267macro(:allocate_array_of_bytes_tlab) do |array_klass, char_count| 268 if Options.arch == :arm32 269 Intrinsic(:UNREACHABLE).Terminator.void 270 ReturnVoid().void 271 next 272 end 273 274 elements_num := And(char_count, "0x00000000ffffffff").word 275 # Account for u16 char size 276 _size := Cast(elements_num).word 277 # Add sizeof(Array) and do align 278 _size := AndI(AddI(_size).Imm(Constants::ARRAY_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word 279 # Load pointer to the TLAB from TLS 280 _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr 281 # Load pointer to the start address of free memory in the TLAB 282 _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr 283 # Load pointer to the end address of free memory in the TLAB 284 _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr 285 # Check if there is enough space 286 If(Sub(_end, _start).word, _size).B.Unlikely.b { 287 Goto(:SlowPathEntrypoint) 288 } 289 Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG 290 if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__ 291 call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void 292 end 293 # Store class of the object 294 store_class(_start, array_klass) 295 # Store array length 296 StoreI(_start, elements_num).Imm(Constants::ARRAY_LENGTH_OFFSET).word 297 # Update the TLAB state 298 StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr 299 # Return a pointer to the newly allocated array 300 _allocated_array := _start 301end 302 303# Try to allocate Array of u16 chars in TLAB. 304# The result is either a pointer to a new array or null if there is no enough space in TLAB. 305macro(:allocate_array_of_chars_tlab) do |array_klass, char_count| 306 if Options.arch == :arm32 307 Intrinsic(:UNREACHABLE).Terminator.void 308 ReturnVoid().void 309 next 310 end 311 312 elements_num := And(char_count, "0x00000000ffffffff").word 313 # Account for u16 char size 314 _size := Shl(elements_num, 1).word 315 # Add sizeof(Array) and do align 316 _size := AndI(AddI(_size).Imm(Constants::ARRAY_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word 317 # Load pointer to the TLAB from TLS 318 _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr 319 # Load pointer to the start address of free memory in the TLAB 320 _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr 321 # Load pointer to the end address of free memory in the TLAB 322 _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr 323 # Check if there is enough space 324 If(Sub(_end, _start).word, _size).B.Unlikely.b { 325 Goto(:SlowPathEntrypoint) 326 } 327 Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG 328 if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__ 329 call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void 330 end 331 # Store class of the object 332 store_class(_start, array_klass) 333 # Store array length 334 StoreI(_start, elements_num).Imm(Constants::ARRAY_LENGTH_OFFSET).word 335 # Update the TLAB state 336 StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr 337 # Return a pointer to the newly allocated array 338 _allocated_array := _start 339end 340 341 342def GenerateCreateStringFromStringTlab(string_compression_enabled) 343 suffix = (string_compression_enabled ? "Compressed" : "") 344 available_regs = $panda_mask 345 function("CreateStringFromStringTlab#{suffix}".to_sym, 346 params: {str: 'ref'}, 347 regmap: $full_regmap, 348 regalloc_set: available_regs, 349 mode: [:FastPath]) { 350 351 if Options.arch == :arm32 352 Intrinsic(:UNREACHABLE).Terminator.void 353 ReturnVoid().void 354 next 355 end 356 357 # There is no check of the argument against NullPointer as 358 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 359 klass := load_class(str) 360 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 361 hashcode := LoadI(str).Imm(Constants::STRING_HASHCODE_OFFSET).u32 362 data_size := unpack_length(Cast(length).u64, string_compression_enabled, 1) 363 364 new_str := allocate_string_tlab(klass, Cast(data_size).word) 365 StoreI(new_str, length).Imm(Constants::STRING_LENGTH_OFFSET).u32 366 StoreI(new_str, hashcode).Imm(Constants::STRING_HASHCODE_OFFSET).u32 367 368 # Copy string data 369 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 370 dst_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 371 offs := Cast(0).u64 372 If(data_size, 8).AE.Likely.b { 373 stop := AndI(data_size).Imm(7).u64 374 Label(:CopyLoop_8b) 375 offs1 := Phi(offs, offs2).u64 376 Store(dst_str_data, offs, Load(src_str_data, offs).u64).u64 377 offs2 := AddI(offs1).Imm(8).u64 378 If(offs2, stop).B.Likely.b { 379 Goto(:CopyLoop_8b) 380 } 381 } 382 offs3 := Phi(offs, offs2).u64 383 If(offs3, data_size).B.Likely.b { 384 Label(:CopyLoop_1b) 385 offs4 := Phi(offs3, offs5).u64 386 Store(dst_str_data, offs4, Load(src_str_data, offs4).u8).u8 387 offs5 := AddI(offs4).Imm(1).u64 388 If(offs5, data_size).B.Likely.b { 389 Goto(:CopyLoop_1b) 390 } 391 } 392 393 # String is supposed to be a constant object, so all its data should be visible by all threads 394 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 395 Return(new_str).ptr 396 397 Label(:SlowPathEntrypoint) 398 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_STRING_SLOW_PATH") 399 Intrinsic(:SLOW_PATH_ENTRY, str).AddImm(entrypoint).MethodAsImm("CreateStringFromStringOddSavedBridge").Terminator.ptr 400 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 401 } 402end # def GenerateCreateStringFromStringTlab 403 404 405### 406# Checks if starting from arr_data the specified number of chars (char_count) can be compressed 407# 408# Utf16 char is ASCII if (utf16_char - 1U < utf::UTF8_1B_MAX) 409# See runtime/include/coretypes/string.h - IsASCIICharacter 410# 411scoped_macro(:is_array_of_compressable_chars) do |arr_data, char_count| 412 # Check 4-chars block at once if it's possible 413 n_blocks := ShrI(char_count).Imm(2).u64 414 If(n_blocks, 0).A.Likely.b { 415 # 0x007F is utf::UTF8_1B_MAX 416 utf8_1b_max := Cast(0x007F007F007F007F).u64 417 utf8_1b_max_mask := Not(utf8_1b_max).u64 418 i1 := Cast(0).u64 419 Label(:CanBeCompressedLoop_4chars) 420 i2 := Phi(i1, i3).u64 421 four_chars := Load(arr_data, ShlI(i2).Imm(3).u64).u64 422 # First: check if there are chars greater than utf::UTF8_1B_MAX 423 If(And(four_chars, utf8_1b_max_mask).u64, 0).NE.Unlikely.b { 424 compressable1 := Cast(0).b 425 Goto(:CanBeCompressedLoopDone) 426 } 427 # Second: check if there are chars equal to zero 428 If(And(SubI(four_chars).Imm(0x0001000100010001).u64, utf8_1b_max_mask).u64, 0).NE.Unlikely.b { 429 compressable2 := Cast(0).b 430 Goto(:CanBeCompressedLoopDone) 431 } 432 i3 := AddI(i2).Imm(1).u64 433 If(i3, n_blocks).B.Likely.b { 434 Goto(:CanBeCompressedLoop_4chars) 435 } 436 } 437 # check the rest of the chars 438 If(AndI(char_count).Imm(3).u64, 0).A.Likely.b { 439 i4 := ShlI(n_blocks).Imm(2).u64 # number of already copied chars if any 440Label(:CanBeCompressedLoop) 441 i5 := Phi(i4, i6).u64 442 ch := Load(arr_data, ShlI(i5).Imm(1).u64).u16 443 If(SubI(ch).Imm(Constants::STRING_MUTF8_1B_MIN).u16, Cast(Constants::STRING_MUTF8_1B_MAX).u16).AE.Unlikely.b { 444 compressable3 := Cast(0).b 445 Goto(:CanBeCompressedLoopDone) 446 } 447 i6 := AddI(i5).Imm(1).u64 448 If(i6, char_count).B.Likely.b { 449 Goto(:CanBeCompressedLoop) 450 } 451 } 452 compressable4 := Cast(1).b 453Label(:CanBeCompressedLoopDone) 454 compressable := Phi(compressable1, compressable2, compressable3, compressable4).b 455end # is_array_of_compressable_chars 456 457 458### 459# Copy u16 chars compressing them to u8. 460# It is assumed that all u16 chars are compressable. 461# 462scoped_macro(:compress_u16_to_u8_chars) do |src_data, dst_data, char_count| 463if Options.arch == :arm64 464 # Copy 32-byte chunks (if any) compressing them into 16-byte chunks 465 If(char_count, 16).AE.Likely.b { 466 stop := Add(src_data, ShlI(ShrI(char_count).Imm(4).u64).Imm(5).u64).ptr 467Label(:CopyLoop_32b) 468 src_data1 := Phi(src_data, src_data2).ptr 469 dst_data1 := Phi(dst_data, dst_data2).ptr 470 Intrinsic(:COMPRESS_SIXTEEN_UTF16_TO_UTF8_CHARS_USING_SIMD, src_data1, dst_data1).void 471 src_data2 := AddI(src_data1).Imm(32).ptr 472 dst_data2 := AddI(dst_data1).Imm(16).ptr 473 If(src_data2, stop).LT.Likely.b { 474 Goto(:CopyLoop_32b) 475 } 476 char_count1 := AndI(char_count).Imm(0xF).u64 477 } 478 src_data3 := Phi(src_data, src_data2).ptr 479 dst_data3 := Phi(dst_data, dst_data2).ptr 480 char_count2 := Phi(char_count, char_count1).u64 481 482 # Copy 16-byte chunk (if any) compressing it into 8-byte chunk 483 If(char_count2, 8).AE.Likely.b { 484 Intrinsic(:COMPRESS_EIGHT_UTF16_TO_UTF8_CHARS_USING_SIMD, src_data3, dst_data3).void 485 src_data4 := AddI(src_data3).Imm(16).ptr 486 dst_data4 := AddI(dst_data3).Imm(8).ptr 487 } 488 src_data5 := Phi(src_data3, src_data4).ptr 489 dst_data5 := Phi(dst_data3, dst_data4).ptr 490 # Copy 2-byte chunks compressing them into 1-byte 491 n_2b := AndI(char_count2).Imm(7).u64 492 If(n_2b, 0).A.Likely.b { 493 j1 := Cast(0).u64 494Label(:CopyLoop_2b) 495 j := Phi(j1, j2).u64 496 Store(dst_data5, j, Load(src_data5, ShlI(j).Imm(1).u64).u8).u8 497 j2 := AddI(j).Imm(1).u64 498 If(j2, n_2b).B.Likely.b { 499 Goto(:CopyLoop_2b) 500 } 501 } 502else # if Options.arch == :arm64 503 # Copy 8-byte chunks compressing them into 4-byte chunks 504 n_8b := ShrI(char_count).Imm(2).u64 505 If(n_8b, 0).A.Likely.b { 506 i1 := Cast(0).u64 507Label(:CopyLoop_8b) 508 i := Phi(i1, i2).u64 509 chunk := Load(src_data, ShlI(i).Imm(3).u64).u64 510 chunk := AndI(chunk).Imm(0x00ff00ff00ff00ff).u64 # zero high part of each two-byte 511 chunk := AndI(Or(chunk, ShrI(chunk).Imm(8).u64).u64).Imm(0x0000ffff0000ffff).u64 512 chunk := Or(chunk, ShrI(chunk).Imm(16).u64).u64 513 Store(dst_data, ShlI(i).Imm(2).u64, Cast(chunk).u32).u32 514 i2 := AddI(i).Imm(1).u64 515 If(i2, n_8b).B.Likely.b { 516 Goto(:CopyLoop_8b) 517 } 518 } 519 # Copy 2-byte chunks compressing them into 1-byte 520 If(AndI(char_count).Imm(3).u64, 0).A.Likely.b { 521 j1 := ShlI(n_8b).Imm(2).u64 # number of already copied chars if any 522Label(:CopyLoop_2b) 523 j := Phi(j1, j2).u64 524 Store(dst_data, j, Load(src_data, ShlI(j).Imm(1).u64).u8).u8 525 j2 := AddI(j).Imm(1).u64 526 If(j2, char_count).B.Likely.b { 527 Goto(:CopyLoop_2b) 528 } 529 } 530end # if Options.arch == :arm64 531end # compress_u16_to_u8_chars 532 533### 534# Copy dwords (8-bytes) 535# 536scoped_macro(:copy_dwords) do |src, dst, size| 537 count := AndI(size).Imm(~0x7).word 538 i1 := Cast(0).word 539Label(:CopyLoop_8b) 540 i := Phi(i1, i2).word 541 If(i, count).AE.Unlikely { 542 Goto(:End) 543 } 544 Store(dst, i, Load(src, i).word).word 545 i2 := AddI(i).Imm(8).word 546 Goto(:CopyLoop_8b) 547Label(:End) 548end # copy_dwords 549 550### 551# Copy u8 chars from src to dst 552# 553scoped_macro(:copy_u8_chars) do |src, dst, count| 554 # Copy 8-byte chunks 555 len := Cast(count).word 556 copy_dwords(src, dst, len) 557 558 # copy the tail if needed 559 i1 := AndI(len).Imm(~0x7).word 560Label(:CopyLoop) 561 i := Phi(i1, i2).word 562 If(i, len).AE.Unlikely { 563 Goto(:End) 564 } 565 Store(dst, i, Load(src, i).u8).u8 566 i2 := AddI(i).Imm(1).word 567 Goto(:CopyLoop) 568 569Label(:End) 570end # copy_u8_chars 571 572### 573# Copy u16 chars from src to dst 574# 575scoped_macro(:copy_u16_chars) do |src, dst, count| 576 # Copy 8-byte chunks 577 len := Cast(ShlI(count).Imm(1).u32).word 578 copy_dwords(src, dst, len) 579 580 # copy the tail if needed 581 i1 := AndI(len).Imm(-8).u64 582Label(:CopyLoop) 583 i := Phi(i1, i2).u64 584 If(i, len).AE.Unlikely.b { 585 Goto(:End) 586 } 587 Store(dst, i, Load(src, i).u16).u16 588 i2 := AddI(i).Imm(2).u64 589 Goto(:CopyLoop) 590 591Label(:End) 592end # copy_u16_chars 593 594### 595# Copy u8 chars expanding them to u16 chars 596# 597scoped_macro(:expand_u8_to_u16_chars) do |src, dst, count| 598 i0 := Cast(0).u64 599 len := Cast(count).u64 600Label(:CopyLoop) 601 i := Phi(i0, i1).u64 602 If(i, len).AE.Unlikely.b { 603 Goto(:End) 604 } 605 Store(dst, ShlI(i).Imm(1).u64, Cast(Load(src, i).u8).u16).u16 606 i1 := AddI(i).Imm(1).u64 607 Goto(:CopyLoop) 608Label(:End) 609end # expand_u8_to_u16_chars 610 611### 612# Copy data from compressed string to array of utf16 chars 613# 614scoped_macro(:copy_compressed_string_to_array_of_chars) do |str_data, arr_data, char_count| 615 # String contains 8-bit chars 616 If(char_count, 0).A.Likely.b { 617 i1 := Cast(0).u64 618Label(:CopyLoop) 619 i := Phi(i1, i2).u64 620 Store(arr_data, ShlI(i).Imm(1).u64, Cast(Load(str_data, i).u8).u16).u16 621 i2 := AddI(i).Imm(1).u64 622 If(i2, char_count).B.Likely.b { 623 Goto(:CopyLoop) 624 } 625 } 626end # copy_compressed_string_to_array_of_chars 627 628# calculate the difference between string chunks 629# stored in 64-bits numbers, byte/hword swap is 630# required before comparison for the correct result 631 632scoped_macro(:calculate_chars_difference_lat) do |d1, d2| 633 ldata1 := Intrinsic(:REVERSE_BYTES_U64, d1).u64 634 ldata2 := Intrinsic(:REVERSE_BYTES_U64, d2).u64 635 ret := Cmp(ldata1, ldata2).i32 636end 637 638scoped_macro(:calculate_chars_difference_utf) do |d1, d2| 639 udata1 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(d1).f64).f64).u64 640 udata2 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(d2).f64).f64).u64 641 ret := Cmp(udata1, udata2).i32 642end 643 644scoped_macro(:calculate_chars_difference) do |d1, d2, utf| 645 data1 := Cast(d1).u64 646 data2 := Cast(d2).u64 647 648 If(utf, 0).EQ.b { 649 cmp1 := calculate_chars_difference_lat(data1, data2) 650 } Else { 651 cmp2 := calculate_chars_difference_utf(data1, data2) 652 } 653 ret := Phi(cmp1, cmp2).i32 654end 655 656scoped_macro(:compare_LU_strings) do |lat, utf, len| 657 # considering the data buffer to be allocated to 658 # 8-bytes alignment, we can safely read 8-bytes chunks 659 last_idx := SubI(len).Imm(4).i64 660 i1 := 0 661Label(:Loop) 662 i := Phi(i1, i2).i64 663 If(i, last_idx).GE.Unlikely.b { 664 # can safely read 8 bytes behind - length and hashcode 665 junk_bits := ShlI(Sub(i, last_idx).u64).Imm(4).u64 666 last_lv := Load(lat, last_idx).u32 667 last_v := Shr(Bitcast(Intrinsic(:EXPAND_U8_TO_U16, Bitcast(last_lv).f32).f64).u64, junk_bits).u64 668 last_u := Shr(Load(utf, ShlI(last_idx).Imm(1).u64).u64, junk_bits).u64 669 If(last_v, last_u).NE.b { 670 Goto(:NotEqual) 671 } 672 Goto(:End) 673 } 674 675 lv := Load(lat, i).u32 676 v := Bitcast(Intrinsic(:EXPAND_U8_TO_U16, Bitcast(lv).f32).f64).u64 677 u := Load(utf, ShlI(i).Imm(1).u64).u64 678 If(v, u).NE.b { 679 Goto(:NotEqual) 680 } 681 i2 := AddI(i).Imm(4).i64 682 Goto(:Loop) 683 684 # the version assuming 128-bits types support 685 # 686 # v := Intrinsic(:EXPAND_U8_TO_U16, Load(lat, i).f64).f128 687 # u := Load(utf, i).f128 688 # v1 = Intrinsic(:GET_LOW_PART, v).u64 689 # u1 = Intrinsic(:GET_LOW_PART, u).u64 690 # n11 := Bitcast(v1).u64 691 # n21 := Bitcast(u1).u64 692 # If(n11, n21).NE.b { 693 # d11 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, v1).f64).u64 694 # d21 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, u1).f64).u64 695 # Goto(:NotEqual) 696 # } 697 # v2 = Intrinsic(:GET_HIGH_PART, v).u64 698 # u2 = Intrinsic(:GET_HIGH_PART, u).u64 699 # n12 := Bitcast(v2).u64 700 # n22 := Bitcast(u2).u64 701 # If(n12, n22).NE.b { 702 # d12 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, v2).f64).u64 703 # d22 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, u2).f64).u64 704 # Goto(:NotEqual) 705 # } 706 # 707 # Label(:NotEqual) 708 # d1 := Phi(d11, d12) 709 # d2 := Phi(d21, d22) 710 # res := Sub(d1, d2).u64 711 712Label(:NotEqual) 713 v_ne := Phi(last_v, v).u64 714 u_ne := Phi(last_u, u).u64 715 ret_ne := calculate_chars_difference_utf(v_ne, u_ne).i32 716 Goto(:End) 717 718Label(:End) 719 # -1: the prefix is the same but the utf string has to be 720 # longer as it would have had latin encoding otherwise 721 ret := Phi(-1, ret_ne).i32 722end 723 724scoped_macro(:compare_mixed_strings) do |buf1, buf2, len, utf_is_first| 725 If(utf_is_first, 0).EQ.b { 726 ret1 := compare_LU_strings(buf1, buf2, len) 727 } Else { 728 ret2 := Neg(compare_LU_strings(buf2, buf1, len).i32).i32 729 } 730 ret := Phi(ret1, ret2).i32 731end 732 733def GenerateCreateStringFromCharArrayTlab(string_compression_enabled) 734 suffix = (string_compression_enabled ? "Compressed" : "") 735 available_regs = $panda_mask 736 function("CreateStringFromCharArrayTlab#{suffix}".to_sym, 737 params: {char_offset: 'u32', char_count: 'u32', char_array: 'ref', string_klass: 'ref'}, 738 regmap: $full_regmap, 739 regalloc_set: available_regs, 740 mode: [:FastPath]) { 741 742 if Options.arch == :arm32 743 Intrinsic(:UNREACHABLE).Terminator.void 744 ReturnVoid().void 745 next 746 end 747 748 # There is no check of the arguments against NullPointer as 749 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 750 arr_offs := AddI(ShlI(Cast(char_offset).u64).Imm(1).u64).Imm(Constants::ARRAY_DATA_OFFSET).u64 751 arr_data := Add(Cast(char_array).SrcType(Constants::COMPILER_REFERENCE).ptr, arr_offs).ptr 752 753 # Allocate a new string 754 if string_compression_enabled 755 compressable := is_array_of_compressable_chars(arr_data, Cast(char_count).u64) 756 If(compressable, 1).EQ.Likely.b { 757 data_size1 := Cast(char_count).word 758 } Else { 759 data_size2 := Cast(ShlI(char_count).Imm(1).u32).word 760 } 761 data_size := Phi(data_size1, data_size2).word 762 else 763 data_size := Cast(ShlI(char_count).Imm(1).u32).word 764 end 765 new_str := allocate_string_tlab(string_klass, data_size) 766 str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 767 768 # Copy data from char_array to the new string 769 # String length field is set according to SetLength() from runtime/include/coretypes/string.h 770 if string_compression_enabled 771 If(compressable, 1).EQ.Likely.b { 772 compress_u16_to_u8_chars(arr_data, str_data, Cast(char_count).u64) 773 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 774 } Else { 775 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 776 StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 777 } 778 else 779 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 780 StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 781 end 782 # String is supposed to be a constant object, so all its data should be visible by all threads 783 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 784 Return(new_str).ptr 785 786 Label(:SlowPathEntrypoint) 787 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_CHAR_ARRAY_SLOW_PATH") 788 Intrinsic(:SLOW_PATH_ENTRY, char_offset, char_count, char_array).AddImm(entrypoint).MethodAsImm("CreateStringFromCharArray4ArgBridge").Terminator.ptr 789 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 790 } 791end # def GenerateCreateStringFromCharArrayTlab 792 793 794def GenerateCreateStringFromZeroBasedCharArrayTlab(string_compression_enabled) 795 suffix = (string_compression_enabled ? "Compressed" : "") 796 available_regs = $panda_mask 797 function("CreateStringFromZeroBasedCharArrayTlab#{suffix}".to_sym, 798 params: {char_count: 'u32', char_array: 'ref', string_klass: 'ref'}, 799 regmap: $full_regmap, 800 regalloc_set: available_regs, 801 mode: [:FastPath]) { 802 803 if Options.arch == :arm32 804 Intrinsic(:UNREACHABLE).Terminator.void 805 ReturnVoid().void 806 next 807 end 808 809 # There is no check of the arguments against NullPointer as 810 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 811 arr_data := Add(Cast(char_array).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr 812 813 # Allocate a new string 814 if string_compression_enabled 815 compressable := is_array_of_compressable_chars(arr_data, Cast(char_count).u64) 816 If(compressable, 1).EQ.Likely.b { 817 data_size1 := Cast(char_count).word 818 } Else { 819 data_size2 := Cast(ShlI(char_count).Imm(1).u32).word 820 } 821 data_size := Phi(data_size1, data_size2).word 822 else 823 data_size := Cast(ShlI(char_count).Imm(1).u32).word 824 end 825 new_str := allocate_string_tlab(string_klass, data_size) 826 str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 827 828 # Copy data from char_array to the new string 829 # String length field is set according to SetLength() from runtime/include/coretypes/string.h 830 if string_compression_enabled 831 If(compressable, 1).EQ.Likely.b { 832 compress_u16_to_u8_chars(arr_data, str_data, Cast(char_count).u64) 833 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 834 } Else { 835 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 836 StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 837 } 838 else 839 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 840 StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 841 end 842 # String is supposed to be a constant object, so all its data should be visible by all threads 843 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 844 Return(new_str).ptr 845 846 Label(:SlowPathEntrypoint) 847 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_SLOW_PATH") 848 Intrinsic(:SLOW_PATH_ENTRY, char_count, char_array).AddImm(entrypoint).MethodAsImm("CreateStringFromZeroBasedCharArray3ArgBridge").Terminator.ptr 849 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 850 } 851end # def GenerateCreateStringFromZeroBasedCharArrayTlab 852 853 854def GenerateSubstringFromStringTlab(string_compression_enabled) 855 suffix = (string_compression_enabled ? "Compressed" : "") 856 available_regs = $panda_mask 857 function("SubStringFromStringTlab#{suffix}".to_sym, 858 params: {str: 'ref', begin_index: 'i32', end_index: 'i32'}, 859 regmap: $full_regmap, 860 regalloc_set: available_regs, 861 mode: [:FastPath]) { 862 863 if Options.arch == :arm32 864 Intrinsic(:UNREACHABLE).Terminator.void 865 ReturnVoid().void 866 next 867 end 868 869 # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded) 870 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 871 if string_compression_enabled 872 length := ShrI(length_packed).Imm(1).u32 873 else 874 length := length_packed 875 end 876 877 # If begin_index < 0, then it is assumed to be equal to zero 878 If(begin_index, Cast(0).i32).LT.Unlikely.b { 879 bx1 := Cast(0).i32 880 } 881 bx2 := Phi(begin_index, bx1).u32 882 # If end_index < 0, then it is assumed to be equal to zero 883 If(end_index, Cast(0).i32).LT.Unlikely.b { 884 ex1 := Cast(0).i32 885 } 886 ex2 := Phi(end_index, ex1).u32 887 # If begin_index > str.length(), then make it equal to str.length() 888 If(bx2, length).A.Unlikely.b { 889 bx3 := length 890 } 891 bx4 := Phi(bx2, bx3).u32 892 # If end_index > str.length(), then make it equal to str.length() 893 If(ex2, length).A.Unlikely.b { 894 ex3 := length 895 } 896 ex4 := Phi(ex2, ex3).u32 897 # If begin_index > end_index, then swap them. 898 If(bx4, ex4).GT.Unlikely.b { 899 bx5 := ex4 900 ex5 := bx4 901 } 902 bx6 := Phi(bx4, bx5).u32 903 ex6 := Phi(ex4, ex5).u32 904 905 If(bx6, 0).EQ.Likely.b { 906 If(ex6, length).EQ.Unlikely.b { 907 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 908 } 909 } 910 911 if string_compression_enabled 912 not_compressed := AndI(length_packed).Imm(1).u32 913 offset := Shl(bx6, not_compressed).u32 914 else 915 offset := ShlI(bx6).Imm(1).u32 916 end 917 918 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 919 src_str_data := Add(src_str_data, Cast(offset).u64).ptr 920 921 klass := load_class(str) 922 char_count := Sub(ex6, bx6).u32 923 924 # Allocate a new string 925 if string_compression_enabled 926 If(not_compressed, 1).EQ.Unlikely.b { 927 compressable := is_array_of_compressable_chars(src_str_data, Cast(char_count).u64) 928 If(compressable, 1).EQ.Likely.b { 929 data_size1 := Cast(char_count).word 930 } Else { 931 data_size2 := Cast(ShlI(char_count).Imm(1).u32).word 932 } 933 data_size := Phi(data_size1, data_size2).word 934 new_str := allocate_string_tlab(klass, data_size) 935 new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 936 If(compressable, 1).EQ.Likely.b { 937 compress_u16_to_u8_chars(src_str_data, new_str_data, Cast(char_count).u64) 938 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 939 } Else { 940 copy_u16_chars(src_str_data, new_str_data, Cast(char_count).u64) 941 StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 942 } 943 # String is supposed to be a constant object, so all its data should be visible by all threads 944 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 945 Return(new_str).ptr 946 } Else { 947 # Source string is already compressed 948 new_str := allocate_string_tlab(klass, Cast(char_count).word) 949 new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 950 copy_u8_chars(src_str_data, new_str_data, Cast(char_count).u64) 951 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 952 # String is supposed to be a constant object, so all its data should be visible by all threads 953 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 954 Return(new_str).ptr 955 } 956 else 957 data_size := Cast(ShlI(char_count).Imm(1).u32).word 958 new_str := allocate_string_tlab(klass, data_size) 959 new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 960 copy_u16_chars(src_str_data, new_str_data, Cast(char_count).u64) 961 StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 962 # String is supposed to be a constant object, so all its data should be visible by all threads 963 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 964 Return(new_str).ptr 965 end 966 967 Label(:SlowPathEntrypoint) 968 entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH") 969 Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr 970 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 971 } 972end # def GenerateSubstringFromStringTlab 973 974 975def GenerateStringGetCharsTlab(string_compression_enabled) 976 suffix = (string_compression_enabled ? "Compressed" : "") 977 available_regs = $panda_mask 978 function("StringGetCharsTlab#{suffix}".to_sym, 979 params: {str: 'ref', begin_index: 'i32', end_index: 'i32', array_klass: 'ref'}, 980 regmap: $full_regmap, 981 regalloc_set: available_regs, 982 mode: [:FastPath]) { 983 984 if Options.arch == :arm32 985 Intrinsic(:UNREACHABLE).Terminator.void 986 ReturnVoid().void 987 next 988 end 989 990 If(begin_index, end_index).GT.Unlikely.b { 991 Goto(:SlowPathEntrypoint) # Out of range 992 } 993 If(begin_index, Cast(0).i32).LT.Unlikely.b { 994 Goto(:SlowPathEntrypoint) # Out of range 995 } 996 997 # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded) 998 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32; 999 if string_compression_enabled 1000 If(Cast(end_index).u32, ShrI(length).Imm(1).u32).A.Unlikely.b { 1001 Goto(:SlowPathEntrypoint) # Out of range 1002 } 1003 not_compressed := AndI(length).Imm(1).u32 1004 offset := Shl(begin_index, not_compressed).u32 1005 else 1006 If(Cast(end_index).u32, length).A.Unlikely.b { 1007 Goto(:SlowPathEntrypoint) # Out of range 1008 } 1009 not_compressed := Cast(1).u32 1010 offset := ShlI(begin_index).Imm(1).u32 1011 end 1012 1013 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 1014 src_str_data := Add(src_str_data, Cast(offset).u64).ptr 1015 1016 # Allocate a new array of u16 chars 1017 char_count := Sub(Cast(end_index).u32, Cast(begin_index).u32).u64 1018 new_arr := allocate_array_of_chars_tlab(array_klass, Cast(char_count).word) 1019 new_arr_data := Add(new_arr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr 1020 If(not_compressed, Cast(0).u32).EQ.Likely.b { 1021 expand_u8_to_u16_chars(src_str_data, new_arr_data, char_count) 1022 } 1023 If(not_compressed, Cast(1).u32).EQ.Unlikely.b { 1024 copy_u16_chars(src_str_data, new_arr_data, char_count) 1025 } 1026 # String is supposed to be a constant object, so all its data should be visible by all threads 1027 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 1028 Return(new_arr).ptr 1029 1030 Label(:SlowPathEntrypoint) 1031 entrypoint = get_entrypoint_offset("STRING_GET_CHARS_SLOW_PATH") 1032 Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("StringGetChars4ArgBridge").Terminator.ptr 1033 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 1034 } 1035end # def GenerateStringGetCharsTlab 1036 1037 1038### 1039# Compute string hashcode 1040# 1041scoped_macro(:u8_string_hashcode) do |str_data, char_count| 1042 hash := Cast(0).u32 1043 If(char_count, 0).A.Likely.b { 1044 i1 := Cast(0).u32 1045 imm31 := Cast(31).u32 1046Label(:Loop_hash) 1047 hash1 := Phi(hash, hash2).u32 1048 i := Phi(i1, i2).u32 1049 ch := Cast(Load(str_data, i).u8).SrcType(Constants::COMPILER_UINT8).u32 1050 hash2 := Add(Mul(hash1, imm31).u32, ch).u32 1051 i2 := AddI(i).Imm(1).u32 1052 If(i2, char_count).B.Likely.b { 1053 Goto(:Loop_hash) 1054 } 1055 } 1056 hashcode := Phi(hash, hash2).u32 1057end # u8_string_hashcode 1058 1059scoped_macro(:u16_string_hashcode) do |str_data, char_count| 1060 hash := Cast(0).u32 1061 If(char_count, 0).A.Likely.b { 1062 i1 := Cast(0).u64 1063 imm31 := Cast(31).u32 1064 stop := ShlI(Cast(char_count).u64).Imm(1).u64 1065Label(:Loop_hash) 1066 hash1 := Phi(hash, hash2).u32 1067 i := Phi(i1, i2).u64 1068 ch := Cast(Load(str_data, i).u16).SrcType(Constants::COMPILER_UINT16).u32 1069 hash2 := Add(Mul(hash1, imm31).u32, ch).u32 1070 i2 := AddI(i).Imm(2).u64 1071 If(i2, stop).B.Likely.b { 1072 Goto(:Loop_hash) 1073 } 1074 } 1075 hashcode := Phi(hash, hash2).u32 1076end # u16_string_hashcode 1077 1078 1079def GenerateStringHashCode(string_compression_enabled) 1080 suffix = (string_compression_enabled ? "Compressed" : "") 1081if Options.arch == :arm64 1082 available_regs = $temps_mask + :arg0 + :callee0 + :callee1 + :callee2 + :callee3 1083else 1084 available_regs = $temps_mask + :arg0 + :callee0 + :caller0 + :caller1 1085end 1086 function("StringHashCode#{suffix}".to_sym, 1087 params: {str: 'ref'}, 1088 regmap: $full_regmap, 1089 regalloc_set: available_regs, 1090 mode: [:FastPath]) { 1091 1092 if Options.arch == :arm32 1093 Intrinsic(:UNREACHABLE).Terminator.void 1094 ReturnVoid().void 1095 next 1096 end 1097 1098 # 1. There is no check of the argument against NullPointer as 1099 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 1100 # 2. Don't check if hashcode is equal to 0 as it's done in the codegen. 1101 1102 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 1103 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 1104 if string_compression_enabled 1105 length := ShrI(length_packed).Imm(1).u32 1106 not_compressed := AndI(length_packed).Imm(1).u32 1107 If(not_compressed, 0).EQ.Likely.b { 1108 # String contains 8-bit chars 1109 h := u8_string_hashcode(str_data, length) 1110 StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32 1111 Return(h).u32 1112 } Else { 1113 # String contains 16-bit chars 1114 h := u16_string_hashcode(str_data, length) 1115 StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32 1116 Return(h).u32 1117 } 1118 else 1119 # String contains 16-bit chars 1120 h := u16_string_hashcode(str_data, length_packed) 1121 StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32 1122 Return(h).u32 1123 end 1124 } 1125end # def GenerateStringHashCode 1126