1#!/usr/bin/env ruby 2 3# Copyright (c) 2021-2024 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16macro(:load_diff) do |ptr1, ptr2, offset| 17 Xor(LoadI(ptr1).Imm(Constants::STRING_DATA_OFFSET + " + " + offset.to_s).u64, 18 LoadI(ptr2).Imm(Constants::STRING_DATA_OFFSET + " + " + offset.to_s).u64).u64 19end 20 21scoped_macro(:unpack_length) do |length, compression, length_shift| 22 if compression 23 char_length := ShrI(length).Imm(length_shift).u64 24 not_compressed := AndI(length).Imm(1).u64 25 unpacked_length := Shl(char_length, not_compressed).u64 26 elsif length_shift != 0 27 unpacked_length := ShrI(length).Imm(length_shift).u64 28 end 29end 30 31def GenerateStringEquals(lang, dynamic, compression) 32 suffix = (compression ? "Compressed" : "") 33 length_shift = (dynamic ? 2 : 1) 34 mode = [:FastPath] 35 mode.push(:DynamicMethod, :DynamicStub) if dynamic 36 37 reg_mask = Options.arch == :arm64 ? 38 RegMask.new($full_regmap, :arg0, :arg1, :callee0, :callee1, :callee2, :callee3, :callee4, :callee5, :callee6, :callee7, :tmp0, :tmp1) : 39 $panda_mask + :tmp0 + :tmp1 + :tr 40 41 function("#{lang}StringEqualsUnroll#{suffix}".to_sym, 42 params: {str1_ref: 'ref', str2_ref: 'ref'}, 43 regmap: $full_regmap, 44 regalloc_set: reg_mask, 45 mode: mode, 46 lang: lang.empty? ? 'PANDA_ASSEMBLY' : lang.upcase) { 47 if Options.arch == :arm32 48 Intrinsic(:UNREACHABLE).void.Terminator 49 next 50 end 51 str1 := Cast(str1_ref).SrcType(Constants::COMPILER_REFERENCE).ptr 52 str2 := Cast(str2_ref).SrcType(Constants::COMPILER_REFERENCE).ptr 53 # Compare first 32 bytes 54 buf1 := LoadI(str1).Imm(Constants::STRING_DATA_OFFSET).u64 55 buf2 := LoadI(str2).Imm(Constants::STRING_DATA_OFFSET).u64 56 If(buf1, buf2).NE.Unlikely.b { 57 Goto(:NotEqual) 58 } 59 diff := load_diff(str1, str2, 8) 60 diff := Or(diff, load_diff(str1, str2, 16)).u64 61 diff := Or(diff, load_diff(str1, str2, 24)).u64 62 If(diff, 0).NE.Unlikely.b { 63 Goto(:NotEqual) 64 } 65 length := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32 66 length := Cast(length).SrcType("DataType::UINT32").u64 67 length := unpack_length(length, compression, length_shift) 68 last_buf_index := SubI(length).Imm(32).u64 69 last_ptr1 := Add(str1, last_buf_index).ptr 70 last_ptr2 := Add(str2, last_buf_index).ptr 71 first_ptr1 := AddI(str1).Imm(32).ptr 72 first_ptr2 := AddI(str2).Imm(32).ptr 73 Label(:Loop) 74 ptr1 := Phi(first_ptr1, next_ptr1).ptr 75 ptr2 := Phi(first_ptr2, next_ptr2).ptr 76 diff := load_diff(ptr1, ptr2, 0) 77 diff := Or(diff, load_diff(ptr1, ptr2, 8)).u64 78 diff := Or(diff, load_diff(ptr1, ptr2, 16)).u64 79 diff := Or(diff, load_diff(ptr1, ptr2, 24)).u64 80 If(diff, 0).NE.Unlikely.b { 81 Goto(:NotEqual) 82 } 83 next_ptr1 := Add(ptr1, 32).ptr 84 If(next_ptr1, last_ptr1).GE.Unlikely.b { 85 diff := load_diff(last_ptr1, last_ptr2, 0) 86 diff := Or(diff, load_diff(last_ptr1, last_ptr2, 8)).u64 87 diff := Or(diff, load_diff(last_ptr1, last_ptr2, 16)).u64 88 diff := Or(diff, load_diff(last_ptr1, last_ptr2, 24)).u64 89 If(diff, 0).NE.Unlikely.b { 90 Goto(:NotEqual) 91 } 92 Return(1).b 93 } 94 next_ptr2 := Add(ptr2, 32).ptr 95 Goto(:Loop) 96 Label(:NotEqual) 97 Return(0).b 98 } 99 100 reg_mask = Options.arch == :arm64 ? RegMask.new($full_regmap, :arg0, :arg1, :tmp0, :tmp1, :callee0, :callee2) : 101 RegMask.new($full_regmap, :arg0, :arg1, :tmp0, :tmp1, :callee0, :caller1) 102 103 function("#{lang}StringEquals#{suffix}".to_sym, 104 params: {str1: 'ref', str2: 'ref'}, 105 regmap: $full_regmap, 106 regalloc_set: reg_mask, 107 mode: mode, 108 lang: lang.empty? ? 'PANDA_ASSEMBLY' : lang.upcase) { 109 # Arm32 is not supported 110 # length shift should be from 0 to 2 111 if Options.arch == :arm32 || length_shift > 2 112 Intrinsic(:UNREACHABLE).void.Terminator 113 next 114 end 115 unless dynamic 116 If(str2, 0).EQ.Unlikely.b { 117 Goto(:NotEqual) 118 } 119 end 120 If(str1, str2).EQ.Unlikely.b { 121 Return(1).b 122 } 123 if dynamic 124 length1 := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32 125 length2 := LoadI(str2).Imm(Constants::STRING_LENGTH_OFFSET).u32 126 length1 := AndI(length1).Imm("~(2U)").u32 127 length2 := AndI(length2).Imm("~(2U)").u32 128 If(length1, length2).NE.Unlikely.b { 129 Goto(:NotEqual) 130 } 131 length := Cast(length1.u32).u64 132 else 133 # assume STRING_LENGTH_OFFSET == OBJECT_CLASS_OFFSET + 4 134 class_and_length1 := LoadI(str1).Imm(Constants::OBJECT_CLASS_OFFSET).u64 135 class_and_length2 := LoadI(str2).Imm(Constants::OBJECT_CLASS_OFFSET).u64 136 If(class_and_length1, class_and_length2).NE.Unlikely.b { 137 Goto(:NotEqual) 138 } 139 length := Shr(class_and_length1, 32).u64 140 end 141 length := unpack_length(length, compression, length_shift) 142 If(length, 8).GT.b { 143 Goto(:Long) 144 } 145 odd_bytes := Sub(8, length).u64 146 last_idx := Sub(Constants::STRING_DATA_OFFSET, odd_bytes).u64 147 buf1 := Load(str1, last_idx).u64 148 buf2 := Load(str2, last_idx).u64 149 odd_bits := Shl(odd_bytes, 3).u64 150 diff := Shr(Xor(buf1, buf2).u64, odd_bits).u64 151 res := Compare(diff, 0).EQ.b 152 if dynamic 153 # If length is 0, odd_bits is 64 and Shr above does nothing, so we effectively compare last 8 bytes of two strings 154 # before their data (length and hash code). Hash code for empty string is always 0, but value stored in length field 155 # of equal strings can be different in dynamic implementation, so we check (length == 0) separately 156 res := Or(res, Compare(length, 0).EQ.b).b 157 end 158 Return(res).b 159 160 Label(:Long) 161 unroll := Compare(length, 64).GE.b 162 IfImm(unroll).Imm(0).SrcType("DataType::BOOL").NE.b { 163 LiveOut(str1).DstReg(regmap[:arg0]).ref 164 LiveOut(str2).DstReg(regmap[:arg1]).ref 165 entrypoint_id = "#{lang.empty? ? '' : lang.upcase + '_'}STRING_EQUALS_UNROLL" + (compression ? "_COMPRESSED" : ""); 166 entrypoint_offset = get_entrypoint_offset(entrypoint_id) 167 entrypoint_name = "#{lang.empty? ? '' : lang}StringEqualsUnroll" + (compression ? "Compressed" : ""); 168 Intrinsic(:TAIL_CALL).AddImm(entrypoint_offset).MethodAsImm(entrypoint_name).Terminator.b 169 } 170 first_idx := Constants::STRING_DATA_OFFSET 171 last_idx := Sub(Add(first_idx, length).u64, 8).u64 172 173 Label(:Loop) 174 idx := Phi(first_idx, next_idx).u64 175 buf1 := Load(str1, idx).u64 176 buf2 := Load(str2, idx).u64 177 If(buf1, buf2).NE.Unlikely.b { 178 Goto(:NotEqual) 179 } 180 next_idx := Add(idx, 8).u64 181 If(next_idx, last_idx).GE.Unlikely.b { 182 buf1 := Load(str1, last_idx).u64 183 buf2 := Load(str2, last_idx).u64 184 res := Compare(buf1, buf2).EQ.b 185 Return(res).b 186 } 187 Goto(:Loop) 188 Label(:NotEqual) 189 Return(0).b 190 } 191end 192 193# Try to allocate String in TLAB. 194# The result is either a pointer to a new string or null if there is no enough space in TLAB. 195macro(:allocate_string_tlab) do |string_klass, data_size| 196 if Options.arch == :arm32 197 Intrinsic(:UNREACHABLE).Terminator.void 198 ReturnVoid().void 199 next 200 end 201 202 # Add sizeof(String) and do align 203 _data_size := Cast(data_size).word 204 _size := AndI(AddI(_data_size).Imm(Constants::STRING_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word 205 # Load pointer to the TLAB from TLS 206 _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr 207 # Load pointer to the start address of free memory in the TLAB 208 _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr 209 # Load pointer to the end address of free memory in the TLAB 210 _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr 211 # Check if there is enough space 212 If(Sub(_end, _start).word, _size).B.Unlikely.b { 213 Goto(:SlowPathEntrypoint) 214 } 215 Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG 216 if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__ 217 call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void 218 end 219 # Store class of the object 220 StoreI(_start, string_klass).Imm(Constants::OBJECT_CLASS_OFFSET).ref 221 # Update the TLAB state 222 StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr 223 # Return a pointer to the newly allocated string 224 _allocated_string := _start 225end 226 227# Try to allocate Array of i8 chars in TLAB. 228# The result is either a pointer to a new array or null if there is no enough space in TLAB. 229macro(:allocate_array_of_bytes_tlab) do |array_klass, char_count| 230 if Options.arch == :arm32 231 Intrinsic(:UNREACHABLE).Terminator.void 232 ReturnVoid().void 233 next 234 end 235 236 elements_num := And(char_count, "0x00000000ffffffff").word 237 # Account for u16 char size 238 _size := Cast(elements_num).word 239 # Add sizeof(Array) and do align 240 _size := AndI(AddI(_size).Imm(Constants::ARRAY_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word 241 # Load pointer to the TLAB from TLS 242 _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr 243 # Load pointer to the start address of free memory in the TLAB 244 _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr 245 # Load pointer to the end address of free memory in the TLAB 246 _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr 247 # Check if there is enough space 248 If(Sub(_end, _start).word, _size).B.Unlikely.b { 249 Goto(:SlowPathEntrypoint) 250 } 251 Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG 252 if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__ 253 call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void 254 end 255 # Store class of the object 256 StoreI(_start, array_klass).Imm(Constants::OBJECT_CLASS_OFFSET).ref 257 # Store array length 258 StoreI(_start, elements_num).Imm(Constants::ARRAY_LENGTH_OFFSET).word 259 # Update the TLAB state 260 StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr 261 # Return a pointer to the newly allocated array 262 _allocated_array := _start 263end 264 265# Try to allocate Array of u16 chars in TLAB. 266# The result is either a pointer to a new array or null if there is no enough space in TLAB. 267macro(:allocate_array_of_chars_tlab) do |array_klass, char_count| 268 if Options.arch == :arm32 269 Intrinsic(:UNREACHABLE).Terminator.void 270 ReturnVoid().void 271 next 272 end 273 274 elements_num := And(char_count, "0x00000000ffffffff").word 275 # Account for u16 char size 276 _size := Shl(elements_num, 1).word 277 # Add sizeof(Array) and do align 278 _size := AndI(AddI(_size).Imm(Constants::ARRAY_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word 279 # Load pointer to the TLAB from TLS 280 _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr 281 # Load pointer to the start address of free memory in the TLAB 282 _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr 283 # Load pointer to the end address of free memory in the TLAB 284 _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr 285 # Check if there is enough space 286 If(Sub(_end, _start).word, _size).B.Unlikely.b { 287 Goto(:SlowPathEntrypoint) 288 } 289 Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG 290 if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__ 291 call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void 292 end 293 # Store class of the object 294 StoreI(_start, array_klass).Imm(Constants::OBJECT_CLASS_OFFSET).ref 295 # Store array length 296 StoreI(_start, elements_num).Imm(Constants::ARRAY_LENGTH_OFFSET).word 297 # Update the TLAB state 298 StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr 299 # Return a pointer to the newly allocated array 300 _allocated_array := _start 301end 302 303 304def GenerateCreateStringFromStringTlab(string_compression_enabled) 305 suffix = (string_compression_enabled ? "Compressed" : "") 306 available_regs = $panda_mask 307 function("CreateStringFromStringTlab#{suffix}".to_sym, 308 params: {str: 'ref'}, 309 regmap: $full_regmap, 310 regalloc_set: available_regs, 311 mode: [:FastPath]) { 312 313 if Options.arch == :arm32 314 Intrinsic(:UNREACHABLE).Terminator.void 315 ReturnVoid().void 316 next 317 end 318 319 # There is no check of the argument against NullPointer as 320 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 321 klass := LoadI(str).Imm(Constants::OBJECT_CLASS_OFFSET).ref 322 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 323 hashcode := LoadI(str).Imm(Constants::STRING_HASHCODE_OFFSET).u32 324 data_size := unpack_length(Cast(length).u64, string_compression_enabled, 1) 325 326 new_str := allocate_string_tlab(klass, Cast(data_size).word) 327 StoreI(new_str, length).Imm(Constants::STRING_LENGTH_OFFSET).u32 328 StoreI(new_str, hashcode).Imm(Constants::STRING_HASHCODE_OFFSET).u32 329 330 # Copy string data 331 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 332 dst_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 333 offs := Cast(0).u64 334 If(data_size, 8).AE.Likely.b { 335 stop := AndI(data_size).Imm(7).u64 336 Label(:CopyLoop_8b) 337 offs1 := Phi(offs, offs2).u64 338 Store(dst_str_data, offs, Load(src_str_data, offs).u64).u64 339 offs2 := AddI(offs1).Imm(8).u64 340 If(offs2, stop).B.Likely.b { 341 Goto(:CopyLoop_8b) 342 } 343 } 344 offs3 := Phi(offs, offs2).u64 345 If(offs3, data_size).B.Likely.b { 346 Label(:CopyLoop_1b) 347 offs4 := Phi(offs3, offs5).u64 348 Store(dst_str_data, offs4, Load(src_str_data, offs4).u8).u8 349 offs5 := AddI(offs4).Imm(1).u64 350 If(offs5, data_size).B.Likely.b { 351 Goto(:CopyLoop_1b) 352 } 353 } 354 355 # String is supposed to be a constant object, so all its data should be visible by all threads 356 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 357 Return(new_str).ptr 358 359 Label(:SlowPathEntrypoint) 360 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_STRING_SLOW_PATH") 361 Intrinsic(:SLOW_PATH_ENTRY, str).AddImm(entrypoint).MethodAsImm("CreateStringFromStringOddSavedBridge").Terminator.ptr 362 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 363 } 364end # def GenerateCreateStringFromStringTlab 365 366 367### 368# Checks if starting from arr_data the specified number of chars (char_count) can be compressed 369# 370# Utf16 char is ASCII if (uft16_char - 1U < utf::UTF8_1B_MAX) 371# See runtime/include/coretypes/string.h - IsASCIICharacter 372# 373scoped_macro(:is_array_of_compressable_chars) do |arr_data, char_count| 374 # Check 4-chars block at once if it's possible 375 n_blocks := ShrI(char_count).Imm(2).u64 376 If(n_blocks, 0).A.Likely.b { 377 # 0x007F is utf::UTF8_1B_MAX 378 utf8_1b_max := Cast(0x007F007F007F007F).u64 379 utf8_1b_max_mask := Not(utf8_1b_max).u64 380 i1 := Cast(0).u64 381 Label(:CanBeCompressedLoop_4chars) 382 i2 := Phi(i1, i3).u64 383 four_chars := Load(arr_data, ShlI(i2).Imm(3).u64).u64 384 # First: check if there are chars greater than utf::UTF8_1B_MAX 385 If(And(four_chars, utf8_1b_max_mask).u64, 0).NE.Unlikely.b { 386 compressable1 := Cast(0).b 387 Goto(:CanBeCompressedLoopDone) 388 } 389 # Second: check if there are chars equal to zero 390 If(And(SubI(four_chars).Imm(0x0001000100010001).u64, utf8_1b_max_mask).u64, 0).NE.Unlikely.b { 391 compressable2 := Cast(0).b 392 Goto(:CanBeCompressedLoopDone) 393 } 394 i3 := AddI(i2).Imm(1).u64 395 If(i3, n_blocks).B.Likely.b { 396 Goto(:CanBeCompressedLoop_4chars) 397 } 398 } 399 # check the rest of the chars 400 If(AndI(char_count).Imm(3).u64, 0).A.Likely.b { 401 i4 := ShlI(n_blocks).Imm(2).u64 # number of already copied chars if any 402Label(:CanBeCompressedLoop) 403 i5 := Phi(i4, i6).u64 404 ch := Load(arr_data, ShlI(i5).Imm(1).u64).u16 405 If(SubI(ch).Imm(1).u16, Cast(Constants::STRING_MUTF8_1B_MAX).u16).AE.Unlikely.b { 406 compressable3 := Cast(0).b 407 Goto(:CanBeCompressedLoopDone) 408 } 409 i6 := AddI(i5).Imm(1).u64 410 If(i6, char_count).B.Likely.b { 411 Goto(:CanBeCompressedLoop) 412 } 413 } 414 compressable4 := Cast(1).b 415Label(:CanBeCompressedLoopDone) 416 compressable := Phi(compressable1, compressable2, compressable3, compressable4).b 417end # is_array_of_compressable_chars 418 419 420### 421# Copy u16 chars compressing them to u8. 422# It is assumed that all u16 chars are compressable. 423# 424scoped_macro(:compress_u16_to_u8_chars) do |src_data, dst_data, char_count| 425if Options.arch == :arm64 426 # Copy 32-byte chunks (if any) compressing them into 16-byte chunks 427 If(char_count, 16).AE.Likely.b { 428 stop := Add(src_data, ShlI(ShrI(char_count).Imm(4).u64).Imm(5).u64).ptr 429Label(:CopyLoop_32b) 430 src_data1 := Phi(src_data, src_data2).ptr 431 dst_data1 := Phi(dst_data, dst_data2).ptr 432 Intrinsic(:COMPRESS_SIXTEEN_UTF16_TO_UTF8_CHARS_USING_SIMD, src_data1, dst_data1).void 433 src_data2 := AddI(src_data1).Imm(32).ptr 434 dst_data2 := AddI(dst_data1).Imm(16).ptr 435 If(src_data2, stop).LT.Likely.b { 436 Goto(:CopyLoop_32b) 437 } 438 char_count1 := AndI(char_count).Imm(0xF).u64 439 } 440 src_data3 := Phi(src_data, src_data2).ptr 441 dst_data3 := Phi(dst_data, dst_data2).ptr 442 char_count2 := Phi(char_count, char_count1).u64 443 444 # Copy 16-byte chunk (if any) compressing it into 8-byte chunk 445 If(char_count2, 8).AE.Likely.b { 446 Intrinsic(:COMPRESS_EIGHT_UTF16_TO_UTF8_CHARS_USING_SIMD, src_data3, dst_data3).void 447 src_data4 := AddI(src_data3).Imm(16).ptr 448 dst_data4 := AddI(dst_data3).Imm(8).ptr 449 } 450 src_data5 := Phi(src_data3, src_data4).ptr 451 dst_data5 := Phi(dst_data3, dst_data4).ptr 452 # Copy 2-byte chunks compressing them into 1-byte 453 n_2b := AndI(char_count2).Imm(7).u64 454 If(n_2b, 0).A.Likely.b { 455 j1 := Cast(0).u64 456Label(:CopyLoop_2b) 457 j := Phi(j1, j2).u64 458 Store(dst_data5, j, Load(src_data5, ShlI(j).Imm(1).u64).u8).u8 459 j2 := AddI(j).Imm(1).u64 460 If(j2, n_2b).B.Likely.b { 461 Goto(:CopyLoop_2b) 462 } 463 } 464else # if Options.arch == :arm64 465 # Copy 8-byte chunks compressing them into 4-byte chunks 466 n_8b := ShrI(char_count).Imm(2).u64 467 If(n_8b, 0).A.Likely.b { 468 i1 := Cast(0).u64 469Label(:CopyLoop_8b) 470 i := Phi(i1, i2).u64 471 chunk := Load(src_data, ShlI(i).Imm(3).u64).u64 472 chunk := AndI(chunk).Imm(0x00ff00ff00ff00ff).u64 # zero high part of each two-byte 473 chunk := AndI(Or(chunk, ShrI(chunk).Imm(8).u64).u64).Imm(0x0000ffff0000ffff).u64 474 chunk := Or(chunk, ShrI(chunk).Imm(16).u64).u64 475 Store(dst_data, ShlI(i).Imm(2).u64, Cast(chunk).u32).u32 476 i2 := AddI(i).Imm(1).u64 477 If(i2, n_8b).B.Likely.b { 478 Goto(:CopyLoop_8b) 479 } 480 } 481 # Copy 2-byte chunks compressing them into 1-byte 482 If(AndI(char_count).Imm(3).u64, 0).A.Likely.b { 483 j1 := ShlI(n_8b).Imm(2).u64 # number of already copied chars if any 484Label(:CopyLoop_2b) 485 j := Phi(j1, j2).u64 486 Store(dst_data, j, Load(src_data, ShlI(j).Imm(1).u64).u8).u8 487 j2 := AddI(j).Imm(1).u64 488 If(j2, char_count).B.Likely.b { 489 Goto(:CopyLoop_2b) 490 } 491 } 492end # if Options.arch == :arm64 493end # compress_u16_to_u8_chars 494 495### 496# Copy dwords (8-bytes) 497# 498scoped_macro(:copy_dwords) do |src, dst, size| 499 count := AndI(size).Imm(~0x7).u64 500 i1 := Cast(0).u64 501Label(:CopyLoop_8b) 502 i := Phi(i1, i2).u64 503 If(i, count).AE.Unlikely.b { 504 Goto(:End) 505 } 506 Store(dst, i, Load(src, i).u64).u64 507 i2 := AddI(i).Imm(8).u64 508 Goto(:CopyLoop_8b) 509Label(:End) 510end # copy_dwords 511 512 513### 514# Copy u8 chars from src to dst 515# 516scoped_macro(:copy_u8_chars) do |src, dst, count| 517 # Copy 8-byte chunks 518 len := Cast(count).word 519 copy_dwords(src, dst, len) 520 521 # copy the tail if needed 522 i1 := AndI(len).Imm(~0x7).u64 523Label(:CopyLoop) 524 i := Phi(i1, i2).u64 525 If(i, len).AE.Unlikely.b { 526 Goto(:End) 527 } 528 Store(dst, i, Load(src, i).u8).u8 529 i2 := AddI(i).Imm(1).u64 530 Goto(:CopyLoop) 531 532Label(:End) 533end # copy_u8_chars 534 535### 536# Copy u16 chars from src to dst 537# 538scoped_macro(:copy_u16_chars) do |src, dst, count| 539 # Copy 8-byte chunks 540 len := Cast(ShlI(count).Imm(1).u32).word 541 copy_dwords(src, dst, len) 542 543 # copy the tail if needed 544 i1 := AndI(len).Imm(-8).u64 545Label(:CopyLoop) 546 i := Phi(i1, i2).u64 547 If(i, len).AE.Unlikely.b { 548 Goto(:End) 549 } 550 Store(dst, i, Load(src, i).u16).u16 551 i2 := AddI(i).Imm(2).u64 552 Goto(:CopyLoop) 553 554Label(:End) 555end # copy_u16_chars 556 557### 558# Copy u8 chars expanding them to u16 chars 559# 560scoped_macro(:expand_u8_to_u16_chars) do |src, dst, count| 561 i0 := Cast(0).u64 562 len := Cast(count).u64 563Label(:CopyLoop) 564 i := Phi(i0, i1).u64 565 If(i, len).AE.Unlikely.b { 566 Goto(:End) 567 } 568 Store(dst, ShlI(i).Imm(1).u64, Cast(Load(src, i).u8).u16).u16 569 i1 := AddI(i).Imm(1).u64 570 Goto(:CopyLoop) 571Label(:End) 572end # expand_u8_to_u16_chars 573 574### 575# Copy data from compressed string to array of utf16 chars 576# 577scoped_macro(:copy_compressed_string_to_array_of_chars) do |str_data, arr_data, char_count| 578 # String contains 8-bit chars 579 If(char_count, 0).A.Likely.b { 580 i1 := Cast(0).u64 581Label(:CopyLoop) 582 i := Phi(i1, i2).u64 583 Store(arr_data, ShlI(i).Imm(1).u64, Cast(Load(str_data, i).u8).u16).u16 584 i2 := AddI(i).Imm(1).u64 585 If(i2, char_count).B.Likely.b { 586 Goto(:CopyLoop) 587 } 588 } 589end # copy_compressed_string_to_array_of_chars 590 591# calculate the difference between string chunks 592# stored in 64-bits numbers, byte/hword swap is 593# required before comparison for the correct result 594scoped_macro(:calculate_chars_difference) do |d1, d2, utf| 595 data1 := Cast(d1).u64 596 data2 := Cast(d2).u64 597 598 IfImm(Compare(utf, 0).EQ.b).Imm(0).NE.b { 599 ldata1 := Intrinsic(:REVERSE_BYTES_U64, data1).u64 600 ldata2 := Intrinsic(:REVERSE_BYTES_U64, data2).u64 601 cmp1 := Cmp(ldata1, ldata2).i32 602 } Else { 603 udata1 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(Cast(data1).u64).f64).f64).u64 604 udata2 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(Cast(data2).u64).f64).f64).u64 605 cmp2 := Cmp(udata1, udata2).i32 606 } 607 ret := Phi(cmp1, cmp2).i32 608end 609 610scoped_macro(:compare_LU_1_chars) do |lat, utf| 611 ret := Sub(Cast(lat).u32, Cast(utf).u32).i32 612end 613 614scoped_macro(:compare_LU_2_chars) do |lat, utf| 615 res1 := ShlI(Sub(AndI(lat).Imm(0xff).u32, AndI(utf).Imm(0xffff).u32).i32).Imm(16).u32 616 res2 := Sub(ShrI(lat).Imm(8).u32, ShrI(utf).Imm(16).u32).i32 617 res := Add(res1, res2).i32 618end 619 620scoped_macro(:compare_LU_strings) do |lat, utf, len| 621 IfImm(Compare(len, 2).LT.b).Imm(0).NE.b { 622 ret1 := compare_LU_1_chars(LoadI(lat).Imm(0).u8, LoadI(utf).Imm(0).u16).i32 623 Goto(:End) 624 } 625 626 IfImm(Compare(len, 3).LT.b).Imm(0).NE.b { 627 ret2 := compare_LU_2_chars(LoadI(lat).Imm(0).u16, LoadI(utf).Imm(0).u32).i32 628 Goto(:End) 629 } 630 631 i1 := Cast(0).u64 632Label(:Loop) 633 i := Phi(i1, i2).u64 634 lv1 := Load(lat, i).u32 635 v1 := Bitcast(Intrinsic(:EXPAND_U8_TO_U16, Bitcast(lv1).f32).f64).u64 636 u1 := Load(utf, ShlI(i).Imm(1).u64).u64 637 If(u1, v1).NE.b { 638 Goto(:NotEqual) 639 } 640 641 lv2 := Load(lat, AddI(i).Imm(4).u32).u32 642 v2 := Bitcast(Intrinsic(:EXPAND_U8_TO_U16, Bitcast(lv2).f32).f64).u64 643 u2 := Load(utf, AddI(ShlI(i).Imm(1).u64).Imm(8).u64).u64 644 If(u2, v2).NE.b { 645 Goto(:NotEqual) 646 } 647 648 i2 := AddI(i).Imm(8).u64 649 If(i, len).B.b { 650 Goto(:Loop) 651 } 652 653 # the version assuming 128-bits types support 654 # 655 # v := Intrinsic(:EXPAND_U8_TO_U16, Load(lat, i).f64).f128 656 # u := Load(utf, i).f128 657 # v1 = Intrinsic(:GET_LOW_PART, v).u64 658 # u1 = Intrinsic(:GET_LOW_PART, u).u64 659 # n11 := Bitcast(v1).u64 660 # n21 := Bitcast(u1).u64 661 # If(n11, n21).NE.b { 662 # d11 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, v1).f64).u64 663 # d21 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, u1).f64).u64 664 # Goto(:NotEqual) 665 # } 666 # v2 = Intrinsic(:GET_HIGH_PART, v).u64 667 # u2 = Intrinsic(:GET_HIGH_PART, u).u64 668 # n12 := Bitcast(v2).u64 669 # n22 := Bitcast(u2).u64 670 # If(n12, n22).NE.b { 671 # d12 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, v2).f64).u64 672 # d22 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, u2).f64).u64 673 # Goto(:NotEqual) 674 # } 675 # 676 # Label(:NotEqual) 677 # d1 := Phi(d11, d12) 678 # d2 := Phi(d21, d22) 679 # res := Sub(d1, d2).u64 680 681 ret0 := -1 # the prefix is the same but the utf string has to be 682 # longer as it would have had latin encoding otherwise 683 Goto(:End) 684 685Label(:NotEqual) 686 u3 := Phi(u1, u2).u64 687 v3 := Phi(v1, v2).u64 688 d1 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(v3).f64).f64).u64 689 d2 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(u3).f64).f64).u64 690 ret3 := Cmp(d1, d2).i32 691 692Label(:End) 693 ret := Phi(ret1, ret2, ret0, ret3).i32 694end 695 696scoped_macro(:compare_mixed_strings) do |buf1, buf2, len, utf_is_first| 697 IfImm(Compare(utf_is_first, 0).NE.b).Imm(0).EQ.b { 698 ret1 := compare_LU_strings(buf1, buf2, len) 699 } Else { 700 ret2 := Neg(compare_LU_strings(buf2, buf1, len).i32).i32 701 } 702 ret := Phi(ret1, ret2).i32 703end 704 705def GenerateCreateStringFromCharArrayTlab(string_compression_enabled) 706 suffix = (string_compression_enabled ? "Compressed" : "") 707 available_regs = $panda_mask 708 function("CreateStringFromCharArrayTlab#{suffix}".to_sym, 709 params: {char_offset: 'u32', char_count: 'u32', char_array: 'ref', string_klass: 'ref'}, 710 regmap: $full_regmap, 711 regalloc_set: available_regs, 712 mode: [:FastPath]) { 713 714 if Options.arch == :arm32 715 Intrinsic(:UNREACHABLE).Terminator.void 716 ReturnVoid().void 717 next 718 end 719 720 # There is no check of the arguments against NullPointer as 721 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 722 arr_offs := AddI(ShlI(Cast(char_offset).u64).Imm(1).u64).Imm(Constants::ARRAY_DATA_OFFSET).u64 723 arr_data := Add(Cast(char_array).SrcType(Constants::COMPILER_REFERENCE).ptr, arr_offs).ptr 724 725 # Allocate a new string 726 if string_compression_enabled 727 compressable := is_array_of_compressable_chars(arr_data, Cast(char_count).u64) 728 If(compressable, 1).EQ.Likely.b { 729 data_size1 := Cast(char_count).word 730 } Else { 731 data_size2 := Cast(ShlI(char_count).Imm(1).u32).word 732 } 733 data_size := Phi(data_size1, data_size2).word 734 else 735 data_size := Cast(ShlI(char_count).Imm(1).u32).word 736 end 737 new_str := allocate_string_tlab(string_klass, data_size) 738 str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 739 740 # Copy data from char_array to the new string 741 # String length field is set according to SetLength() from runtime/include/coretypes/string.h 742 if string_compression_enabled 743 If(compressable, 1).EQ.Likely.b { 744 compress_u16_to_u8_chars(arr_data, str_data, Cast(char_count).u64) 745 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 746 } Else { 747 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 748 StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 749 } 750 else 751 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 752 StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 753 end 754 # String is supposed to be a constant object, so all its data should be visible by all threads 755 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 756 Return(new_str).ptr 757 758 Label(:SlowPathEntrypoint) 759 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_CHAR_ARRAY_SLOW_PATH") 760 Intrinsic(:SLOW_PATH_ENTRY, char_offset, char_count, char_array).AddImm(entrypoint).MethodAsImm("CreateStringFromCharArray4ArgBridge").Terminator.ptr 761 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 762 } 763end # def GenerateCreateStringFromCharArrayTlab 764 765 766def GenerateCreateStringFromZeroBasedCharArrayTlab(string_compression_enabled) 767 suffix = (string_compression_enabled ? "Compressed" : "") 768 available_regs = $panda_mask 769 function("CreateStringFromZeroBasedCharArrayTlab#{suffix}".to_sym, 770 params: {char_count: 'u32', char_array: 'ref', string_klass: 'ref'}, 771 regmap: $full_regmap, 772 regalloc_set: available_regs, 773 mode: [:FastPath]) { 774 775 if Options.arch == :arm32 776 Intrinsic(:UNREACHABLE).Terminator.void 777 ReturnVoid().void 778 next 779 end 780 781 # There is no check of the arguments against NullPointer as 782 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 783 arr_data := Add(Cast(char_array).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr 784 785 # Allocate a new string 786 if string_compression_enabled 787 compressable := is_array_of_compressable_chars(arr_data, Cast(char_count).u64) 788 If(compressable, 1).EQ.Likely.b { 789 data_size1 := Cast(char_count).word 790 } Else { 791 data_size2 := Cast(ShlI(char_count).Imm(1).u32).word 792 } 793 data_size := Phi(data_size1, data_size2).word 794 else 795 data_size := Cast(ShlI(char_count).Imm(1).u32).word 796 end 797 new_str := allocate_string_tlab(string_klass, data_size) 798 str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 799 800 # Copy data from char_array to the new string 801 # String length field is set according to SetLength() from runtime/include/coretypes/string.h 802 if string_compression_enabled 803 If(compressable, 1).EQ.Likely.b { 804 compress_u16_to_u8_chars(arr_data, str_data, Cast(char_count).u64) 805 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 806 } Else { 807 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 808 StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 809 } 810 else 811 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 812 StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 813 end 814 # String is supposed to be a constant object, so all its data should be visible by all threads 815 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 816 Return(new_str).ptr 817 818 Label(:SlowPathEntrypoint) 819 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_SLOW_PATH") 820 Intrinsic(:SLOW_PATH_ENTRY, char_count, char_array).AddImm(entrypoint).MethodAsImm("CreateStringFromZeroBasedCharArray3ArgBridge").Terminator.ptr 821 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 822 } 823end # def GenerateCreateStringFromZeroBasedCharArrayTlab 824 825 826def GenerateSubstringFromStringTlab(string_compression_enabled) 827 suffix = (string_compression_enabled ? "Compressed" : "") 828 available_regs = $panda_mask 829 function("SubStringFromStringTlab#{suffix}".to_sym, 830 params: {str: 'ref', begin_index: 'i32', end_index: 'i32'}, 831 regmap: $full_regmap, 832 regalloc_set: available_regs, 833 mode: [:FastPath]) { 834 835 if Options.arch == :arm32 836 Intrinsic(:UNREACHABLE).Terminator.void 837 ReturnVoid().void 838 next 839 end 840 841 # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded) 842 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 843 if string_compression_enabled 844 length := ShrI(length_packed).Imm(1).u32 845 else 846 length := length_packed 847 end 848 849 # If begin_index < 0, then it is assumed to be equal to zero 850 If(begin_index, Cast(0).i32).LT.Unlikely.b { 851 bx1 := Cast(0).i32 852 } 853 bx2 := Phi(begin_index, bx1).u32 854 # If end_index < 0, then it is assumed to be equal to zero 855 If(end_index, Cast(0).i32).LT.Unlikely.b { 856 ex1 := Cast(0).i32 857 } 858 ex2 := Phi(end_index, ex1).u32 859 # If begin_index > str.length(), then make it equal to str.length() 860 If(bx2, length).A.Unlikely.b { 861 bx3 := length 862 } 863 bx4 := Phi(bx2, bx3).u32 864 # If end_index > str.length(), then make it equal to str.length() 865 If(ex2, length).A.Unlikely.b { 866 ex3 := length 867 } 868 ex4 := Phi(ex2, ex3).u32 869 # If begin_index > end_index, then swap them. 870 If(bx4, ex4).GT.Unlikely.b { 871 bx5 := ex4 872 ex5 := bx4 873 } 874 bx6 := Phi(bx4, bx5).u32 875 ex6 := Phi(ex4, ex5).u32 876 877 If(bx6, 0).EQ.Likely.b { 878 If(ex6, length).EQ.Unlikely.b { 879 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 880 } 881 } 882 883 if string_compression_enabled 884 not_compressed := AndI(length_packed).Imm(1).u32 885 offset := Shl(bx6, not_compressed).u32 886 else 887 offset := ShlI(bx6).Imm(1).u32 888 end 889 890 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 891 src_str_data := Add(src_str_data, Cast(offset).u64).ptr 892 893 klass := LoadI(str).Imm(Constants::OBJECT_CLASS_OFFSET).ref 894 char_count := Sub(ex6, bx6).u32 895 896 # Allocate a new string 897 if string_compression_enabled 898 If(not_compressed, 1).EQ.Unlikely.b { 899 compressable := is_array_of_compressable_chars(src_str_data, Cast(char_count).u64) 900 If(compressable, 1).EQ.Likely.b { 901 data_size1 := Cast(char_count).word 902 } Else { 903 data_size2 := Cast(ShlI(char_count).Imm(1).u32).word 904 } 905 data_size := Phi(data_size1, data_size2).word 906 new_str := allocate_string_tlab(klass, data_size) 907 new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 908 If(compressable, 1).EQ.Likely.b { 909 compress_u16_to_u8_chars(src_str_data, new_str_data, Cast(char_count).u64) 910 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 911 } Else { 912 copy_u16_chars(src_str_data, new_str_data, Cast(char_count).u64) 913 StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 914 } 915 # String is supposed to be a constant object, so all its data should be visible by all threads 916 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 917 Return(new_str).ptr 918 } Else { 919 # Source string is already compressed 920 new_str := allocate_string_tlab(klass, Cast(char_count).word) 921 new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 922 copy_u8_chars(src_str_data, new_str_data, Cast(char_count).u64) 923 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 924 # String is supposed to be a constant object, so all its data should be visible by all threads 925 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 926 Return(new_str).ptr 927 } 928 else 929 data_size := Cast(ShlI(char_count).Imm(1).u32).word 930 new_str := allocate_string_tlab(klass, data_size) 931 new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 932 copy_u16_chars(src_str_data, new_str_data, Cast(char_count).u64) 933 StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 934 # String is supposed to be a constant object, so all its data should be visible by all threads 935 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 936 Return(new_str).ptr 937 end 938 939 Label(:SlowPathEntrypoint) 940 entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH") 941 Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr 942 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 943 } 944end # def GenerateSubstringFromStringTlab 945 946 947def GenerateStringGetCharsTlab(string_compression_enabled) 948 suffix = (string_compression_enabled ? "Compressed" : "") 949 available_regs = $panda_mask 950 function("StringGetCharsTlab#{suffix}".to_sym, 951 params: {str: 'ref', begin_index: 'i32', end_index: 'i32', array_klass: 'ref'}, 952 regmap: $full_regmap, 953 regalloc_set: available_regs, 954 mode: [:FastPath]) { 955 956 if Options.arch == :arm32 957 Intrinsic(:UNREACHABLE).Terminator.void 958 ReturnVoid().void 959 next 960 end 961 962 If(begin_index, end_index).GT.Unlikely.b { 963 Goto(:SlowPathEntrypoint) # Out of range 964 } 965 If(begin_index, Cast(0).i32).LT.Unlikely.b { 966 Goto(:SlowPathEntrypoint) # Out of range 967 } 968 969 # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded) 970 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32; 971 if string_compression_enabled 972 If(Cast(end_index).u32, ShrI(length).Imm(1).u32).A.Unlikely.b { 973 Goto(:SlowPathEntrypoint) # Out of range 974 } 975 not_compressed := AndI(length).Imm(1).u32 976 offset := Shl(begin_index, not_compressed).u32 977 else 978 If(Cast(end_index).u32, length).A.Unlikely.b { 979 Goto(:SlowPathEntrypoint) # Out of range 980 } 981 not_compressed := Cast(1).u32 982 offset := ShlI(begin_index).Imm(1).u32 983 end 984 985 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 986 src_str_data := Add(src_str_data, Cast(offset).u64).ptr 987 988 # Allocate a new array of u16 chars 989 char_count := Sub(Cast(end_index).u32, Cast(begin_index).u32).u64 990 new_arr := allocate_array_of_chars_tlab(array_klass, Cast(char_count).word) 991 new_arr_data := Add(new_arr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr 992 If(not_compressed, Cast(0).u32).EQ.Likely.b { 993 expand_u8_to_u16_chars(src_str_data, new_arr_data, char_count) 994 } 995 If(not_compressed, Cast(1).u32).EQ.Unlikely.b { 996 copy_u16_chars(src_str_data, new_arr_data, char_count) 997 } 998 # String is supposed to be a constant object, so all its data should be visible by all threads 999 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 1000 Return(new_arr).ptr 1001 1002 Label(:SlowPathEntrypoint) 1003 entrypoint = get_entrypoint_offset("STRING_GET_CHARS_SLOW_PATH") 1004 Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("StringGetChars4ArgBridge").Terminator.ptr 1005 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 1006 } 1007end # def GenerateStringGetCharsTlab 1008 1009 1010### 1011# Compute string hashcode 1012# 1013scoped_macro(:u8_string_hashcode) do |str_data, char_count| 1014 hash := Cast(0).u32 1015 If(char_count, 0).A.Likely.b { 1016 i1 := Cast(0).u32 1017 imm31 := Cast(31).u32 1018Label(:Loop_hash) 1019 hash1 := Phi(hash, hash2).u32 1020 i := Phi(i1, i2).u32 1021 ch := Cast(Load(str_data, i).u8).SrcType(Constants::COMPILER_UINT8).u32 1022 hash2 := Add(Mul(hash1, imm31).u32, ch).u32 1023 i2 := AddI(i).Imm(1).u32 1024 If(i2, char_count).B.Likely.b { 1025 Goto(:Loop_hash) 1026 } 1027 } 1028 hashcode := Phi(hash, hash2).u32 1029end # u8_string_hashcode 1030 1031scoped_macro(:u16_string_hashcode) do |str_data, char_count| 1032 hash := Cast(0).u32 1033 If(char_count, 0).A.Likely.b { 1034 i1 := Cast(0).u64 1035 imm31 := Cast(31).u32 1036 stop := ShlI(Cast(char_count).u64).Imm(1).u64 1037Label(:Loop_hash) 1038 hash1 := Phi(hash, hash2).u32 1039 i := Phi(i1, i2).u64 1040 ch := Cast(Load(str_data, i).u16).SrcType(Constants::COMPILER_UINT16).u32 1041 hash2 := Add(Mul(hash1, imm31).u32, ch).u32 1042 i2 := AddI(i).Imm(2).u64 1043 If(i2, stop).B.Likely.b { 1044 Goto(:Loop_hash) 1045 } 1046 } 1047 hashcode := Phi(hash, hash2).u32 1048end # u16_string_hashcode 1049 1050 1051def GenerateStringHashCode(string_compression_enabled) 1052 suffix = (string_compression_enabled ? "Compressed" : "") 1053if Options.arch == :arm64 1054 available_regs = $temps_mask + :arg0 + :callee0 + :callee1 + :callee2 + :callee3 1055else 1056 available_regs = $temps_mask + :arg0 + :callee0 + :caller0 + :caller1 1057end 1058 function("StringHashCode#{suffix}".to_sym, 1059 params: {str: 'ref'}, 1060 regmap: $full_regmap, 1061 regalloc_set: available_regs, 1062 mode: [:FastPath]) { 1063 1064 if Options.arch == :arm32 1065 Intrinsic(:UNREACHABLE).Terminator.void 1066 ReturnVoid().void 1067 next 1068 end 1069 1070 # 1. There is no check of the argument against NullPointer as 1071 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 1072 # 2. Don't check if hashcode is equal to 0 as it's done in the codegen. 1073 1074 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 1075 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 1076 if string_compression_enabled 1077 length := ShrI(length_packed).Imm(1).u32 1078 not_compressed := AndI(length_packed).Imm(1).u32 1079 If(not_compressed, 0).EQ.Likely.b { 1080 # String contains 8-bit chars 1081 h := u8_string_hashcode(str_data, length) 1082 StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32 1083 Return(h).u32 1084 } Else { 1085 # String contains 16-bit chars 1086 h := u16_string_hashcode(str_data, length) 1087 StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32 1088 Return(h).u32 1089 } 1090 else 1091 # String contains 16-bit chars 1092 h := u16_string_hashcode(str_data, length_packed) 1093 StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32 1094 Return(h).u32 1095 end 1096 } 1097end # def GenerateStringHashCode 1098