1#!/usr/bin/env ruby 2 3# Copyright (c) 2021-2024 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16macro(:load_diff) do |ptr1, ptr2, offset| 17 Xor(LoadI(ptr1).Imm(Constants::STRING_DATA_OFFSET + " + " + offset.to_s).u64, 18 LoadI(ptr2).Imm(Constants::STRING_DATA_OFFSET + " + " + offset.to_s).u64).u64 19end 20 21scoped_macro(:unpack_length) do |length, compression, length_shift| 22 if compression 23 char_length := ShrI(length).Imm(length_shift).u64 24 not_compressed := AndI(length).Imm(1).u64 25 unpacked_length := Shl(char_length, not_compressed).u64 26 elsif length_shift != 0 27 unpacked_length := ShrI(length).Imm(length_shift).u64 28 end 29end 30 31def GenerateStringEquals(lang, dynamic, compression) 32 suffix = (compression ? "Compressed" : "") 33 length_shift = (dynamic ? 2 : 1) 34 mode = [:FastPath] 35 mode.push(:DynamicMethod, :DynamicStub) if dynamic 36 37 reg_mask = Options.arch == :arm64 ? 38 RegMask.new($full_regmap, :arg0, :arg1, :callee0, :callee1, :callee2, :callee3, :callee4, :callee5, :callee6, :callee7, :tmp0, :tmp1) : 39 $panda_mask + :tmp0 + :tmp1 + :tr 40 41 function("#{lang}StringEqualsUnroll#{suffix}".to_sym, 42 params: {str1_ref: 'ref', str2_ref: 'ref'}, 43 regmap: $full_regmap, 44 regalloc_set: reg_mask, 45 mode: mode, 46 lang: lang.empty? ? 'PANDA_ASSEMBLY' : lang.upcase) { 47 if Options.arch == :arm32 48 Intrinsic(:UNREACHABLE).void.Terminator 49 next 50 end 51 str1 := Cast(str1_ref).SrcType(Constants::COMPILER_REFERENCE).ptr 52 str2 := Cast(str2_ref).SrcType(Constants::COMPILER_REFERENCE).ptr 53 # Compare first 32 bytes 54 buf1 := LoadI(str1).Imm(Constants::STRING_DATA_OFFSET).u64 55 buf2 := LoadI(str2).Imm(Constants::STRING_DATA_OFFSET).u64 56 If(buf1, buf2).NE.Unlikely.b { 57 Goto(:NotEqual) 58 } 59 diff := load_diff(str1, str2, 8) 60 diff := Or(diff, load_diff(str1, str2, 16)).u64 61 diff := Or(diff, load_diff(str1, str2, 24)).u64 62 If(diff, 0).NE.Unlikely.b { 63 Goto(:NotEqual) 64 } 65 length := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32 66 length := Cast(length).SrcType("DataType::UINT32").u64 67 length := unpack_length(length, compression, length_shift) 68 last_buf_index := SubI(length).Imm(32).u64 69 last_ptr1 := Add(str1, last_buf_index).ptr 70 last_ptr2 := Add(str2, last_buf_index).ptr 71 first_ptr1 := AddI(str1).Imm(32).ptr 72 first_ptr2 := AddI(str2).Imm(32).ptr 73 Label(:Loop) 74 ptr1 := Phi(first_ptr1, next_ptr1).ptr 75 ptr2 := Phi(first_ptr2, next_ptr2).ptr 76 diff := load_diff(ptr1, ptr2, 0) 77 diff := Or(diff, load_diff(ptr1, ptr2, 8)).u64 78 diff := Or(diff, load_diff(ptr1, ptr2, 16)).u64 79 diff := Or(diff, load_diff(ptr1, ptr2, 24)).u64 80 If(diff, 0).NE.Unlikely.b { 81 Goto(:NotEqual) 82 } 83 next_ptr1 := Add(ptr1, 32).ptr 84 If(next_ptr1, last_ptr1).GE.Unlikely.b { 85 diff := load_diff(last_ptr1, last_ptr2, 0) 86 diff := Or(diff, load_diff(last_ptr1, last_ptr2, 8)).u64 87 diff := Or(diff, load_diff(last_ptr1, last_ptr2, 16)).u64 88 diff := Or(diff, load_diff(last_ptr1, last_ptr2, 24)).u64 89 If(diff, 0).NE.Unlikely.b { 90 Goto(:NotEqual) 91 } 92 Return(1).b 93 } 94 next_ptr2 := Add(ptr2, 32).ptr 95 Goto(:Loop) 96 Label(:NotEqual) 97 Return(0).b 98 } 99 100 reg_mask = Options.arch == :arm64 ? RegMask.new($full_regmap, :arg0, :arg1, :tmp0, :tmp1, :callee0, :callee2) : 101 RegMask.new($full_regmap, :arg0, :arg1, :tmp0, :tmp1, :callee0, :caller1) 102 103 function("#{lang}StringEquals#{suffix}".to_sym, 104 params: {str1: 'ref', str2: 'ref'}, 105 regmap: $full_regmap, 106 regalloc_set: reg_mask, 107 mode: mode, 108 lang: lang.empty? ? 'PANDA_ASSEMBLY' : lang.upcase) { 109 # Arm32 is not supported 110 # length shift should be from 0 to 2 111 if Options.arch == :arm32 || length_shift > 2 112 Intrinsic(:UNREACHABLE).void.Terminator 113 next 114 end 115 unless dynamic 116 If(str2, 0).EQ.Unlikely.b { 117 Goto(:NotEqual) 118 } 119 end 120 If(str1, str2).EQ.Unlikely.b { 121 Return(1).b 122 } 123 if dynamic 124 length1 := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32 125 length2 := LoadI(str2).Imm(Constants::STRING_LENGTH_OFFSET).u32 126 length1 := AndI(length1).Imm("~(2U)").u32 127 length2 := AndI(length2).Imm("~(2U)").u32 128 If(length1, length2).NE.Unlikely.b { 129 Goto(:NotEqual) 130 } 131 length := Cast(length1.u32).u64 132 else 133 # assume STRING_LENGTH_OFFSET == OBJECT_CLASS_OFFSET + 4 134 class_and_length1 := LoadI(str1).Imm(Constants::OBJECT_CLASS_OFFSET).u64 135 class_and_length2 := LoadI(str2).Imm(Constants::OBJECT_CLASS_OFFSET).u64 136 If(class_and_length1, class_and_length2).NE.Unlikely.b { 137 Goto(:NotEqual) 138 } 139 length := Shr(class_and_length1, 32).u64 140 end 141 length := unpack_length(length, compression, length_shift) 142 If(length, 8).GT.b { 143 Goto(:Long) 144 } 145 odd_bytes := Sub(8, length).u64 146 last_idx := Sub(Constants::STRING_DATA_OFFSET, odd_bytes).u64 147 buf1 := Load(str1, last_idx).u64 148 buf2 := Load(str2, last_idx).u64 149 odd_bits := Shl(odd_bytes, 3).u64 150 diff := Shr(Xor(buf1, buf2).u64, odd_bits).u64 151 res := Compare(diff, 0).EQ.b 152 if dynamic 153 # If length is 0, odd_bits is 64 and Shr above does nothing, so we effectively compare last 8 bytes of two strings 154 # before their data (length and hash code). Hash code for empty string is always 0, but value stored in length field 155 # of equal strings can be different in dynamic implementation, so we check (length == 0) separately 156 res := Or(res, Compare(length, 0).EQ.b).b 157 end 158 Return(res).b 159 160 Label(:Long) 161 unroll := Compare(length, 64).GE.b 162 IfImm(unroll).Imm(0).SrcType("DataType::BOOL").NE.b { 163 LiveOut(str1).DstReg(regmap[:arg0]).ref 164 LiveOut(str2).DstReg(regmap[:arg1]).ref 165 entrypoint_id = "#{lang.empty? ? '' : lang.upcase + '_'}STRING_EQUALS_UNROLL" + (compression ? "_COMPRESSED" : ""); 166 entrypoint_offset = get_entrypoint_offset(entrypoint_id) 167 entrypoint_name = "#{lang.empty? ? '' : lang}StringEqualsUnroll" + (compression ? "Compressed" : ""); 168 Intrinsic(:TAIL_CALL).AddImm(entrypoint_offset).MethodAsImm(entrypoint_name).Terminator.b 169 } 170 first_idx := Constants::STRING_DATA_OFFSET 171 last_idx := Sub(Add(first_idx, length).u64, 8).u64 172 173 Label(:Loop) 174 idx := Phi(first_idx, next_idx).u64 175 buf1 := Load(str1, idx).u64 176 buf2 := Load(str2, idx).u64 177 If(buf1, buf2).NE.Unlikely.b { 178 Goto(:NotEqual) 179 } 180 next_idx := Add(idx, 8).u64 181 If(next_idx, last_idx).GE.Unlikely.b { 182 buf1 := Load(str1, last_idx).u64 183 buf2 := Load(str2, last_idx).u64 184 res := Compare(buf1, buf2).EQ.b 185 Return(res).b 186 } 187 Goto(:Loop) 188 Label(:NotEqual) 189 Return(0).b 190 } 191end 192 193 194# Try to allocate String in TLAB. 195# The result is either a pointer to a new string or null if there is no enough space in TLAB. 196macro(:allocate_string_tlab) do |string_klass, data_size| 197 if Options.arch == :arm32 198 Intrinsic(:UNREACHABLE).Terminator.void 199 ReturnVoid().void 200 next 201 end 202 203 # Add sizeof(String) and do align 204 _data_size := Cast(data_size).word 205 _size := AndI(AddI(_data_size).Imm(Constants::STRING_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word 206 # Load pointer to the TLAB from TLS 207 _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr 208 # Load pointer to the start address of free memory in the TLAB 209 _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr 210 # Load pointer to the end address of free memory in the TLAB 211 _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr 212 # Check if there is enough space 213 If(Sub(_end, _start).word, _size).B.Unlikely.b { 214 Goto(:SlowPathEntrypoint) 215 } 216 Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG 217 if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__ 218 call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void 219 end 220 # Store class of the object 221 StoreI(_start, string_klass).Imm(Constants::OBJECT_CLASS_OFFSET).ref 222 # Update the TLAB state 223 StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr 224 # Return a pointer to the newly allocated string 225 _allocated_string := _start 226end 227 228# Try to allocate Array of u16 chars in TLAB. 229# The result is either a pointer to a new array or null if there is no enough space in TLAB. 230macro(:allocate_array_of_chars_tlab) do |array_klass, char_count| 231 if Options.arch == :arm32 232 Intrinsic(:UNREACHABLE).Terminator.void 233 ReturnVoid().void 234 next 235 end 236 237 elements_num := And(char_count, "0x00000000ffffffff").word 238 # Account for u16 char size 239 _size := Shl(elements_num, 1).word 240 # Add sizeof(Array) and do align 241 _size := AndI(AddI(_size).Imm(Constants::ARRAY_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word 242 # Load pointer to the TLAB from TLS 243 _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr 244 # Load pointer to the start address of free memory in the TLAB 245 _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr 246 # Load pointer to the end address of free memory in the TLAB 247 _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr 248 # Check if there is enough space 249 If(Sub(_end, _start).word, _size).B.Unlikely.b { 250 Goto(:SlowPathEntrypoint) 251 } 252 Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG 253 if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__ 254 call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void 255 end 256 # Store class of the object 257 StoreI(_start, array_klass).Imm(Constants::OBJECT_CLASS_OFFSET).ref 258 # Store array length 259 StoreI(_start, elements_num).Imm(Constants::ARRAY_LENGTH_OFFSET).word 260 # Update the TLAB state 261 StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr 262 # Return a pointer to the newly allocated array 263 _allocated_array := _start 264end 265 266 267def GenerateCreateStringFromStringTlab(string_compression_enabled) 268 suffix = (string_compression_enabled ? "Compressed" : "") 269 available_regs = $panda_mask 270 function("CreateStringFromStringTlab#{suffix}".to_sym, 271 params: {str: 'ref'}, 272 regmap: $full_regmap, 273 regalloc_set: available_regs, 274 mode: [:FastPath]) { 275 276 if Options.arch == :arm32 277 Intrinsic(:UNREACHABLE).Terminator.void 278 ReturnVoid().void 279 next 280 end 281 282 # There is no check of the argument against NullPointer as 283 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 284 klass := LoadI(str).Imm(Constants::OBJECT_CLASS_OFFSET).ref 285 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 286 hashcode := LoadI(str).Imm(Constants::STRING_HASHCODE_OFFSET).u32 287 data_size := unpack_length(Cast(length).u64, string_compression_enabled, 1) 288 289 new_str := allocate_string_tlab(klass, Cast(data_size).word) 290 StoreI(new_str, length).Imm(Constants::STRING_LENGTH_OFFSET).u32 291 StoreI(new_str, hashcode).Imm(Constants::STRING_HASHCODE_OFFSET).u32 292 293 # Copy string data 294 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 295 dst_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 296 offs := Cast(0).u64 297 If(data_size, 8).AE.Likely.b { 298 stop := AndI(data_size).Imm(7).u64 299 Label(:CopyLoop_8b) 300 offs1 := Phi(offs, offs2).u64 301 Store(dst_str_data, offs, Load(src_str_data, offs).u64).u64 302 offs2 := AddI(offs1).Imm(8).u64 303 If(offs2, stop).B.Likely.b { 304 Goto(:CopyLoop_8b) 305 } 306 } 307 offs3 := Phi(offs, offs2).u64 308 If(offs3, data_size).B.Likely.b { 309 Label(:CopyLoop_1b) 310 offs4 := Phi(offs3, offs5).u64 311 Store(dst_str_data, offs4, Load(src_str_data, offs4).u8).u8 312 offs5 := AddI(offs4).Imm(1).u64 313 If(offs5, data_size).B.Likely.b { 314 Goto(:CopyLoop_1b) 315 } 316 } 317 318 # String is supposed to be a constant object, so all its data should be visible by all threads 319 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 320 Return(new_str).ptr 321 322 Label(:SlowPathEntrypoint) 323 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_STRING_SLOW_PATH") 324 Intrinsic(:SLOW_PATH_ENTRY, str).AddImm(entrypoint).MethodAsImm("CreateStringFromStringOddSavedBridge").Terminator.ptr 325 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 326 } 327end # def GenerateCreateStringFromStringTlab 328 329 330### 331# Checks if starting from arr_data the specified number of chars (char_count) can be compressed 332# 333# Utf16 char is ASCII if (uft16_char - 1U < utf::UTF8_1B_MAX) 334# See runtime/include/coretypes/string.h - IsASCIICharacter 335# 336scoped_macro(:is_array_of_compressable_chars) do |arr_data, char_count| 337 # Check 4-chars block at once if it's possible 338 n_blocks := ShrI(char_count).Imm(2).u64 339 If(n_blocks, 0).A.Likely.b { 340 # 0x007F is utf::UTF8_1B_MAX 341 utf8_1b_max := Cast(0x007F007F007F007F).u64 342 utf8_1b_max_mask := Not(utf8_1b_max).u64 343 i1 := Cast(0).u64 344 Label(:CanBeCompressedLoop_4chars) 345 i2 := Phi(i1, i3).u64 346 four_chars := Load(arr_data, ShlI(i2).Imm(3).u64).u64 347 # First: check if there are chars greater than utf::UTF8_1B_MAX 348 If(And(four_chars, utf8_1b_max_mask).u64, 0).NE.Unlikely.b { 349 compressable1 := Cast(0).b 350 Goto(:CanBeCompressedLoopDone) 351 } 352 # Second: check if there are chars equal to zero 353 If(And(SubI(four_chars).Imm(0x0001000100010001).u64, utf8_1b_max_mask).u64, 0).NE.Unlikely.b { 354 compressable2 := Cast(0).b 355 Goto(:CanBeCompressedLoopDone) 356 } 357 i3 := AddI(i2).Imm(1).u64 358 If(i3, n_blocks).B.Likely.b { 359 Goto(:CanBeCompressedLoop_4chars) 360 } 361 } 362 # check the rest of the chars 363 If(AndI(char_count).Imm(3).u64, 0).A.Likely.b { 364 i4 := ShlI(n_blocks).Imm(2).u64 # number of already copied chars if any 365Label(:CanBeCompressedLoop) 366 i5 := Phi(i4, i6).u64 367 ch := Load(arr_data, ShlI(i5).Imm(1).u64).u16 368 If(SubI(ch).Imm(1).u16, Cast(Constants::STRING_MUTF8_1B_MAX).u16).AE.Unlikely.b { 369 compressable3 := Cast(0).b 370 Goto(:CanBeCompressedLoopDone) 371 } 372 i6 := AddI(i5).Imm(1).u64 373 If(i6, char_count).B.Likely.b { 374 Goto(:CanBeCompressedLoop) 375 } 376 } 377 compressable4 := Cast(1).b 378Label(:CanBeCompressedLoopDone) 379 compressable := Phi(compressable1, compressable2, compressable3, compressable4).b 380end # is_array_of_compressable_chars 381 382 383### 384# Copy u16 chars compressing them to u8. 385# It is assumed that all u16 chars are compressable. 386# 387scoped_macro(:compress_u16_to_u8_chars) do |src_data, dst_data, char_count| 388if Options.arch == :arm64 389 # Copy 32-byte chunks (if any) compressing them into 16-byte chunks 390 If(char_count, 16).AE.Likely.b { 391 stop := Add(src_data, ShlI(ShrI(char_count).Imm(4).u64).Imm(5).u64).ptr 392Label(:CopyLoop_32b) 393 src_data1 := Phi(src_data, src_data2).ptr 394 dst_data1 := Phi(dst_data, dst_data2).ptr 395 Intrinsic(:COMPRESS_SIXTEEN_UTF16_TO_UTF8_CHARS_USING_SIMD, src_data1, dst_data1).void 396 src_data2 := AddI(src_data1).Imm(32).ptr 397 dst_data2 := AddI(dst_data1).Imm(16).ptr 398 If(src_data2, stop).LT.Likely.b { 399 Goto(:CopyLoop_32b) 400 } 401 char_count1 := AndI(char_count).Imm(0xF).u64 402 } 403 src_data3 := Phi(src_data, src_data2).ptr 404 dst_data3 := Phi(dst_data, dst_data2).ptr 405 char_count2 := Phi(char_count, char_count1).u64 406 407 # Copy 16-byte chunk (if any) compressing it into 8-byte chunk 408 If(char_count2, 8).AE.Likely.b { 409 Intrinsic(:COMPRESS_EIGHT_UTF16_TO_UTF8_CHARS_USING_SIMD, src_data3, dst_data3).void 410 src_data4 := AddI(src_data3).Imm(16).ptr 411 dst_data4 := AddI(dst_data3).Imm(8).ptr 412 } 413 src_data5 := Phi(src_data3, src_data4).ptr 414 dst_data5 := Phi(dst_data3, dst_data4).ptr 415 # Copy 2-byte chunks compressing them into 1-byte 416 n_2b := AndI(char_count2).Imm(7).u64 417 If(n_2b, 0).A.Likely.b { 418 j1 := Cast(0).u64 419Label(:CopyLoop_2b) 420 j := Phi(j1, j2).u64 421 Store(dst_data5, j, Load(src_data5, ShlI(j).Imm(1).u64).u8).u8 422 j2 := AddI(j).Imm(1).u64 423 If(j2, n_2b).B.Likely.b { 424 Goto(:CopyLoop_2b) 425 } 426 } 427else # if Options.arch == :arm64 428 # Copy 8-byte chunks compressing them into 4-byte chunks 429 n_8b := ShrI(char_count).Imm(2).u64 430 If(n_8b, 0).A.Likely.b { 431 i1 := Cast(0).u64 432Label(:CopyLoop_8b) 433 i := Phi(i1, i2).u64 434 chunk := Load(src_data, ShlI(i).Imm(3).u64).u64 435 chunk := AndI(Or(chunk, ShrI(chunk).Imm(8).u64).u64).Imm(0x0000ffff0000ffff).u64 436 chunk := Or(chunk, ShrI(chunk).Imm(16).u64).u64 437 Store(dst_data, ShlI(i).Imm(2).u64, Cast(chunk).u32).u32 438 i2 := AddI(i).Imm(1).u64 439 If(i2, n_8b).B.Likely.b { 440 Goto(:CopyLoop_8b) 441 } 442 } 443 # Copy 2-byte chunks compressing them into 1-byte 444 If(AndI(char_count).Imm(3).u64, 0).A.Likely.b { 445 j1 := ShlI(n_8b).Imm(2).u64 # number of already copied chars if any 446Label(:CopyLoop_2b) 447 j := Phi(j1, j2).u64 448 Store(dst_data, j, Load(src_data, ShlI(j).Imm(1).u64).u8).u8 449 j2 := AddI(j).Imm(1).u64 450 If(j2, char_count).B.Likely.b { 451 Goto(:CopyLoop_2b) 452 } 453 } 454end # if Options.arch == :arm64 455end # compress_u16_to_u8_chars 456 457### 458# Copy dwords (8-bytes) 459# 460scoped_macro(:copy_dwords) do |src, dst, size| 461 count := AndI(size).Imm(~0x7).u64 462 i1 := Cast(0).u64 463Label(:CopyLoop_8b) 464 i := Phi(i1, i2).u64 465 If(i, count).AE.Unlikely.b { 466 Goto(:End) 467 } 468 Store(dst, i, Load(src, i).u64).u64 469 i2 := AddI(i).Imm(8).u64 470 Goto(:CopyLoop_8b) 471Label(:End) 472end # copy_dwords 473 474 475### 476# Copy u8 chars from src to dst 477# 478scoped_macro(:copy_u8_chars) do |src, dst, count| 479 # Copy 8-byte chunks 480 len := Cast(count).word 481 copy_dwords(src, dst, len) 482 483 # copy the tail if needed 484 i1 := AndI(len).Imm(~0x7).u64 485Label(:CopyLoop) 486 i := Phi(i1, i2).u64 487 If(i, len).AE.Unlikely.b { 488 Goto(:End) 489 } 490 Store(dst, i, Load(src, i).u8).u8 491 i2 := AddI(i).Imm(1).u64 492 Goto(:CopyLoop) 493 494Label(:End) 495end # copy_u8_chars 496 497### 498# Copy u16 chars from src to dst 499# 500scoped_macro(:copy_u16_chars) do |src, dst, count| 501 # Copy 8-byte chunks 502 len := Cast(ShlI(count).Imm(1).u32).word 503 copy_dwords(src, dst, len) 504 505 # copy the tail if needed 506 i1 := AndI(len).Imm(-8).u64 507Label(:CopyLoop) 508 i := Phi(i1, i2).u64 509 If(i, len).AE.Unlikely.b { 510 Goto(:End) 511 } 512 Store(dst, i, Load(src, i).u16).u16 513 i2 := AddI(i).Imm(2).u64 514 Goto(:CopyLoop) 515 516Label(:End) 517end # copy_u16_chars 518 519### 520# Copy u8 chars expanding them to u16 chars 521# 522scoped_macro(:expand_u8_to_u16_chars) do |src, dst, count| 523 i0 := Cast(0).u64 524 len := Cast(count).u64 525Label(:CopyLoop) 526 i := Phi(i0, i1).u64 527 If(i, len).AE.Unlikely.b { 528 Goto(:End) 529 } 530 Store(dst, ShlI(i).Imm(1).u64, Cast(Load(src, i).u8).u16).u16 531 i1 := AddI(i).Imm(1).u64 532 Goto(:CopyLoop) 533Label(:End) 534end # expand_u8_to_u16_chars 535 536### 537# Copy data from compressed string to array of utf16 chars 538# 539scoped_macro(:copy_compressed_string_to_array_of_chars) do |str_data, arr_data, char_count| 540 # String contains 8-bit chars 541 If(char_count, 0).A.Likely.b { 542 i1 := Cast(0).u64 543Label(:CopyLoop) 544 i := Phi(i1, i2).u64 545 Store(arr_data, ShlI(i).Imm(1).u64, Cast(Load(str_data, i).u8).u16).u16 546 i2 := AddI(i).Imm(1).u64 547 If(i2, char_count).B.Likely.b { 548 Goto(:CopyLoop) 549 } 550 } 551end # copy_compressed_string_to_array_of_chars 552 553# calculate the difference between string chunks 554# stored in 64-bits numbers, byte/hword swap is 555# required before comparison for the correct result 556scoped_macro(:calculate_chars_difference) do |d1, d2, utf| 557 data1 := Cast(d1).u64 558 data2 := Cast(d2).u64 559 560 IfImm(Compare(utf, 0).EQ.b).Imm(0).NE.b { 561 ldata1 := Intrinsic(:REVERSE_BYTES_U64, data1).u64 562 ldata2 := Intrinsic(:REVERSE_BYTES_U64, data2).u64 563 cmp1 := Cmp(ldata1, ldata2).i32 564 } Else { 565 udata1 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(Cast(data1).u64).f64).f64).u64 566 udata2 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(Cast(data2).u64).f64).f64).u64 567 cmp2 := Cmp(udata1, udata2).i32 568 } 569 ret := Phi(cmp1, cmp2).i32 570end 571 572scoped_macro(:compare_LU_1_chars) do |lat, utf| 573 ret := Sub(Cast(lat).u32, Cast(utf).u32).i32 574end 575 576scoped_macro(:compare_LU_2_chars) do |lat, utf| 577 res1 := ShlI(Sub(AndI(lat).Imm(0xff).u32, AndI(utf).Imm(0xffff).u32).i32).Imm(16).u32 578 res2 := Sub(ShrI(lat).Imm(8).u32, ShrI(utf).Imm(16).u32).i32 579 res := Add(res1, res2).i32 580end 581 582scoped_macro(:compare_LU_strings) do |lat, utf, len| 583 IfImm(Compare(len, 2).LT.b).Imm(0).NE.b { 584 ret1 := compare_LU_1_chars(LoadI(lat).Imm(0).u8, LoadI(utf).Imm(0).u16).i32 585 Goto(:End) 586 } 587 588 IfImm(Compare(len, 3).LT.b).Imm(0).NE.b { 589 ret2 := compare_LU_2_chars(LoadI(lat).Imm(0).u16, LoadI(utf).Imm(0).u32).i32 590 Goto(:End) 591 } 592 593 i1 := Cast(0).u64 594Label(:Loop) 595 i := Phi(i1, i2).u64 596 lv1 := Load(lat, i).u32 597 v1 := Bitcast(Intrinsic(:EXPAND_U8_TO_U16, Bitcast(lv1).f32).f64).u64 598 u1 := Load(utf, ShlI(i).Imm(1).u64).u64 599 If(u1, v1).NE.b { 600 Goto(:NotEqual) 601 } 602 603 lv2 := Load(lat, AddI(i).Imm(4).u32).u32 604 v2 := Bitcast(Intrinsic(:EXPAND_U8_TO_U16, Bitcast(lv2).f32).f64).u64 605 u2 := Load(utf, AddI(ShlI(i).Imm(1).u64).Imm(8).u64).u64 606 If(u2, v2).NE.b { 607 Goto(:NotEqual) 608 } 609 610 i2 := AddI(i).Imm(8).u64 611 If(i, len).B.b { 612 Goto(:Loop) 613 } 614 615 # the version assuming 128-bits types support 616 # 617 # v := Intrinsic(:EXPAND_U8_TO_U16, Load(lat, i).f64).f128 618 # u := Load(utf, i).f128 619 # v1 = Intrinsic(:GET_LOW_PART, v).u64 620 # u1 = Intrinsic(:GET_LOW_PART, u).u64 621 # n11 := Bitcast(v1).u64 622 # n21 := Bitcast(u1).u64 623 # If(n11, n21).NE.b { 624 # d11 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, v1).f64).u64 625 # d21 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, u1).f64).u64 626 # Goto(:NotEqual) 627 # } 628 # v2 = Intrinsic(:GET_HIGH_PART, v).u64 629 # u2 = Intrinsic(:GET_HIGH_PART, u).u64 630 # n12 := Bitcast(v2).u64 631 # n22 := Bitcast(u2).u64 632 # If(n12, n22).NE.b { 633 # d12 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, v2).f64).u64 634 # d22 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, u2).f64).u64 635 # Goto(:NotEqual) 636 # } 637 # 638 # Label(:NotEqual) 639 # d1 := Phi(d11, d12) 640 # d2 := Phi(d21, d22) 641 # res := Sub(d1, d2).u64 642 643 ret0 := -1 # the prefix is the same but the utf string has to be 644 # longer as it would have had latin encoding otherwise 645 Goto(:End) 646 647Label(:NotEqual) 648 u3 := Phi(u1, u2).u64 649 v3 := Phi(v1, v2).u64 650 d1 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(v3).f64).f64).u64 651 d2 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(u3).f64).f64).u64 652 ret3 := Cmp(d1, d2).i32 653 654Label(:End) 655 ret := Phi(ret1, ret2, ret0, ret3).i32 656end 657 658scoped_macro(:compare_mixed_strings) do |buf1, buf2, len, utf_is_first| 659 IfImm(Compare(utf_is_first, 0).NE.b).Imm(0).EQ.b { 660 ret1 := compare_LU_strings(buf1, buf2, len) 661 } Else { 662 ret2 := Neg(compare_LU_strings(buf2, buf1, len).i32).i32 663 } 664 ret := Phi(ret1, ret2).i32 665end 666 667def GenerateCreateStringFromCharArrayTlab(string_compression_enabled) 668 suffix = (string_compression_enabled ? "Compressed" : "") 669 available_regs = $panda_mask 670 function("CreateStringFromCharArrayTlab#{suffix}".to_sym, 671 params: {char_offset: 'u32', char_count: 'u32', char_array: 'ref', string_klass: 'ref'}, 672 regmap: $full_regmap, 673 regalloc_set: available_regs, 674 mode: [:FastPath]) { 675 676 if Options.arch == :arm32 677 Intrinsic(:UNREACHABLE).Terminator.void 678 ReturnVoid().void 679 next 680 end 681 682 # There is no check of the arguments against NullPointer as 683 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 684 arr_offs := AddI(ShlI(Cast(char_offset).u64).Imm(1).u64).Imm(Constants::ARRAY_DATA_OFFSET).u64 685 arr_data := Add(Cast(char_array).SrcType(Constants::COMPILER_REFERENCE).ptr, arr_offs).ptr 686 687 # Allocate a new string 688 if string_compression_enabled 689 compressable := is_array_of_compressable_chars(arr_data, Cast(char_count).u64) 690 If(compressable, 1).EQ.Likely.b { 691 data_size1 := Cast(char_count).word 692 } Else { 693 data_size2 := Cast(ShlI(char_count).Imm(1).u32).word 694 } 695 data_size := Phi(data_size1, data_size2).word 696 else 697 data_size := Cast(ShlI(char_count).Imm(1).u32).word 698 end 699 new_str := allocate_string_tlab(string_klass, data_size) 700 str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 701 702 # Copy data from char_array to the new string 703 # String length field is set according to SetLength() from runtime/include/coretypes/string.h 704 if string_compression_enabled 705 If(compressable, 1).EQ.Likely.b { 706 compress_u16_to_u8_chars(arr_data, str_data, Cast(char_count).u64) 707 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 708 } Else { 709 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 710 StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 711 } 712 else 713 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 714 StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 715 end 716 # String is supposed to be a constant object, so all its data should be visible by all threads 717 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 718 Return(new_str).ptr 719 720 Label(:SlowPathEntrypoint) 721 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_CHAR_ARRAY_SLOW_PATH") 722 Intrinsic(:SLOW_PATH_ENTRY, char_offset, char_count, char_array).AddImm(entrypoint).MethodAsImm("CreateStringFromCharArray4ArgBridge").Terminator.ptr 723 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 724 } 725end # def GenerateCreateStringFromCharArrayTlab 726 727 728def GenerateCreateStringFromZeroBasedCharArrayTlab(string_compression_enabled) 729 suffix = (string_compression_enabled ? "Compressed" : "") 730 available_regs = $panda_mask 731 function("CreateStringFromZeroBasedCharArrayTlab#{suffix}".to_sym, 732 params: {char_count: 'u32', char_array: 'ref', string_klass: 'ref'}, 733 regmap: $full_regmap, 734 regalloc_set: available_regs, 735 mode: [:FastPath]) { 736 737 if Options.arch == :arm32 738 Intrinsic(:UNREACHABLE).Terminator.void 739 ReturnVoid().void 740 next 741 end 742 743 # There is no check of the arguments against NullPointer as 744 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 745 arr_data := Add(Cast(char_array).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr 746 747 # Allocate a new string 748 if string_compression_enabled 749 compressable := is_array_of_compressable_chars(arr_data, Cast(char_count).u64) 750 If(compressable, 1).EQ.Likely.b { 751 data_size1 := Cast(char_count).word 752 } Else { 753 data_size2 := Cast(ShlI(char_count).Imm(1).u32).word 754 } 755 data_size := Phi(data_size1, data_size2).word 756 else 757 data_size := Cast(ShlI(char_count).Imm(1).u32).word 758 end 759 new_str := allocate_string_tlab(string_klass, data_size) 760 str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 761 762 # Copy data from char_array to the new string 763 # String length field is set according to SetLength() from runtime/include/coretypes/string.h 764 if string_compression_enabled 765 If(compressable, 1).EQ.Likely.b { 766 compress_u16_to_u8_chars(arr_data, str_data, Cast(char_count).u64) 767 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 768 } Else { 769 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 770 StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 771 } 772 else 773 copy_u16_chars(arr_data, str_data, Cast(char_count).u64) 774 StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 775 end 776 # String is supposed to be a constant object, so all its data should be visible by all threads 777 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 778 Return(new_str).ptr 779 780 Label(:SlowPathEntrypoint) 781 entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_SLOW_PATH") 782 Intrinsic(:SLOW_PATH_ENTRY, char_count, char_array).AddImm(entrypoint).MethodAsImm("CreateStringFromZeroBasedCharArray3ArgBridge").Terminator.ptr 783 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 784 } 785end # def GenerateCreateStringFromZeroBasedCharArrayTlab 786 787 788def GenerateSubstringFromStringTlab(string_compression_enabled) 789 suffix = (string_compression_enabled ? "Compressed" : "") 790 available_regs = $panda_mask 791 function("SubStringFromStringTlab#{suffix}".to_sym, 792 params: {str: 'ref', begin_index: 'i32', end_index: 'i32'}, 793 regmap: $full_regmap, 794 regalloc_set: available_regs, 795 mode: [:FastPath]) { 796 797 if Options.arch == :arm32 798 Intrinsic(:UNREACHABLE).Terminator.void 799 ReturnVoid().void 800 next 801 end 802 803 # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded) 804 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 805 if string_compression_enabled 806 length := ShrI(length_packed).Imm(1).u32 807 else 808 length := length_packed 809 end 810 811 # If begin_index < 0, then it is assumed to be equal to zero 812 If(begin_index, Cast(0).i32).LT.Unlikely.b { 813 bx1 := Cast(0).i32 814 } 815 bx2 := Phi(begin_index, bx1).u32 816 # If end_index < 0, then it is assumed to be equal to zero 817 If(end_index, Cast(0).i32).LT.Unlikely.b { 818 ex1 := Cast(0).i32 819 } 820 ex2 := Phi(end_index, ex1).u32 821 # If begin_index > str.length(), then make it equal to str.length() 822 If(bx2, length).A.Unlikely.b { 823 bx3 := length 824 } 825 bx4 := Phi(bx2, bx3).u32 826 # If end_index > str.length(), then make it equal to str.length() 827 If(ex2, length).A.Unlikely.b { 828 ex3 := length 829 } 830 ex4 := Phi(ex2, ex3).u32 831 # If begin_index > end_index, then swap them. 832 If(bx4, ex4).GT.Unlikely.b { 833 bx5 := ex4 834 ex5 := bx4 835 } 836 bx6 := Phi(bx4, bx5).u32 837 ex6 := Phi(ex4, ex5).u32 838 839 If(bx6, 0).EQ.Likely.b { 840 If(ex6, length).EQ.Unlikely.b { 841 Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr 842 } 843 } 844 845 if string_compression_enabled 846 not_compressed := AndI(length_packed).Imm(1).u32 847 offset := Shl(bx6, not_compressed).u32 848 else 849 offset := ShlI(bx6).Imm(1).u32 850 end 851 852 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 853 src_str_data := Add(src_str_data, Cast(offset).u64).ptr 854 855 klass := LoadI(str).Imm(Constants::OBJECT_CLASS_OFFSET).ref 856 char_count := Sub(ex6, bx6).u32 857 858 # Allocate a new string 859 if string_compression_enabled 860 If(not_compressed, 1).EQ.Unlikely.b { 861 compressable := is_array_of_compressable_chars(src_str_data, Cast(char_count).u64) 862 If(compressable, 1).EQ.Likely.b { 863 data_size1 := Cast(char_count).word 864 } Else { 865 data_size2 := Cast(ShlI(char_count).Imm(1).u32).word 866 } 867 data_size := Phi(data_size1, data_size2).word 868 new_str := allocate_string_tlab(klass, data_size) 869 new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 870 If(compressable, 1).EQ.Likely.b { 871 compress_u16_to_u8_chars(src_str_data, new_str_data, Cast(char_count).u64) 872 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 873 } Else { 874 copy_u16_chars(src_str_data, new_str_data, Cast(char_count).u64) 875 StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 876 } 877 # String is supposed to be a constant object, so all its data should be visible by all threads 878 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 879 Return(new_str).ptr 880 } Else { 881 # Source string is already compressed 882 new_str := allocate_string_tlab(klass, Cast(char_count).word) 883 new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 884 copy_u8_chars(src_str_data, new_str_data, Cast(char_count).u64) 885 StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32 886 # String is supposed to be a constant object, so all its data should be visible by all threads 887 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 888 Return(new_str).ptr 889 } 890 else 891 data_size := Cast(ShlI(char_count).Imm(1).u32).word 892 new_str := allocate_string_tlab(klass, data_size) 893 new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 894 copy_u16_chars(src_str_data, new_str_data, Cast(char_count).u64) 895 StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32 896 # String is supposed to be a constant object, so all its data should be visible by all threads 897 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 898 Return(new_str).ptr 899 end 900 901 Label(:SlowPathEntrypoint) 902 entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH") 903 Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr 904 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 905 } 906end # def GenerateSubstringFromStringTlab 907 908 909def GenerateStringGetCharsTlab(string_compression_enabled) 910 suffix = (string_compression_enabled ? "Compressed" : "") 911 available_regs = $panda_mask 912 function("StringGetCharsTlab#{suffix}".to_sym, 913 params: {str: 'ref', begin_index: 'i32', end_index: 'i32', array_klass: 'ref'}, 914 regmap: $full_regmap, 915 regalloc_set: available_regs, 916 mode: [:FastPath]) { 917 918 if Options.arch == :arm32 919 Intrinsic(:UNREACHABLE).Terminator.void 920 ReturnVoid().void 921 next 922 end 923 924 If(begin_index, end_index).GT.Unlikely.b { 925 Goto(:SlowPathEntrypoint) # Out of range 926 } 927 If(begin_index, Cast(0).i32).LT.Unlikely.b { 928 Goto(:SlowPathEntrypoint) # Out of range 929 } 930 931 # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded) 932 length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32; 933 if string_compression_enabled 934 If(Cast(end_index).u32, ShrI(length).Imm(1).u32).A.Unlikely.b { 935 Goto(:SlowPathEntrypoint) # Out of range 936 } 937 not_compressed := AndI(length).Imm(1).u32 938 offset := Shl(begin_index, not_compressed).u32 939 else 940 If(Cast(end_index).u32, length).A.Unlikely.b { 941 Goto(:SlowPathEntrypoint) # Out of range 942 } 943 not_compressed := Cast(1).u32 944 offset := ShlI(begin_index).Imm(1).u32 945 end 946 947 src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr 948 src_str_data := Add(src_str_data, Cast(offset).u64).ptr 949 950 # Allocate a new array of u16 chars 951 char_count := Sub(Cast(end_index).u32, Cast(begin_index).u32).u64 952 new_arr := allocate_array_of_chars_tlab(array_klass, Cast(char_count).word) 953 new_arr_data := Add(new_arr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr 954 If(not_compressed, Cast(0).u32).EQ.Likely.b { 955 expand_u8_to_u16_chars(src_str_data, new_arr_data, char_count) 956 } 957 If(not_compressed, Cast(1).u32).EQ.Unlikely.b { 958 copy_u16_chars(src_str_data, new_arr_data, char_count) 959 } 960 # String is supposed to be a constant object, so all its data should be visible by all threads 961 Intrinsic(:DATA_MEMORY_BARRIER_FULL).void 962 Return(new_arr).ptr 963 964 Label(:SlowPathEntrypoint) 965 entrypoint = get_entrypoint_offset("STRING_GET_CHARS_SLOW_PATH") 966 Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("StringGetChars4ArgBridge").Terminator.ptr 967 Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG 968 } 969end # def GenerateStringGetCharsTlab 970 971 972### 973# Compute string hashcode 974# 975scoped_macro(:u8_string_hashcode) do |str_data, char_count| 976 hash := Cast(0).u32 977 If(char_count, 0).A.Likely.b { 978 i1 := Cast(0).u32 979 imm31 := Cast(31).u32 980Label(:Loop_hash) 981 hash1 := Phi(hash, hash2).u32 982 i := Phi(i1, i2).u32 983 ch := Cast(Load(str_data, i).u8).SrcType(Constants::COMPILER_UINT8).u32 984 hash2 := Add(Mul(hash1, imm31).u32, ch).u32 985 i2 := AddI(i).Imm(1).u32 986 If(i2, char_count).B.Likely.b { 987 Goto(:Loop_hash) 988 } 989 } 990 hashcode := Phi(hash, hash2).u32 991end # u8_string_hashcode 992 993scoped_macro(:u16_string_hashcode) do |str_data, char_count| 994 hash := Cast(0).u32 995 If(char_count, 0).A.Likely.b { 996 i1 := Cast(0).u64 997 imm31 := Cast(31).u32 998 stop := ShlI(Cast(char_count).u64).Imm(1).u64 999Label(:Loop_hash) 1000 hash1 := Phi(hash, hash2).u32 1001 i := Phi(i1, i2).u64 1002 ch := Cast(Load(str_data, i).u16).SrcType(Constants::COMPILER_UINT16).u32 1003 hash2 := Add(Mul(hash1, imm31).u32, ch).u32 1004 i2 := AddI(i).Imm(2).u64 1005 If(i2, stop).B.Likely.b { 1006 Goto(:Loop_hash) 1007 } 1008 } 1009 hashcode := Phi(hash, hash2).u32 1010end # u16_string_hashcode 1011 1012 1013def GenerateStringHashCode(string_compression_enabled) 1014 suffix = (string_compression_enabled ? "Compressed" : "") 1015if Options.arch == :arm64 1016 available_regs = $temps_mask + :arg0 + :callee0 + :callee1 + :callee2 + :callee3 1017else 1018 available_regs = $temps_mask + :arg0 + :callee0 + :caller0 + :caller1 1019end 1020 function("StringHashCode#{suffix}".to_sym, 1021 params: {str: 'ref'}, 1022 regmap: $full_regmap, 1023 regalloc_set: available_regs, 1024 mode: [:FastPath]) { 1025 1026 if Options.arch == :arm32 1027 Intrinsic(:UNREACHABLE).Terminator.void 1028 ReturnVoid().void 1029 next 1030 end 1031 1032 # 1. There is no check of the argument against NullPointer as 1033 # it's done in the InstBuilder (see AddArgNullcheckIfNeeded) 1034 # 2. Don't check if hashcode is equal to 0 as it's done in the codegen. 1035 1036 str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr 1037 length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32 1038 if string_compression_enabled 1039 length := ShrI(length_packed).Imm(1).u32 1040 not_compressed := AndI(length_packed).Imm(1).u32 1041 If(not_compressed, 0).EQ.Likely.b { 1042 # String contains 8-bit chars 1043 h := u8_string_hashcode(str_data, length) 1044 StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32 1045 Return(h).u32 1046 } Else { 1047 # String contains 16-bit chars 1048 h := u16_string_hashcode(str_data, length) 1049 StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32 1050 Return(h).u32 1051 } 1052 else 1053 # String contains 16-bit chars 1054 h := u16_string_hashcode(str_data, length_packed) 1055 StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32 1056 Return(h).u32 1057 end 1058 } 1059end # def GenerateStringHashCode 1060