Lines Matching refs:pinned
67 #define LOAD_INSTANCE_FIELD(dst, name, load_size, pinned) \ argument
68 __ LoadFromInstance(dst, LoadInstanceIntoRegister(pinned, dst), \
73 #define LOAD_TAGGED_PTR_INSTANCE_FIELD(dst, name, pinned) \ argument
76 __ LoadTaggedPointerFromInstance(dst, LoadInstanceIntoRegister(pinned, dst), \
653 LiftoffRegList pinned) { in ProcessParameter() argument
666 : __ GetUnusedRegister(rc, pinned); in ProcessParameter()
836 LiftoffRegList pinned = kGpParamRegisters; in StartFunctionBody() local
837 LiftoffRegister tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in StartFunctionBody()
844 pinned); in StartFunctionBody()
849 LiftoffRegList pinned = kGpParamRegisters; in StartFunctionBody() local
850 LiftoffRegister tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in StartFunctionBody()
852 pinned); in StartFunctionBody()
855 __ Load(tmp, tmp.gp(), no_reg, offset, LoadType::kI32Load, pinned); in StartFunctionBody()
1113 LiftoffRegList pinned; in EmitDebuggingInfo() local
1115 pinned.set(max_steps); in EmitDebuggingInfo()
1116 LiftoffRegister max_steps_addr = __ GetUnusedRegister(kGpReg, pinned); in EmitDebuggingInfo()
1117 pinned.set(max_steps_addr); in EmitDebuggingInfo()
1122 pinned); in EmitDebuggingInfo()
1130 pinned); in EmitDebuggingInfo()
1203 LiftoffRegList pinned; in GetExceptionProperty() local
1205 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in GetExceptionProperty()
1206 LoadExceptionSymbol(tag_symbol_reg.gp(), pinned, root_index); in GetExceptionProperty()
1208 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in GetExceptionProperty()
1209 LOAD_TAGGED_PTR_INSTANCE_FIELD(context_reg.gp(), NativeContext, pinned); in GetExceptionProperty()
1248 LiftoffRegList pinned; in CatchException() local
1249 pinned.set(caught_tag); in CatchException()
1252 Register imm_tag = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in CatchException()
1253 LOAD_TAGGED_PTR_INSTANCE_FIELD(imm_tag, TagsTable, pinned); in CatchException()
1566 LiftoffRegList pinned = {dst}; in EmitUnOp() local
1568 CheckNan(dst, pinned, result_kind); in EmitUnOp()
1571 CheckS128Nan(dst, pinned, result_lane_kind); in EmitUnOp()
1754 LiftoffRegList pinned; in UnOp() local
1755 LiftoffRegister ref = pinned.set(__ PopToRegister()); in UnOp()
1756 LiftoffRegister null = __ GetUnusedRegister(kGpReg, pinned); in UnOp()
1757 LoadNullValue(null.gp(), pinned); in UnOp()
1791 LiftoffRegList pinned = {lhs}; in EmitBinOpImm() local
1793 ? __ GetUnusedRegister(result_rc, {lhs}, pinned) in EmitBinOpImm()
1794 : __ GetUnusedRegister(result_rc, pinned); in EmitBinOpImm()
1822 LiftoffRegList pinned = {dst}; in EmitBinOp() local
1824 CheckNan(dst, pinned, result_kind); in EmitBinOp()
1827 CheckS128Nan(dst, pinned, result_lane_kind); in EmitBinOp()
2198 LiftoffRegList pinned; in RefAsNonNull() local
2199 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned)); in RefAsNonNull()
2200 MaybeEmitNullCheck(decoder, obj.gp(), pinned, arg.type); in RefAsNonNull()
2210 LiftoffRegList pinned; in TraceFunctionExit() local
2212 LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TraceFunctionExit()
2221 __ LoadToRegister(__ cache_state()->stack_state.back(), pinned); in TraceFunctionExit()
2223 __ StoreTaggedPointer(info.gp(), no_reg, 0, return_reg, pinned); in TraceFunctionExit()
2226 StoreType::ForValueKind(return_kind), pinned); in TraceFunctionExit()
2250 LiftoffRegList pinned; in TierupCheckOnExit() local
2251 LiftoffRegister budget = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TierupCheckOnExit()
2252 LiftoffRegister array = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TierupCheckOnExit()
2254 pinned); in TierupCheckOnExit()
2258 __ Store(array.gp(), no_reg, offset, budget, StoreType::kI32Store, pinned); in TierupCheckOnExit()
2356 LiftoffRegList* pinned, uint32_t* offset) { in GetGlobalBaseAndOffset() argument
2357 Register addr = pinned->set(__ GetUnusedRegister(kGpReg, {})).gp(); in GetGlobalBaseAndOffset()
2360 *pinned); in GetGlobalBaseAndOffset()
2362 global->index * sizeof(Address), kPointerLoadType, *pinned); in GetGlobalBaseAndOffset()
2365 LOAD_INSTANCE_FIELD(addr, GlobalsStart, kSystemPointerSize, *pinned); in GetGlobalBaseAndOffset()
2372 const WasmGlobal* global, LiftoffRegList* pinned, Register* base, in GetBaseAndOffsetForImportedMutableExternRefGlobal() argument
2375 pinned->set(__ GetUnusedRegister(kGpReg, *pinned)).gp(); in GetBaseAndOffsetForImportedMutableExternRefGlobal()
2377 ImportedMutableGlobalsBuffers, *pinned); in GetBaseAndOffsetForImportedMutableExternRefGlobal()
2382 *pinned); in GetBaseAndOffsetForImportedMutableExternRefGlobal()
2388 pinned->set(__ GetUnusedRegister(kGpReg, *pinned)).gp(); in GetBaseAndOffsetForImportedMutableExternRefGlobal()
2391 kSystemPointerSize, *pinned); in GetBaseAndOffsetForImportedMutableExternRefGlobal()
2396 *pinned); in GetBaseAndOffsetForImportedMutableExternRefGlobal()
2412 LiftoffRegList pinned; in GlobalGet() local
2415 GetBaseAndOffsetForImportedMutableExternRefGlobal(global, &pinned, in GlobalGet()
2417 __ LoadTaggedPointer(base, base, offset, 0, pinned); in GlobalGet()
2422 LiftoffRegList pinned; in GlobalGet() local
2424 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in GlobalGet()
2426 pinned); in GlobalGet()
2427 Register value = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in GlobalGet()
2431 pinned); in GlobalGet()
2435 LiftoffRegList pinned; in GlobalGet() local
2437 Register addr = GetGlobalBaseAndOffset(global, &pinned, &offset); in GlobalGet()
2439 pinned.set(__ GetUnusedRegister(reg_class_for(kind), pinned)); in GlobalGet()
2441 __ Load(value, addr, no_reg, offset, type, pinned, nullptr, false); in GlobalGet()
2455 LiftoffRegList pinned; in GlobalSet() local
2456 LiftoffRegister value = pinned.set(__ PopToRegister(pinned)); in GlobalSet()
2459 GetBaseAndOffsetForImportedMutableExternRefGlobal(global, &pinned, in GlobalSet()
2461 __ StoreTaggedPointer(base, offset, 0, value, pinned); in GlobalSet()
2465 LiftoffRegList pinned; in GlobalSet() local
2467 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in GlobalSet()
2469 pinned); in GlobalSet()
2470 LiftoffRegister value = pinned.set(__ PopToRegister(pinned)); in GlobalSet()
2474 value, pinned); in GlobalSet()
2477 LiftoffRegList pinned; in GlobalSet() local
2479 Register addr = GetGlobalBaseAndOffset(global, &pinned, &offset); in GlobalSet()
2480 LiftoffRegister reg = pinned.set(__ PopToRegister(pinned)); in GlobalSet()
2487 LiftoffRegList pinned; in TableGet() local
2490 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TableGet()
2511 LiftoffRegList pinned; in TableSet() local
2514 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TableSet()
2554 LiftoffRegList pinned; in AssertNull() local
2555 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned)); in AssertNull()
2558 LiftoffRegister null = __ GetUnusedRegister(kGpReg, pinned); in AssertNull()
2559 LoadNullValue(null.gp(), pinned); in AssertNull()
2570 LiftoffRegList pinned; in Select() local
2571 Register condition = pinned.set(__ PopToRegister()).gp(); in Select()
2575 LiftoffRegister false_value = pinned.set(__ PopToRegister(pinned)); in Select()
2576 LiftoffRegister true_value = __ PopToRegister(pinned); in Select()
2712 LiftoffRegList pinned; in BrTable() local
2713 LiftoffRegister value = pinned.set(__ PopToRegister()); in BrTable()
2718 LiftoffRegister tmp = __ GetUnusedRegister(kGpReg, pinned); in BrTable()
2799 LiftoffRegList pinned, ForceCheck force_check) { in BoundsCheckMem() argument
2850 pinned.set(index_ptrsize); in BoundsCheckMem()
2852 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in BoundsCheckMem()
2853 LiftoffRegister mem_size = __ GetUnusedRegister(kGpReg, pinned); in BoundsCheckMem()
2854 LOAD_INSTANCE_FIELD(mem_size.gp(), MemorySize, kSystemPointerSize, pinned); in BoundsCheckMem()
2879 LiftoffRegList pinned) { in AlignmentCheckMem() argument
2883 Register address = __ GetUnusedRegister(kGpReg, pinned).gp(); in AlignmentCheckMem()
2909 LiftoffRegList pinned; in TraceMemoryOperation() local
2910 if (index != no_reg) pinned.set(index); in TraceMemoryOperation()
2913 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TraceMemoryOperation()
2922 LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TraceMemoryOperation()
2939 pinned); in TraceMemoryOperation()
2942 StoreType::kI32Store8, pinned); in TraceMemoryOperation()
2945 StoreType::kI32Store8, pinned); in TraceMemoryOperation()
2981 Register GetMemoryStart(LiftoffRegList pinned) { in GetMemoryStart() argument
2984 memory_start = __ GetUnusedRegister(kGpReg, pinned).gp(); in GetMemoryStart()
2986 pinned); in GetMemoryStart()
3012 LiftoffRegList pinned; in LoadMem() local
3013 Register mem = pinned.set(GetMemoryStart(pinned)); in LoadMem()
3014 LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned)); in LoadMem()
3015 __ Load(value, mem, no_reg, offset, type, pinned, nullptr, true, in LoadMem()
3025 LiftoffRegList pinned = {index}; in LoadMem() local
3029 Register mem = pinned.set(GetMemoryStart(pinned)); in LoadMem()
3030 LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned)); in LoadMem()
3033 __ Load(value, mem, index, offset, type, pinned, &protected_load_pc, true, in LoadMem()
3069 LiftoffRegList pinned = {index}; in LoadTransform() local
3071 Register addr = GetMemoryStart(pinned); in LoadTransform()
3100 LiftoffRegList pinned; in LoadLane() local
3101 LiftoffRegister value = pinned.set(__ PopToRegister()); in LoadLane()
3104 full_index, pinned, kDontForceCheck); in LoadLane()
3108 pinned.set(index); in LoadLane()
3110 Register addr = GetMemoryStart(pinned); in LoadLane()
3135 LiftoffRegList pinned; in StoreMem() local
3136 LiftoffRegister value = pinned.set(__ PopToRegister()); in StoreMem()
3145 Register mem = pinned.set(GetMemoryStart(pinned)); in StoreMem()
3146 __ Store(mem, no_reg, offset, value, type, pinned, nullptr, true); in StoreMem()
3148 LiftoffRegister full_index = __ PopToRegister(pinned); in StoreMem()
3150 pinned, kDontForceCheck); in StoreMem()
3153 pinned.set(index); in StoreMem()
3158 Register mem = pinned.set(GetMemoryStart(pinned)); in StoreMem()
3179 LiftoffRegList pinned; in StoreLane() local
3180 LiftoffRegister value = pinned.set(__ PopToRegister()); in StoreLane()
3181 LiftoffRegister full_index = __ PopToRegister(pinned); in StoreLane()
3183 full_index, pinned, kDontForceCheck); in StoreLane()
3187 pinned.set(index); in StoreLane()
3189 Register addr = pinned.set(GetMemoryStart(pinned)); in StoreLane()
3219 LiftoffRegList pinned; in MemoryGrow() local
3220 LiftoffRegister input = pinned.set(__ PopToRegister()); in MemoryGrow()
3223 LiftoffRegister result = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in MemoryGrow()
3235 LiftoffRegister high_word = __ GetUnusedRegister(kGpReg, pinned); in MemoryGrow()
3261 if (kNeedI64RegPair) result64 = __ GetUnusedRegister(kGpRegPair, pinned); in MemoryGrow()
3402 LiftoffRegList pinned; in BrOnNull() local
3403 LiftoffRegister ref = pinned.set(__ PopToRegister(pinned)); in BrOnNull()
3404 Register null = __ GetUnusedRegister(kGpReg, pinned).gp(); in BrOnNull()
3405 LoadNullValue(null, pinned); in BrOnNull()
3408 if (pass_null_along_branch) LoadNullValue(null, pinned); in BrOnNull()
3424 LiftoffRegList pinned; in BrOnNonNull() local
3425 LiftoffRegister ref = pinned.set(__ PopToRegister(pinned)); in BrOnNonNull()
3429 Register null = __ GetUnusedRegister(kGpReg, pinned).gp(); in BrOnNonNull()
3430 LoadNullValue(null, pinned); in BrOnNonNull()
3456 LiftoffRegList pinned = {dst}; in EmitTerOp() local
3458 CheckNan(dst, pinned, result_kind); in EmitTerOp()
3506 LiftoffRegList pinned = {dst}; in EmitSimdFloatRoundingOpWithCFallback() local
3507 CheckS128Nan(dst, pinned, result_lane_kind); in EmitSimdFloatRoundingOpWithCFallback()
4163 Register value, LiftoffRegList pinned) { in Store32BitExceptionValue() argument
4164 LiftoffRegister tmp_reg = __ GetUnusedRegister(kGpReg, pinned); in Store32BitExceptionValue()
4172 tmp_reg, pinned, LiftoffAssembler::kSkipWriteBarrier); in Store32BitExceptionValue()
4181 tmp_reg, pinned, LiftoffAssembler::kSkipWriteBarrier); in Store32BitExceptionValue()
4185 LiftoffRegister value, LiftoffRegList pinned) { in Store64BitExceptionValue() argument
4188 pinned); in Store64BitExceptionValue()
4190 pinned); in Store64BitExceptionValue()
4193 pinned); in Store64BitExceptionValue()
4196 pinned); in Store64BitExceptionValue()
4202 LiftoffRegList pinned) { in Load16BitExceptionValue() argument
4205 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(*index), pinned); in Load16BitExceptionValue()
4210 uint32_t* index, LiftoffRegList pinned) { in Load32BitExceptionValue() argument
4211 LiftoffRegister upper = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in Load32BitExceptionValue()
4212 Load16BitExceptionValue(upper, values_array, index, pinned); in Load32BitExceptionValue()
4214 Load16BitExceptionValue(LiftoffRegister(dst), values_array, index, pinned); in Load32BitExceptionValue()
4220 LiftoffRegList pinned) { in Load64BitExceptionValue() argument
4222 Load32BitExceptionValue(dst.high_gp(), values_array, index, pinned); in Load64BitExceptionValue()
4223 Load32BitExceptionValue(dst.low_gp(), values_array, index, pinned); in Load64BitExceptionValue()
4225 Load16BitExceptionValue(dst, values_array, index, pinned); in Load64BitExceptionValue()
4228 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in Load64BitExceptionValue()
4229 Load16BitExceptionValue(tmp_reg, values_array, index, pinned); in Load64BitExceptionValue()
4232 Load16BitExceptionValue(tmp_reg, values_array, index, pinned); in Load64BitExceptionValue()
4235 Load16BitExceptionValue(tmp_reg, values_array, index, pinned); in Load64BitExceptionValue()
4241 int* index_in_array, LiftoffRegList pinned) { in StoreExceptionValue() argument
4242 LiftoffRegister value = pinned.set(__ PopToRegister(pinned)); in StoreExceptionValue()
4246 pinned); in StoreExceptionValue()
4250 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in StoreExceptionValue()
4253 pinned); in StoreExceptionValue()
4257 Store64BitExceptionValue(values_array, index_in_array, value, pinned); in StoreExceptionValue()
4261 pinned.set(__ GetUnusedRegister(reg_class_for(kI64), pinned)); in StoreExceptionValue()
4264 Store64BitExceptionValue(values_array, index_in_array, tmp_reg, pinned); in StoreExceptionValue()
4269 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in StoreExceptionValue()
4273 pinned); in StoreExceptionValue()
4285 value, pinned); in StoreExceptionValue()
4297 uint32_t* index, LiftoffRegList pinned) { in LoadExceptionValue() argument
4299 LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned)); in LoadExceptionValue()
4302 Load32BitExceptionValue(value.gp(), values_array, index, pinned); in LoadExceptionValue()
4306 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in LoadExceptionValue()
4307 Load32BitExceptionValue(tmp_reg.gp(), values_array, index, pinned); in LoadExceptionValue()
4313 Load64BitExceptionValue(value, values_array, index, pinned); in LoadExceptionValue()
4318 pinned.set(__ GetUnusedRegister(rc_i64, pinned)); in LoadExceptionValue()
4319 Load64BitExceptionValue(tmp_reg, values_array, index, pinned); in LoadExceptionValue()
4326 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in LoadExceptionValue()
4327 Load32BitExceptionValue(tmp_reg.gp(), values_array, index, pinned); in LoadExceptionValue()
4330 Load32BitExceptionValue(tmp_reg.gp(), values_array, index, pinned); in LoadExceptionValue()
4341 pinned); in LoadExceptionValue()
4357 LiftoffRegList pinned; in GetExceptionValues() local
4361 pinned.set(values_array); in GetExceptionValues()
4365 LoadExceptionValue(param.kind(), values_array, &index, pinned); in GetExceptionValues()
4404 LiftoffRegList pinned; in Throw() local
4409 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in Throw()
4423 pinned.set(values_array); in Throw()
4433 StoreExceptionValue(type, values_array.gp(), &index, pinned); in Throw()
4440 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in Throw()
4441 LOAD_TAGGED_PTR_INSTANCE_FIELD(exception_tag.gp(), TagsTable, pinned); in Throw()
4460 LiftoffRegList pinned; in AtomicStoreMem() local
4461 LiftoffRegister value = pinned.set(__ PopToRegister()); in AtomicStoreMem()
4462 LiftoffRegister full_index = __ PopToRegister(pinned); in AtomicStoreMem()
4464 full_index, pinned, kDoForceCheck); in AtomicStoreMem()
4467 pinned.set(index); in AtomicStoreMem()
4468 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned); in AtomicStoreMem()
4471 Register addr = pinned.set(GetMemoryStart(pinned)); in AtomicStoreMem()
4489 LiftoffRegList pinned = {index}; in AtomicLoadMem() local
4490 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned); in AtomicLoadMem()
4493 Register addr = pinned.set(GetMemoryStart(pinned)); in AtomicLoadMem()
4495 LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned)); in AtomicLoadMem()
4496 __ AtomicLoad(value, addr, index, offset, type, pinned); in AtomicLoadMem()
4512 LiftoffRegList pinned; in AtomicBinop() local
4513 LiftoffRegister value = pinned.set(__ PopToRegister()); in AtomicBinop()
4521 result = pinned.set(__ GetUnusedRegister(value.reg_class(), pinned)); in AtomicBinop()
4523 pinned.clear(value); in AtomicBinop()
4528 pinned.set(__ GetUnusedRegister(value.reg_class(), pinned)); in AtomicBinop()
4530 LiftoffRegister full_index = __ PopToRegister(pinned); in AtomicBinop()
4532 full_index, pinned, kDoForceCheck); in AtomicBinop()
4535 pinned.set(index); in AtomicBinop()
4536 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned); in AtomicBinop()
4540 Register addr = pinned.set(GetMemoryStart(pinned)); in AtomicBinop()
4558 LiftoffRegList pinned = {index}; in AtomicCompareExchange() local
4559 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned); in AtomicCompareExchange()
4562 Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in AtomicCompareExchange()
4563 LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize, pinned); in AtomicCompareExchange()
4568 pinned.clear(LiftoffRegister(index)); in AtomicCompareExchange()
4569 LiftoffRegister new_value = pinned.set(__ PopToRegister(pinned)); in AtomicCompareExchange()
4570 LiftoffRegister expected = pinned.set(__ PopToRegister(pinned)); in AtomicCompareExchange()
4586 LiftoffRegList pinned; in AtomicCompareExchange()
4587 LiftoffRegister new_value = pinned.set(__ PopToRegister()); in AtomicCompareExchange()
4588 LiftoffRegister expected = pinned.set(__ PopToRegister(pinned)); in AtomicCompareExchange()
4589 LiftoffRegister full_index = __ PopToRegister(pinned); in AtomicCompareExchange()
4591 full_index, pinned, kDoForceCheck); in AtomicCompareExchange()
4593 pinned.set(index); in AtomicCompareExchange()
4594 AlignmentCheckMem(decoder, type.size(), imm.offset, index, pinned); in AtomicCompareExchange()
4597 Register addr = pinned.set(GetMemoryStart(pinned)); in AtomicCompareExchange()
4599 pinned.set(__ GetUnusedRegister(reg_class_for(result_kind), pinned)); in AtomicCompareExchange()
4638 LiftoffRegList pinned = {index_reg}; in AtomicWait() local
4640 pinned); in AtomicWait()
4645 ? pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp() in AtomicWait()
4685 LiftoffRegList pinned = {index_reg}; in AtomicNotify() local
4686 AlignmentCheckMem(decoder, kInt32Size, imm.offset, index_reg, pinned); in AtomicNotify()
4691 ? pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp() in AtomicNotify()
4847 LiftoffRegList* pinned) { in PopMemTypeToRegister() argument
4848 LiftoffRegister reg = __ PopToRegister(*pinned); in PopMemTypeToRegister()
4853 if (pinned->has(reg) || __ cache_state()->is_used(reg)) { in PopMemTypeToRegister()
4854 intptr_reg = __ GetUnusedRegister(kGpReg, *pinned); in PopMemTypeToRegister()
4860 pinned->set(intptr_reg); in PopMemTypeToRegister()
4868 pinned->set(reg.low()); in PopMemTypeToRegister()
4874 pinned->has(reg.high()) in PopMemTypeToRegister()
4875 ? __ GetUnusedRegister(kGpReg, *pinned).gp() in PopMemTypeToRegister()
4876 : __ GetUnusedRegister(kGpReg, {reg.high()}, *pinned).gp(); in PopMemTypeToRegister()
4877 pinned->set(*high_word); in PopMemTypeToRegister()
4892 LiftoffRegList pinned; in MemoryInit() local
4893 LiftoffRegister size = pinned.set(__ PopToRegister()); in MemoryInit()
4894 LiftoffRegister src = pinned.set(__ PopToRegister(pinned)); in MemoryInit()
4896 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned); in MemoryInit()
4900 instance = __ GetUnusedRegister(kGpReg, pinned).gp(); in MemoryInit()
4903 pinned.set(instance); in MemoryInit()
4912 pinned.clear(mem_offsets_high_word); in MemoryInit()
4916 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in MemoryInit()
4932 LiftoffRegList pinned; in DataDrop() local
4935 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in DataDrop()
4937 pinned); in DataDrop()
4940 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in DataDrop()
4946 LiftoffRegister null_reg = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in DataDrop()
4949 pinned); in DataDrop()
4956 LiftoffRegList pinned; in MemoryCopy() local
4957 LiftoffRegister size = pinned.set( in MemoryCopy()
4958 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned)); in MemoryCopy()
4959 LiftoffRegister src = pinned.set( in MemoryCopy()
4960 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned)); in MemoryCopy()
4961 LiftoffRegister dst = pinned.set( in MemoryCopy()
4962 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned)); in MemoryCopy()
4966 instance = __ GetUnusedRegister(kGpReg, pinned).gp(); in MemoryCopy()
4994 LiftoffRegList pinned; in MemoryFill() local
4995 LiftoffRegister size = pinned.set( in MemoryFill()
4996 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned)); in MemoryFill()
4997 LiftoffRegister value = pinned.set(__ PopToRegister(pinned)); in MemoryFill()
4998 LiftoffRegister dst = pinned.set( in MemoryFill()
4999 PopMemTypeToRegister(decoder, &mem_offsets_high_word, &pinned)); in MemoryFill()
5003 instance = __ GetUnusedRegister(kGpReg, pinned).gp(); in MemoryFill()
5035 LiftoffRegList pinned; in TableInit() local
5037 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TableInit()
5043 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TableInit()
5064 LiftoffRegList pinned; in ElemDrop() local
5066 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in ElemDrop()
5068 kSystemPointerSize, pinned); in ElemDrop()
5071 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in ElemDrop()
5076 LiftoffRegister one_reg = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in ElemDrop()
5079 StoreType::kI32Store8, pinned); in ElemDrop()
5084 LiftoffRegList pinned; in TableCopy() local
5087 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TableCopy()
5093 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TableCopy()
5115 LiftoffRegList pinned; in TableGrow() local
5118 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TableGrow()
5142 LiftoffRegList pinned; in TableSize() local
5144 Register tables = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in TableSize()
5145 LOAD_TAGGED_PTR_INSTANCE_FIELD(tables, Tables, pinned); in TableSize()
5150 ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), pinned); in TableSize()
5159 pinned); in TableSize()
5167 LiftoffRegList pinned; in TableFill() local
5170 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TableFill()
5200 LiftoffRegList pinned = {obj}; in StructNew() local
5206 ? pinned.set(__ PopToRegister(pinned)) in StructNew()
5207 : pinned.set(__ GetUnusedRegister( in StructNew()
5208 reg_class_for(field_kind), pinned)); in StructNew()
5211 SetDefaultValue(value, field_kind, pinned); in StructNew()
5213 StoreObjectField(obj.gp(), no_reg, offset, value, pinned, field_kind); in StructNew()
5214 pinned.clear(value); in StructNew()
5243 LiftoffRegList pinned; in StructGet() local
5244 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned)); in StructGet()
5245 MaybeEmitNullCheck(decoder, obj.gp(), pinned, struct_obj.type); in StructGet()
5247 __ GetUnusedRegister(reg_class_for(field_kind), pinned); in StructGet()
5249 pinned); in StructGet()
5259 LiftoffRegList pinned; in StructSet() local
5260 LiftoffRegister value = pinned.set(__ PopToRegister(pinned)); in StructSet()
5261 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned)); in StructSet()
5262 MaybeEmitNullCheck(decoder, obj.gp(), pinned, struct_obj.type); in StructSet()
5263 StoreObjectField(obj.gp(), no_reg, offset, value, pinned, field_kind); in StructSet()
5303 LiftoffRegList pinned = {obj}; in ArrayNew() local
5304 LiftoffRegister length = pinned.set(__ PopToModifiableRegister(pinned)); in ArrayNew()
5305 LiftoffRegister value = pinned.set(__ PopToRegister(pinned)); in ArrayNew()
5308 LiftoffRegister offset = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in ArrayNew()
5322 StoreObjectField(obj.gp(), offset.gp(), 0, value, pinned, elem_kind); in ArrayNew()
5351 LiftoffRegList pinned; in ArrayGet() local
5352 LiftoffRegister index = pinned.set(__ PopToModifiableRegister(pinned)); in ArrayGet()
5353 LiftoffRegister array = pinned.set(__ PopToRegister(pinned)); in ArrayGet()
5354 MaybeEmitNullCheck(decoder, array.gp(), pinned, array_obj.type); in ArrayGet()
5355 BoundsCheckArray(decoder, array, index, pinned); in ArrayGet()
5363 __ GetUnusedRegister(reg_class_for(elem_kind), pinned); in ArrayGet()
5366 elem_kind, is_signed, pinned); in ArrayGet()
5373 LiftoffRegList pinned; in ArraySet() local
5374 LiftoffRegister value = pinned.set(__ PopToRegister(pinned)); in ArraySet()
5377 LiftoffRegister index = pinned.set(__ PopToModifiableRegister(pinned)); in ArraySet()
5378 LiftoffRegister array = pinned.set(__ PopToRegister(pinned)); in ArraySet()
5379 MaybeEmitNullCheck(decoder, array.gp(), pinned, array_obj.type); in ArraySet()
5380 BoundsCheckArray(decoder, array, index, pinned); in ArraySet()
5388 value, pinned, elem_kind); in ArraySet()
5392 LiftoffRegList pinned; in ArrayLen() local
5393 LiftoffRegister obj = pinned.set(__ PopToRegister(pinned)); in ArrayLen()
5394 MaybeEmitNullCheck(decoder, obj.gp(), pinned, array_obj.type); in ArrayLen()
5395 LiftoffRegister len = __ GetUnusedRegister(kGpReg, pinned); in ArrayLen()
5397 LoadObjectField(len, obj.gp(), no_reg, kLengthOffset, kI32, false, pinned); in ArrayLen()
5428 LiftoffRegList pinned; in ArrayInit() local
5430 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in ArrayInit()
5436 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in ArrayInit()
5456 LiftoffRegList pinned = {array}; in ArrayInit() local
5457 LiftoffRegister element = pinned.set(__ PopToRegister(pinned)); in ArrayInit()
5459 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in ArrayInit()
5464 element, pinned, elem_kind); in ArrayInit()
5476 LiftoffRegList pinned; in ArrayInitFromData() local
5478 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in ArrayInitFromData()
5569 LiftoffRegList pinned = {}, in SubtypeCheck() argument
5572 LiftoffRegister rtt_reg = pinned.set(__ PopToRegister(pinned));
5573 LiftoffRegister obj_reg = pinned.set(__ PopToRegister(pinned));
5580 : pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5581 LiftoffRegister tmp2 = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
5583 LoadNullValue(tmp1.gp(), pinned);
5600 __ LoadTaggedPointer(tmp1.gp(), tmp1.gp(), no_reg, kTypeInfoOffset, pinned);
5605 pinned);
5611 __ LoadFixedArrayLengthAsInt32(list_length, tmp1.gp(), pinned);
5618 wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(rtt_depth), pinned);
5630 LiftoffRegList pinned; in RefTest() local
5631 LiftoffRegister result = pinned.set(__ GetUnusedRegister(kGpReg, {})); in RefTest()
5633 SubtypeCheck(decoder, obj, rtt, &return_false, kNullFails, pinned, in RefTest()
5706 LiftoffRegList pinned, Register opt_scratch) { in DataCheck() argument
5708 TypeCheckPrelude(obj, no_match, pinned, opt_scratch); in DataCheck()
5710 pinned); in DataCheck()
5715 LiftoffRegList pinned, Register opt_scratch) { in ArrayCheck() argument
5717 TypeCheckPrelude(obj, no_match, pinned, opt_scratch); in ArrayCheck()
5720 LoadType::kI32Load16U, pinned); in ArrayCheck()
5727 LiftoffRegList pinned, Register opt_scratch) { in FuncCheck() argument
5729 TypeCheckPrelude(obj, no_match, pinned, opt_scratch); in FuncCheck()
5732 LoadType::kI32Load16U, pinned); in FuncCheck()
5739 LiftoffRegList pinned, Register opt_scratch) { in I31Check() argument
5740 LiftoffRegister obj_reg = pinned.set(__ PopToRegister(pinned)); in I31Check()
5748 const Value& obj, Label* no_match, LiftoffRegList pinned,
5754 LiftoffRegList pinned; in AbstractTypeCheck() local
5755 LiftoffRegister result = pinned.set(__ GetUnusedRegister(kGpReg, {})); in AbstractTypeCheck()
5757 (this->*type_checker)(object, &no_match, pinned, result.gp()); in AbstractTypeCheck()
5942 LiftoffRegList pinned; in CallDirect() local
5943 Register tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in CallDirect()
5944 Register target = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in CallDirect()
5948 kSystemPointerSize, pinned); in CallDirect()
5950 imm.index * sizeof(Address), kPointerLoadType, pinned); in CallDirect()
5954 ImportedFunctionRefs, pinned); in CallDirect()
5958 ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), pinned); in CallDirect()
6015 LiftoffRegList pinned = {index}; in CallIndirect() local
6017 Register table = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in CallIndirect()
6018 Register tmp_const = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in CallIndirect()
6019 Register scratch = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in CallIndirect()
6023 pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); in CallIndirect()
6025 IndirectFunctionTables, pinned); in CallIndirect()
6031 pinned); in CallIndirect()
6047 pinned); in CallIndirect()
6052 LoadType::kI32Load, pinned); in CallIndirect()
6061 kSystemPointerSize, pinned); in CallIndirect()
6066 kPointerLoadType, pinned); in CallIndirect()
6072 pinned); in CallIndirect()
6093 LOAD_TAGGED_PTR_INSTANCE_FIELD(table, IndirectFunctionTableRefs, pinned); in CallIndirect()
6098 pinned); in CallIndirect()
6102 pinned); in CallIndirect()
6116 kSystemPointerSize, pinned); in CallIndirect()
6121 kPointerLoadType, pinned); in CallIndirect()
6124 pinned); in CallIndirect()
6164 LiftoffRegList pinned; in CallRef() local
6165 LiftoffRegister vector = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in CallRef()
6168 if (funcref.is_reg()) pinned.set(funcref.reg()); in CallRef()
6171 LiftoffRegister index = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in CallRef()
6206 LiftoffRegList pinned; in CallRef() local
6207 LiftoffRegister func_ref = pinned.set(__ PopToModifiableRegister(pinned)); in CallRef()
6208 MaybeEmitNullCheck(decoder, func_ref.gp(), pinned, func_ref_type); in CallRef()
6210 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in CallRef()
6211 LiftoffRegister target = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in CallRef()
6212 LiftoffRegister temp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in CallRef()
6218 pinned); in CallRef()
6221 LOAD_INSTANCE_FIELD(temp.gp(), IsolateRoot, kSystemPointerSize, pinned); in CallRef()
6229 kPointerLoadType, pinned); in CallRef()
6242 pinned); in CallRef()
6274 void LoadNullValue(Register null, LiftoffRegList pinned) { in LoadNullValue() argument
6275 LOAD_INSTANCE_FIELD(null, IsolateRoot, kSystemPointerSize, pinned); in LoadNullValue()
6280 void LoadExceptionSymbol(Register dst, LiftoffRegList pinned, in LoadExceptionSymbol() argument
6282 LOAD_INSTANCE_FIELD(dst, IsolateRoot, kSystemPointerSize, pinned); in LoadExceptionSymbol()
6288 LiftoffRegList pinned, ValueType type) { in MaybeEmitNullCheck() argument
6292 LiftoffRegister null = __ GetUnusedRegister(kGpReg, pinned); in MaybeEmitNullCheck()
6293 LoadNullValue(null.gp(), pinned); in MaybeEmitNullCheck()
6299 LiftoffRegister index, LiftoffRegList pinned) { in BoundsCheckArray() argument
6303 LiftoffRegister length = __ GetUnusedRegister(kGpReg, pinned); in BoundsCheckArray()
6307 pinned); in BoundsCheckArray()
6319 LiftoffRegList pinned) { in LoadObjectField() argument
6321 __ LoadTaggedPointer(dst.gp(), src, offset_reg, offset, pinned); in LoadObjectField()
6325 __ Load(dst, src, offset_reg, offset, load_type, pinned); in LoadObjectField()
6330 LiftoffRegister value, LiftoffRegList pinned, in StoreObjectField() argument
6333 __ StoreTaggedPointer(obj, offset_reg, offset, value, pinned); in StoreObjectField()
6337 __ Store(obj, offset_reg, offset, value, store_type, pinned); in StoreObjectField()
6342 LiftoffRegList pinned) { in SetDefaultValue() argument
6359 return LoadNullValue(reg.gp(), pinned); in SetDefaultValue()
6373 LiftoffRegList pinned, in TypeCheckPrelude() argument
6375 LiftoffRegister obj_reg = pinned.set(__ PopToRegister(pinned)); in TypeCheckPrelude()
6382 : pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TypeCheckPrelude()
6383 LiftoffRegister tmp_reg = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in TypeCheckPrelude()
6386 LoadNullValue(map_reg.gp(), pinned); in TypeCheckPrelude()
6398 LiftoffRegList pinned) { in EmitDataRefCheck() argument
6402 pinned); in EmitDataRefCheck()
6426 void CheckNan(LiftoffRegister src, LiftoffRegList pinned, ValueKind kind) { in CheckNan() argument
6428 auto nondeterminism_addr = __ GetUnusedRegister(kGpReg, pinned); in CheckNan()
6435 void CheckS128Nan(LiftoffRegister dst, LiftoffRegList pinned, in CheckS128Nan() argument
6438 LiftoffRegister tmp_gp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in CheckS128Nan()
6439 LiftoffRegister tmp_s128 = pinned.set(__ GetUnusedRegister(rc, pinned)); in CheckS128Nan()
6441 pinned.set(__ GetUnusedRegister(kGpReg, pinned)); in CheckS128Nan()
6485 Register LoadInstanceIntoRegister(LiftoffRegList pinned, Register fallback) { in LoadInstanceIntoRegister() argument
6489 pinned | LiftoffRegList{fallback}); in LoadInstanceIntoRegister()