1 /*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "ecmascript/compiler/barrier_stub_builder.h"
17 #include "ecmascript/compiler/rt_call_signature.h"
18 #include "ecmascript/compiler/stub_builder-inl.h"
19 #include "ecmascript/js_thread.h"
20
21 /* LOCAL_TO_SHARE_SWAPPED 和 OLD_TO_NEW_SWAPPED_MASK标记读取并发场景分析：
22 * 设置标记：CollectGarbage->RunPhases->Sweep->PrepareSweeping这条路径会设置这两个标记, CollectGarbage只会在当前线程local gc时使用。
23 * 清除标记：（1）由CreateXXXXRememberedSet清除标记，这个调用点有两种情况：
24 * a. js主线程在需要读rset时会调用的方法，一定在js线程。
25 * b. gc时移动对象时从gc线程调用，此时主线程是被suspendAll停下来的等待gc线程的状态。
26 * （2）CollectGarbage -> RunPhases -> Evacuate -> UpdateReference ->
27 * 并行UpdateReferenceTask -> UpdateRSet -> Merge：
28 * 其中有并行任务，会主线程会等待并行任务结束，有同步的保证，可以保证清除能够被主线程观察到。
29 *
30 * LOCAL_TO_SHARE_COLLECTED_MASK标记读取并发场景分析：
31 * 设置标记：SharedGCMarkBase::MarkRoots-> CollectLocalVMRSet（访问多个线程的heap） -> EnumerateRegions -> ExtractLocalToShareRSet，
32 * 这里会有跨线程，但是已经在SuspendAll的保证之下了。
33 * 清除标记：（1）入口在MergeBackAndResetRSetWorkListHandler，这个函数调用点有5个，可能不在当前线程，但是调用点处都有SuspendAll的保证。
34 * （2）js线程在local gc结束时，会把自己线程被Collect的rset拿回来，这里只处理当前线程的，没有并发问题。
35 */
36
37 namespace panda::ecmascript::kungfu {
DoBatchBarrier()38 void BarrierStubBuilder::DoBatchBarrier()
39 {
40 auto env = GetEnvironment();
41 Label entry(env);
42 env->SubCfgEntry(&entry);
43 Label exit(env);
44 Label checkNext(env);
45 BRANCH_UNLIKELY(
46 LoadPrimitive(VariableType::BOOL(), glue_, IntPtr(JSThread::GlueData::GetIsEnableCMCGCOffset(env->Is32Bit()))),
47 &exit, &checkNext);
48 Bind(&checkNext);
49 Label handleMark(env);
50 Label handleBitSet(env);
51 BRANCH_NO_WEIGHT(InSharedHeap(objectRegion_), &handleMark, &handleBitSet);
52 Bind(&handleBitSet);
53 {
54 DoBatchBarrierInternal();
55 Jump(&handleMark);
56 }
57
58 Bind(&handleMark);
59 HandleMark();
60 Jump(&exit);
61 Bind(&exit);
62 env->SubCfgExit();
63 }
64
DoBatchBarrierInternal()65 void BarrierStubBuilder::DoBatchBarrierInternal()
66 {
67 auto env = GetEnvironment();
68 Label entry(env);
69 env->SubCfgEntry(&entry);
70 Label exit(env);
71 Label inYoung(env);
72 Label notInYoung(env);
73 BRANCH_NO_WEIGHT(InYoungGeneration(objectRegion_), &inYoung, ¬InYoung);
74 Bind(¬InYoung);
75 {
76 BarrierBatchBitSet(LocalToShared | OldToNew);
77 Jump(&exit);
78 }
79 Bind(&inYoung);
80 {
81 BarrierBatchBitSet(LocalToShared);
82 Jump(&exit);
83 }
84 Bind(&exit);
85 env->SubCfgExit();
86 }
87
GetBitSetDataAddr(GateRef objectRegion,GateRef loadOffset,int32_t createFunID)88 GateRef BarrierStubBuilder::GetBitSetDataAddr(GateRef objectRegion, GateRef loadOffset, int32_t createFunID)
89 {
90 auto env = GetEnvironment();
91 Label entry(env);
92 env->SubCfgEntry(&entry);
93 Label exit(env);
94
95 DEFVARIABLE(oldBitSet, VariableType::NATIVE_POINTER(),
96 LoadPrimitive(VariableType::NATIVE_POINTER(), objectRegion, loadOffset));
97
98 Label createRset(env);
99 BRANCH_UNLIKELY(IntPtrEqual(*oldBitSet, IntPtr(0)), &createRset, &exit);
100 Bind(&createRset);
101 {
102 oldBitSet = CallNGCRuntime(glue_, createFunID, {objectRegion});
103 Jump(&exit);
104 }
105
106 Bind(&exit);
107 GateRef bitSetDataAddr = PtrAdd(*oldBitSet, IntPtr(RememberedSet::GCBITSET_DATA_OFFSET));
108 env->SubCfgExit();
109 return bitSetDataAddr;
110 }
111
HandleMark()112 void BarrierStubBuilder::HandleMark()
113 {
114 auto env = GetEnvironment();
115 Label entry(env);
116 env->SubCfgEntry(&entry);
117 Label exit(env);
118
119 bool isArch32 = GetEnvironment()->Is32Bit();
120 GateRef gcStateBitField = LoadPrimitive(VariableType::INT64(), glue_,
121 Int64(JSThread::GlueData::GetGCStateBitFieldOffset(isArch32)));
122 GateRef gcState = Int64And(gcStateBitField, Int64(JSThread::CONCURRENT_MARKING_BITFIELD_MASK));
123 GateRef sharedGCStateBitField = LoadPrimitive(VariableType::INT64(), glue_,
124 Int64(JSThread::GlueData::GetSharedGCStateBitFieldOffset(isArch32)));
125 GateRef sharedGCState = Int64And(sharedGCStateBitField, Int64(JSThread::SHARED_CONCURRENT_MARKING_BITFIELD_MASK));
126 Label needMarking(env);
127 GateRef marking = Int64NotEqual(gcState, Int64(static_cast<int64_t>(MarkStatus::READY_TO_MARK)));
128 GateRef sharedMarking = Int64NotEqual(sharedGCState,
129 Int64(static_cast<int64_t>(SharedMarkStatus::READY_TO_CONCURRENT_MARK)));
130 BRANCH_LIKELY(BitAnd(BoolNot(marking), BoolNot(sharedMarking)), &exit, &needMarking);
131 Bind(&needMarking);
132 Label begin(env);
133 Label body(env);
134 Label endLoop(env);
135 DEFVARIABLE(index, VariableType::INT32(), Int32(0));
136 Jump(&begin);
137 LoopBegin(&begin);
138 {
139 BRANCH_LIKELY(Int32UnsignedLessThan(*index, slotCount_), &body, &exit);
140 Bind(&body);
141 {
142 GateRef offset = PtrMul(ZExtInt32ToPtr(*index), IntPtr(JSTaggedValue::TaggedTypeSize()));
143 GateRef value = Load(VariableType::JS_ANY(), glue_, dstAddr_, offset);
144 Label doMarking(env);
145 BRANCH_NO_WEIGHT(TaggedIsHeapObject(value), &doMarking, &endLoop);
146 Bind(&doMarking);
147
148 GateRef valueRegion = ObjectAddressToRange(value);
149 Label callMarking(env);
150 Label doSharedMarking(env);
151 BRANCH_NO_WEIGHT(LogicAndBuilder(env).And(marking).And(BoolNot(InSharedHeap(valueRegion))).Done(),
152 &callMarking, &doSharedMarking);
153 Bind(&callMarking);
154 {
155 CallNGCRuntime(glue_, RTSTUB_ID(MarkingBarrier), {glue_, dstAddr_, offset, value});
156 Jump(&endLoop);
157 }
158 Bind(&doSharedMarking);
159 {
160 Label callSharedMarking(env);
161 BRANCH_NO_WEIGHT(
162 LogicAndBuilder(env).And(sharedMarking).And(InSharedSweepableSpace(valueRegion)).Done(),
163 &callSharedMarking, &endLoop);
164 Bind(&callSharedMarking);
165 {
166 CallNGCRuntime(glue_, RTSTUB_ID(SharedGCMarkingBarrier), {glue_, dstAddr_, offset, value});
167 Jump(&endLoop);
168 }
169 }
170 }
171 }
172 Bind(&endLoop);
173 index = Int32Add(*index, Int32(1));
174 LoopEnd(&begin);
175 Bind(&exit);
176 env->SubCfgExit();
177 }
178
BarrierBatchBitSet(uint8_t bitSetSelect)179 void BarrierStubBuilder::BarrierBatchBitSet(uint8_t bitSetSelect)
180 {
181 auto env = GetEnvironment();
182 Label entry(env);
183 env->SubCfgEntry(&entry);
184
185 Label beforeExit(env);
186 Label exit(env);
187 Label begin(env);
188 Label loopBody(env);
189 Label endLoop(env);
190 GateRef initByteIdx = TruncPtrToInt32(IntPtrLSR(PtrSub(TaggedCastToIntPtr(dstAddr_), objectRegion_),
191 IntPtr(TAGGED_TYPE_SIZE_LOG)));
192 GateRef initQuadIdx = Int32LSR(initByteIdx, Int32(BIT_PER_QUAD_LOG2));
193 DEFVARIABLE(quadIdx, VariableType::INT32(), initQuadIdx);
194 DEFVARIABLE(localToShareBitSet, VariableType::INT64(), Int64(0));
195 DEFVARIABLE(oldToNewBitSet, VariableType::INT64(), Int64(0));
196 DEFVARIABLE(index, VariableType::INT32(), initByteIdx);
197
198 Jump(&begin);
199 LoopBegin(&begin);
200 {
201 BRANCH_NO_WEIGHT(Int32UnsignedLessThan(*index, Int32Add(initByteIdx, slotCount_)), &loopBody, &beforeExit);
202 Bind(&loopBody);
203 {
204 Label checkLocalRegion(env);
205 Label updateLocalToShared(env);
206 Label updateOldToNew(env);
207 GateRef offset = PtrMul(ZExtInt32ToPtr(*index), IntPtr(JSTaggedValue::TaggedTypeSize()));
208 GateRef value = Load(VariableType::JS_ANY(), glue_, objectRegion_, offset);
209 Label doUpdate(env);
210 BRANCH_NO_WEIGHT(TaggedIsHeapObject(value), &doUpdate, &endLoop);
211 Bind(&doUpdate);
212 {
213 GateRef valueRegion = ObjectAddressToRange(value);
214 Label flushLast(env);
215 Label curCheck(env);
216 GateRef curQuadIdx = Int32LSR(*index, Int32(BIT_PER_QUAD_LOG2));
217 GateRef bitOffset = Int32And(*index, Int32(BIT_PER_QUAD_MASK));
218 GateRef mask = Int64LSL(Int64(1), ZExtInt32ToInt64(bitOffset));
219 BRANCH_LIKELY(Int32Equal(curQuadIdx, *quadIdx), &curCheck, &flushLast);
220 Bind(&flushLast);
221 {
222 Label updateIdx(env);
223 FlushBatchBitSet(bitSetSelect, *quadIdx, localToShareBitSet, oldToNewBitSet, &updateIdx);
224 Bind(&updateIdx);
225 quadIdx = curQuadIdx;
226 Jump(&curCheck);
227 }
228 Bind(&curCheck);
229 if ((bitSetSelect & LocalToShared) != 0) {
230 BRANCH_NO_WEIGHT(InSharedSweepableSpace(valueRegion), &updateLocalToShared, &checkLocalRegion);
231 Bind(&updateLocalToShared);
232 {
233 localToShareBitSet = Int64Or(*localToShareBitSet, mask);
234 Jump(&endLoop);
235 }
236 Bind(&checkLocalRegion);
237 }
238 if ((bitSetSelect & OldToNew) != 0) {
239 BRANCH_NO_WEIGHT(InYoungGeneration(valueRegion), &updateOldToNew, &endLoop);
240 Bind(&updateOldToNew);
241 {
242 oldToNewBitSet = Int64Or(*oldToNewBitSet, mask);
243 }
244 }
245 Jump(&endLoop);
246 }
247 }
248 Bind(&endLoop);
249 index = Int32Add(*index, Int32(1));
250 LoopEnd(&begin);
251 }
252 Bind(&beforeExit);
253 FlushBatchBitSet(bitSetSelect, *quadIdx, localToShareBitSet, oldToNewBitSet, &exit);
254 Bind(&exit);
255 env->SubCfgExit();
256 }
257
FlushBatchBitSet(uint8_t bitSetSelect,GateRef quadIdx,Variable & localToShareBitSetVar,Variable & oldToNewBitSetVar,Label * next)258 void BarrierStubBuilder::FlushBatchBitSet(uint8_t bitSetSelect, GateRef quadIdx,
259 Variable &localToShareBitSetVar, Variable &oldToNewBitSetVar, Label* next)
260 {
261 auto env = GetEnvironment();
262 Label batchUpdateLocalToShare(env);
263 Label checkNext(env);
264 Label batchUpdateOldToNew(env);
265 if ((bitSetSelect & LocalToShared) != 0) {
266 BRANCH_LIKELY(Int64NotEqual(*localToShareBitSetVar, Int64(0)), &batchUpdateLocalToShare, &checkNext);
267 Bind(&batchUpdateLocalToShare);
268 {
269 GateRef loadOffset = IntPtr(Region::PackedData::GetLocalToShareSetOffset(env->Is32Bit()));
270 GateRef bitSetAddr = GetBitSetDataAddr(objectRegion_, loadOffset, RTSTUB_ID(CreateLocalToShare));
271
272 GateRef byteIndex = Int32LSL(quadIdx, Int32(BYTE_PER_QUAD_LOG2));
273 GateRef oldValue = LoadPrimitive(VariableType::INT64(), bitSetAddr, ZExtInt32ToInt64(byteIndex));
274 Store(VariableType::INT64(), glue_, bitSetAddr, ZExtInt32ToInt64(byteIndex),
275 Int64Or(oldValue, *localToShareBitSetVar), MemoryAttribute::NoBarrier());
276 localToShareBitSetVar = Int64(0);
277 Jump(&checkNext);
278 }
279 Bind(&checkNext);
280 }
281 if ((bitSetSelect & OldToNew) != 0) {
282 BRANCH_LIKELY(Int64NotEqual(*oldToNewBitSetVar, Int64(0)), &batchUpdateOldToNew, next);
283 Bind(&batchUpdateOldToNew);
284 {
285 GateRef loadOffset = IntPtr(Region::PackedData::GetOldToNewSetOffset(env->Is32Bit()));
286 GateRef bitSetAddr = GetBitSetDataAddr(objectRegion_, loadOffset, RTSTUB_ID(CreateOldToNew));
287 GateRef byteIndex = Int32LSL(quadIdx, Int32(BYTE_PER_QUAD_LOG2));
288 GateRef oldValue = LoadPrimitive(VariableType::INT64(), bitSetAddr, ZExtInt32ToInt64(byteIndex));
289 Store(VariableType::INT64(), glue_, bitSetAddr, ZExtInt32ToInt64(byteIndex),
290 Int64Or(oldValue, *oldToNewBitSetVar), MemoryAttribute::NoBarrier());
291 oldToNewBitSetVar = Int64(0);
292 }
293 }
294 Jump(next);
295 }
296
DoMoveBarrierCrossRegion(GateRef srcAddr,GateRef srcObj)297 void BarrierStubBuilder::DoMoveBarrierCrossRegion(GateRef srcAddr, GateRef srcObj)
298 {
299 auto env = GetEnvironment();
300 Label entry(env);
301 env->SubCfgEntry(&entry);
302 Label exit(env);
303 Label checkNext(env);
304 BRANCH_UNLIKELY(
305 LoadPrimitive(VariableType::BOOL(), glue_, IntPtr(JSThread::GlueData::GetIsEnableCMCGCOffset(env->Is32Bit()))),
306 &exit, &checkNext);
307 Bind(&checkNext);
308 Label handleMark(env);
309 Label handleBitSet(env);
310 Label doMove(env);
311 BRANCH_NO_WEIGHT(Int32Equal(slotCount_, Int32(0)), &exit, &doMove);
312 Bind(&doMove);
313 BRANCH_NO_WEIGHT(InSharedHeap(objectRegion_), &handleMark, &handleBitSet);
314 Bind(&handleBitSet);
315 {
316 GateRef srcRegion = ObjectAddressToRange(srcObj);
317 Label batchBarrier(env);
318 Label srcNotInSharedHeap(env);
319 // copy shared array to local heap, there is no bitset can be copied, do batch barrier for dst object.
320 BRANCH_UNLIKELY(InSharedHeap(srcRegion), &batchBarrier, &srcNotInSharedHeap);
321 Bind(&srcNotInSharedHeap);
322 {
323 GateRef sameKind = LogicOrBuilder(env)
324 .Or(BitAnd(InYoungGeneration(objectRegion_), InYoungGeneration(srcRegion)))
325 .Or(BitAnd(InGeneralOldGeneration(objectRegion_), InGeneralOldGeneration(srcRegion)))
326 .Done();
327 Label sameRegionKind(env);
328 Label crossRegion(env);
329 BRANCH_NO_WEIGHT(sameKind, &sameRegionKind, &crossRegion);
330 Bind(&sameRegionKind);
331 {
332 // dst and src are in same kind region, copy the bitset of them.
333 DoMoveBarrierSameRegionKind(srcAddr, srcRegion, CrossRegion);
334 Jump(&handleMark);
335 }
336 Bind(&crossRegion);
337 {
338 Label copyLocalToShare(env);
339 BRANCH_UNLIKELY(IsLocalToShareSwapped(srcRegion), &batchBarrier, ©LocalToShare);
340 Bind(©LocalToShare);
341 {
342 // dst and src are in different kind region, localToShare bitset can be copied unconditionally,
343 GateRef srcBitStartIdx = Int64LSR(Int64Sub(srcAddr, srcRegion), Int64(TAGGED_TYPE_SIZE_LOG));
344 GateRef dstBitStartIdx = Int64LSR(Int64Sub(dstAddr_, objectRegion_), Int64(TAGGED_TYPE_SIZE_LOG));
345
346 GateRef localToShareOffset = IntPtr(Region::PackedData::GetLocalToShareSetOffset(env->Is32Bit()));
347 GateRef localToShareBitSetAddr =
348 GetBitSetDataAddr(objectRegion_, localToShareOffset, RTSTUB_ID(CreateLocalToShare));
349 GateRef srcLocalToShareBitSetAddr =
350 GetBitSetDataAddr(srcRegion, localToShareOffset, RTSTUB_ID(CreateLocalToShare));
351 BitSetRangeMoveForward(srcLocalToShareBitSetAddr, localToShareBitSetAddr, srcBitStartIdx,
352 dstBitStartIdx, ZExtInt32ToInt64(slotCount_));
353 Label inOld(env);
354 BRANCH_NO_WEIGHT(InYoungGeneration(objectRegion_), &handleMark, &inOld);
355 Bind(&inOld);
356 {
357 // copy young object to old region, do BarrierBatchBitSet bitset for oldToNew bitset.
358 BarrierBatchBitSet(OldToNew);
359 Jump(&handleMark);
360 }
361 }
362 }
363 }
364 Bind(&batchBarrier);
365 {
366 DoBatchBarrierInternal();
367 Jump(&handleMark);
368 }
369 }
370 Bind(&handleMark);
371 HandleMark();
372 Jump(&exit);
373 Bind(&exit);
374 env->SubCfgExit();
375 }
376
DoMoveBarrierInRegion(GateRef srcAddr)377 void BarrierStubBuilder::DoMoveBarrierInRegion(GateRef srcAddr)
378 {
379 auto env = GetEnvironment();
380 Label entry(env);
381 env->SubCfgEntry(&entry);
382 Label exit(env);
383 Label checkNext(env);
384 BRANCH_UNLIKELY(
385 LoadPrimitive(VariableType::BOOL(), glue_, IntPtr(JSThread::GlueData::GetIsEnableCMCGCOffset(env->Is32Bit()))),
386 &exit, &checkNext);
387 Bind(&checkNext);
388 Label handleMark(env);
389 Label handleBitSet(env);
390 Label doMove(env);
391 BRANCH_NO_WEIGHT(Int32Equal(slotCount_, Int32(0)), &exit, &doMove);
392 Bind(&doMove);
393 BRANCH_NO_WEIGHT(InSharedHeap(objectRegion_), &handleMark, &handleBitSet);
394 Bind(&handleBitSet);
395 {
396 DoMoveBarrierSameRegionKind(srcAddr, objectRegion_, InSameRegion);
397 Jump(&handleMark);
398 }
399 Bind(&handleMark);
400 HandleMark();
401 Jump(&exit);
402 Bind(&exit);
403 env->SubCfgExit();
404 }
405
DoMoveBarrierSameRegionKind(GateRef srcAddr,GateRef srcRegion,RegionKind regionKind)406 void BarrierStubBuilder::DoMoveBarrierSameRegionKind(GateRef srcAddr, GateRef srcRegion, RegionKind regionKind)
407 {
408 auto env = GetEnvironment();
409 Label entry(env);
410 env->SubCfgEntry(&entry);
411 Label exit(env);
412 Label inYoung(env);
413 Label inOld(env);
414 Label copyBitSet(env);
415
416 DEFVARIABLE(dstBitSetAddr, VariableType::NATIVE_POINTER(), IntPtr(0));
417 DEFVARIABLE(dstBitSetAddr2, VariableType::NATIVE_POINTER(), IntPtr(0));
418 DEFVARIABLE(srcBitSetAddr, VariableType::NATIVE_POINTER(), IntPtr(0));
419 DEFVARIABLE(srcBitSetAddr2, VariableType::NATIVE_POINTER(), IntPtr(0));
420
421 GateRef localToShareOffset = IntPtr(Region::PackedData::GetLocalToShareSetOffset(env->Is32Bit()));
422 GateRef localToShareBitSetAddr =
423 GetBitSetDataAddr(objectRegion_, localToShareOffset, RTSTUB_ID(CreateLocalToShare));
424 GateRef srcLocalToShareBitSetAddr = Gate::InvalidGateRef;
425 if (regionKind == InSameRegion) {
426 srcLocalToShareBitSetAddr = localToShareBitSetAddr;
427 } else {
428 ASSERT(regionKind == CrossRegion);
429 srcLocalToShareBitSetAddr = GetBitSetDataAddr(srcRegion, localToShareOffset, RTSTUB_ID(CreateLocalToShare));
430 }
431 GateRef localToShareSwapped = IsLocalToShareSwapped(srcRegion);
432 BRANCH_NO_WEIGHT(InYoungGeneration(objectRegion_), &inYoung, &inOld);
433 Bind(&inYoung);
434 {
435 Label batchBarrier(env);
436 Label copyLocalToShare(env);
437 BRANCH_UNLIKELY(localToShareSwapped, &batchBarrier, ©LocalToShare);
438 Bind(&batchBarrier);
439 {
440 // slowpath, localToShareRSet is swapped, it can't be copied, just set the bitset bit by bit.
441 BarrierBatchBitSet(LocalToShared);
442 Jump(&exit);
443 }
444 Bind(©LocalToShare);
445 {
446 dstBitSetAddr = localToShareBitSetAddr;
447 srcBitSetAddr = srcLocalToShareBitSetAddr;
448 Jump(©BitSet);
449 }
450 }
451 Bind(&inOld);
452 {
453 GateRef oldToNewOffset = IntPtr(Region::PackedData::GetOldToNewSetOffset(env->Is32Bit()));
454 GateRef oldToNewBitSetAddr = GetBitSetDataAddr(objectRegion_, oldToNewOffset, RTSTUB_ID(CreateOldToNew));
455 GateRef srcOldToNewBitSetAddr = GetBitSetDataAddr(srcRegion, oldToNewOffset, RTSTUB_ID(CreateOldToNew));
456 // CreateOldToNew may change the RSetFlag, so load it again.
457 GateRef oldToNewSwapped = IsOldToNewSwapped(srcRegion);
458 Label batchBarrier(env);
459 Label tryBatchBarrier(env);
460 Label copyBoth(env);
461 // both localToShareSwapped and oldToNewSwapped are not swapped.
462 BRANCH_LIKELY(BitAnd(BoolNot(localToShareSwapped), BoolNot(oldToNewSwapped)), ©Both, &tryBatchBarrier);
463 Bind(©Both);
464 {
465 dstBitSetAddr = localToShareBitSetAddr;
466 srcBitSetAddr = srcLocalToShareBitSetAddr;
467 dstBitSetAddr2 = oldToNewBitSetAddr;
468 srcBitSetAddr2 = srcOldToNewBitSetAddr;
469 Jump(©BitSet);
470 }
471 Bind(&tryBatchBarrier);
472 {
473 Label copyOne(env);
474 BRANCH_UNLIKELY(BitAnd(localToShareSwapped, oldToNewSwapped), &batchBarrier, ©One);
475 Bind(&batchBarrier);
476 {
477 // slowpath, localToShareRSet and oldToNewRSet are swapped,
478 // it can't be copied, just set the bitset bit by bit.
479 BarrierBatchBitSet(LocalToShared | OldToNew);
480 Jump(&exit);
481 }
482 Bind(©One);
483 {
484 Label localToShareBatchBarrier(env);
485 Label oldToNewBatchBarrier(env);
486 BRANCH_NO_WEIGHT(localToShareSwapped, &localToShareBatchBarrier, &oldToNewBatchBarrier);
487 Bind(&localToShareBatchBarrier);
488 {
489 // slowpath, localToShareRSet is swapped, it can't be copied, just set the bitset bit by bit.
490 // And copy oldToNewRSet.
491 BarrierBatchBitSet(LocalToShared);
492 dstBitSetAddr = oldToNewBitSetAddr;
493 srcBitSetAddr = srcOldToNewBitSetAddr;
494 Jump(©BitSet);
495 }
496 Bind(&oldToNewBatchBarrier);
497 {
498 // slowpath, oldToNewRSet is swapped, it can't be copied, just set the bitset bit by bit.
499 // And copy localToShareRSet.
500 BarrierBatchBitSet(OldToNew);
501 dstBitSetAddr = localToShareBitSetAddr;
502 srcBitSetAddr = srcLocalToShareBitSetAddr;
503 Jump(©BitSet);
504 }
505 }
506 }
507 }
508 Bind(©BitSet);
509 {
510 GateRef srcBitStartIdx = Int64LSR(Int64Sub(srcAddr, srcRegion), Int64(TAGGED_TYPE_SIZE_LOG));
511 GateRef dstBitStartIdx = Int64LSR(Int64Sub(dstAddr_, objectRegion_), Int64(TAGGED_TYPE_SIZE_LOG));
512 Label begin(env);
513 Label endLoop(env);
514 Label next(env);
515 Jump(&begin);
516 LoopBegin(&begin);
517 {
518 if (regionKind == InSameRegion) {
519 BitSetRangeMove(*srcBitSetAddr, *dstBitSetAddr, srcBitStartIdx, dstBitStartIdx,
520 ZExtInt32ToInt64(slotCount_));
521 } else {
522 ASSERT(regionKind == CrossRegion);
523 BitSetRangeMoveForward(*srcBitSetAddr, *dstBitSetAddr, srcBitStartIdx, dstBitStartIdx,
524 ZExtInt32ToInt64(slotCount_));
525 }
526 BRANCH_UNLIKELY(IntPtrEqual(*dstBitSetAddr2, IntPtr(0)), &exit, &next);
527 Bind(&next);
528 {
529 dstBitSetAddr = *dstBitSetAddr2;
530 srcBitSetAddr = *srcBitSetAddr2;
531 dstBitSetAddr2 = IntPtr(0);
532 srcBitSetAddr2 = IntPtr(0);
533 Jump(&endLoop);
534 }
535 }
536 Bind(&endLoop);
537 LoopEnd(&begin);
538 }
539 Bind(&exit);
540 env->SubCfgExit();
541 }
542
BitSetRangeMove(GateRef srcBitSet,GateRef dstBitSet,GateRef srcStart,GateRef dstStart,GateRef length)543 void BarrierStubBuilder::BitSetRangeMove(GateRef srcBitSet, GateRef dstBitSet, GateRef srcStart, GateRef dstStart,
544 GateRef length)
545 {
546 auto env = GetEnvironment();
547 Label entry(env);
548 env->SubCfgEntry(&entry);
549 Label exit(env);
550 length = ZExtInt32ToInt64(length);
551
552 GateRef forward = LogicOrBuilder(env)
553 .Or(Int64LessThan(dstStart, srcStart))
554 .Or(Int64GreaterThanOrEqual(dstStart, Int64Add(srcStart, length))).Done();
555 Label moveForward(env);
556 Label moveBackward(env);
557 BRANCH_NO_WEIGHT(forward, &moveForward, &moveBackward);
558 Bind(&moveForward);
559 {
560 BitSetRangeMoveForward(srcBitSet, dstBitSet, srcStart, dstStart, length);
561 Jump(&exit);
562 }
563 Bind(&moveBackward);
564 {
565 BitSetRangeMoveBackward(srcBitSet, dstBitSet, srcStart, dstStart, length);
566 Jump(&exit);
567 }
568 Bind(&exit);
569 env->SubCfgExit();
570 }
571
BitSetRangeMoveForward(GateRef srcBitSet,GateRef dstBitSet,GateRef srcStart,GateRef dstStart,GateRef length)572 void BarrierStubBuilder::BitSetRangeMoveForward(GateRef srcBitSet, GateRef dstBitSet, GateRef srcStart,
573 GateRef dstStart, GateRef length)
574 {
575 auto env = GetEnvironment();
576 Label entry(env);
577 env->SubCfgEntry(&entry);
578 Label exit(env);
579
580 DEFVARIABLE(remainLength, VariableType::INT64(), length);
581 // srcQuad <- srcStart / 64; // 源起始 uint64_t 元素索引
582 // dstQuad <- dstStart / 64; // 目标起始 uint64_t 元素索引
583 // srcBitOffset <- srcStart % 64; // 源起始元素的位偏移
584 // dstBitOffset <- dstStart % 64; // 目标起始元素的位偏移
585 DEFVARIABLE(srcQuad, VariableType::INT64(), Int64LSR(srcStart, Int64(BIT_PER_QUAD_LOG2)));
586 DEFVARIABLE(dstQuad, VariableType::INT64(), Int64LSR(dstStart, Int64(BIT_PER_QUAD_LOG2)));
587 DEFVARIABLE(srcBitOffset, VariableType::INT64(), Int64And(srcStart, Int64(BIT_PER_QUAD_MASK)));
588 DEFVARIABLE(dstBitOffset, VariableType::INT64(), Int64And(dstStart, Int64(BIT_PER_QUAD_MASK)));
589
590 Label begin(env);
591 Label body(env);
592 Label endLoop(env);
593 Jump(&begin);
594 LoopBegin(&begin);
595
596 // while remainLength > 0
597 // do
598 BRANCH_LIKELY(Int64GreaterThan(*remainLength, Int64(0)), &body, &exit);
599 Bind(&body);
600 {
601 Label beforeEndLoop(env);
602 // bitsInCurrentQuad <- min of {64 - srcBitOffset, 64 - dstBitOffset, remainLength};
603 GateRef bitsInCurrentQuad = ThreeInt64Min(Int64Sub(Int64(BIT_PER_QUAD), *srcBitOffset),
604 Int64Sub(Int64(BIT_PER_QUAD), *dstBitOffset),
605 *remainLength);
606 GateRef srcQuadData = LoadPrimitive(VariableType::INT64(), srcBitSet,
607 Int64LSL(*srcQuad, Int64(BYTE_PER_QUAD_LOG2)));
608 Label setValue(env);
609 BRANCH_NO_WEIGHT(Int64Equal(srcQuadData, Int64(0)), &beforeEndLoop, &setValue);
610 Bind(&setValue);
611 {
612 // srcMask <- 0xFFFFFFFFFFFFFFFF >> (64 - bitsInCurrentQuad);
613 GateRef srcMask = Int64LSR(Int64(ALL_ONE_MASK), Int64Sub(Int64(BIT_PER_QUAD), bitsInCurrentQuad));
614 // srcData <- (srcBitSet[srcQuad * 8] >> srcBitOffset) & srcMask;
615 GateRef srcData = Int64And(Int64LSR(srcQuadData, *srcBitOffset), srcMask);
616 // zeroMask <- ~(srcMask << dstBitOffset);
617 GateRef zeroMask = Int64Not(Int64LSL(srcMask, *dstBitOffset));
618 // dstData <- load from dstBitSet offset dstQuad * 8;
619 GateRef dstData = LoadPrimitive(VariableType::INT64(), dstBitSet,
620 Int64LSL(*dstQuad, Int64(BYTE_PER_QUAD_LOG2)));
621 // newDataMask <- (srcData << dstBitOffset);
622 GateRef newDataMask = Int64LSL(srcData, *dstBitOffset);
623 // newData <- dstData & zeroMask | newDataMask;
624 GateRef newData = Int64Or(Int64And(dstData, zeroMask), newDataMask);
625 // bitSet[dstQuad * 8] <- newData;
626 Store(VariableType::INT64(), glue_, dstBitSet, Int64LSL(*dstQuad, Int64(BYTE_PER_QUAD_LOG2)), newData);
627 Jump(&beforeEndLoop);
628 }
629 Bind(&beforeEndLoop);
630 {
631 // remainLength <- remainLength - bitsInCurrentQuad;
632 remainLength = Int64Sub(*remainLength, bitsInCurrentQuad);
633 // srcBitOffset <- srcBitOffset + bitsInCurrentQuad;
634 srcBitOffset = Int64Add(*srcBitOffset, bitsInCurrentQuad);
635 // dstBitOffset <- dstBitOffset + bitsInCurrentQuad;
636 dstBitOffset = Int64Add(*dstBitOffset, bitsInCurrentQuad);
637 Label srcAdd(env);
638 Label dstAdd(env);
639 Label checkDst(env);
640 BRANCH_NO_WEIGHT(Int64Equal(*srcBitOffset, Int64(BIT_PER_QUAD)), &srcAdd, &checkDst);
641 Bind(&srcAdd);
642 {
643 // if srcBitOffset == 64
644 // then srcQuad <- srcQuad + 1; srcBitOffset <- 0
645 srcQuad = Int64Add(*srcQuad, Int64(1));
646 srcBitOffset = Int64(0);
647 Jump(&checkDst);
648 }
649 Bind(&checkDst);
650 BRANCH_NO_WEIGHT(Int64Equal(*dstBitOffset, Int64(BIT_PER_QUAD)), &dstAdd, &endLoop);
651 Bind(&dstAdd);
652 {
653 // if dstBitOffset == 64
654 // then dstQuad <- dstQuad + 1; dstBitOffset <- 0}
655 dstQuad = Int64Add(*dstQuad, Int64(1));
656 dstBitOffset = Int64(0);
657 Jump(&endLoop);
658 }
659 }
660 Bind(&endLoop);
661 LoopEnd(&begin);
662 }
663 Bind(&exit);
664 env->SubCfgExit();
665 }
666
BitSetRangeMoveBackward(GateRef srcBitSet,GateRef dstBitSet,GateRef srcStart,GateRef dstStart,GateRef length)667 void BarrierStubBuilder::BitSetRangeMoveBackward(GateRef srcBitSet, GateRef dstBitSet, GateRef srcStart,
668 GateRef dstStart, GateRef length)
669 {
670 auto env = GetEnvironment();
671 Label entry(env);
672 env->SubCfgEntry(&entry);
673 Label exit(env);
674 DEFVARIABLE(remainLength, VariableType::INT64(), length);
675 // srcEnd <- srcStart + remainLength - 1;
676 // dstEnd <- dstStart + remainLength - 1;
677 DEFVARIABLE(srcEnd, VariableType::INT64(), Int64Sub(Int64Add(srcStart, length), Int64(1)));
678 DEFVARIABLE(dstEnd, VariableType::INT64(), Int64Sub(Int64Add(dstStart, length), Int64(1)));
679
680 Label begin(env);
681 Label body(env);
682 Label endLoop(env);
683 Jump(&begin);
684 LoopBegin(&begin);
685 // while remainLength > 0
686 BRANCH_LIKELY(Int64GreaterThan(*remainLength, Int64(0)), &body, &exit);
687 Bind(&body);
688 {
689 Label beforeEndLoop(env);
690 // srcBitOffset <- srcEnd % 64;
691 GateRef srcBitOffset = Int64And(*srcEnd, Int64(BIT_PER_QUAD_MASK));
692 // dstBitOffset <- dstEnd % 64;
693 GateRef dstBitOffset = Int64And(*dstEnd, Int64(BIT_PER_QUAD_MASK));
694 // bitsInCurrentQuad <- min of {srcBitOffset + 1, dstBitOffset + 1, remainLength};
695 GateRef bitsInCurrentQuad = ThreeInt64Min(Int64Add(srcBitOffset, Int64(1)),
696 Int64Add(dstBitOffset, Int64(1)),
697 *remainLength);
698 // srcQuadIndex <- srcEnd / 64;
699 GateRef srcQuadIndex = Int64LSR(*srcEnd, Int64(BIT_PER_QUAD_LOG2));
700 GateRef srcQuadData = LoadPrimitive(VariableType::INT64(), srcBitSet,
701 Int64LSL(srcQuadIndex, Int64(BYTE_PER_QUAD_LOG2)));
702 Label setValue(env);
703 BRANCH_NO_WEIGHT(Int64Equal(srcQuadData, Int64(0)), &beforeEndLoop, &setValue);
704 Bind(&setValue);
705 {
706 // mask <- 0xFFFFFFFFFFFFFFFF >> (64 - bitsInCurrentQuad);
707 GateRef mask = Int64LSR(Int64(ALL_ONE_MASK), Int64Sub(Int64(BIT_PER_QUAD), bitsInCurrentQuad));
708 // srcData <- (srcBitSet[srcQuadIndex * 8] >> (srcBitOffset - bitsInCurrentQuad + 1)) & mask;
709 GateRef srcData = Int64And(
710 Int64LSR(srcQuadData, Int64Add(Int64Sub(srcBitOffset, bitsInCurrentQuad), Int64(1))), mask);
711 // zeroMask <- ~(mask << (dstBitOffset - bitsInCurrentQuad + 1));
712 GateRef zeroMask = Int64Not(Int64LSL(mask, Int64Add(Int64Sub(dstBitOffset, bitsInCurrentQuad), Int64(1))));
713 // dstWordIndex <- dstEnd / 64;
714 GateRef dstQuadIndex = Int64LSR(*dstEnd, Int64(BIT_PER_QUAD_LOG2));
715 // dstData <- load from dstBitSet offset dstQuadIndex * 8
716 GateRef dstData = LoadPrimitive(VariableType::INT64(), dstBitSet,
717 Int64LSL(dstQuadIndex, Int64(BYTE_PER_QUAD_LOG2)));
718 // newDataMask <- (srcData << (dstBitOffset - bitsInCurrentQuad + 1))
719 GateRef newDataMask = Int64LSL(srcData, Int64Add(Int64Sub(dstBitOffset, bitsInCurrentQuad), Int64(1)));
720 // newData <- dstData & zeroMask | newDataMask;
721 GateRef newData = Int64Or(Int64And(dstData, zeroMask), newDataMask);
722 // bitSet[dstQuadIndex * 8] <- newData;
723 Store(VariableType::INT64(), glue_, dstBitSet, Int64LSL(dstQuadIndex, Int64(BYTE_PER_QUAD_LOG2)), newData);
724 Jump(&beforeEndLoop);
725 }
726 Bind(&beforeEndLoop);
727 {
728 // remainLength <- remainLength - bitsInCurrentQuad;
729 remainLength = Int64Sub(*remainLength, bitsInCurrentQuad);
730 // srcEnd <- srcEnd - bitsInCurrentQuad;
731 srcEnd = Int64Sub(*srcEnd, bitsInCurrentQuad);
732 // dstEnd <- dstEnd - bitsInCurrentQuad;
733 dstEnd = Int64Sub(*dstEnd, bitsInCurrentQuad);
734 Jump(&endLoop);
735 }
736 Bind(&endLoop);
737 LoopEnd(&begin);
738 }
739 Bind(&exit);
740 env->SubCfgExit();
741 }
742
DoReverseBarrier()743 void BarrierStubBuilder::DoReverseBarrier()
744 {
745 auto env = GetEnvironment();
746 Label entry(env);
747 Label exit(env);
748 Label checkNext(env);
749 Label needMarking(env);
750 Label notMarkRSet(env);
751 Label markInBuffer(env);
752 Label continueProcessing(env);
753 Label isTaggedObject(env);
754 Label RefisTaggedObject(env);
755 Label markRSet(env);
756 Label continueLoopHead(env);
757 Label continueLoopEnd(env);
758 Label notMarkRSetLoopHead(env);
759 Label notMarkRSetLoopEnd(env);
760 Label iLessLength(env);
761 Label indexLessLength(env);
762 Label notIdlePhase(env);
763 env->SubCfgEntry(&entry);
764
765 BRANCH_UNLIKELY(LoadPrimitive(
766 VariableType::BOOL(), glue_, IntPtr(JSThread::GlueData::GetIsEnableCMCGCOffset(env->Is32Bit()))),
767 &needMarking, &checkNext);
768 Bind(&needMarking);
769 {
770 GateRef gcPhase = GetGCPhase(glue_);
771 GateRef objRegionType = GetCMCRegionType(dstObj_);
772 GateRef shouldProcessSATB = ShouldProcessSATB(gcPhase);
773 BRANCH(Int8Equal(gcPhase, Int8(common::GCPhase::GC_PHASE_IDLE)), &exit, ¬IdlePhase);
774 Bind(¬IdlePhase);
775 {
776 BRANCH(CMCIsInYoungSpace(objRegionType), ¬MarkRSet, &continueProcessing);
777 Bind(&continueProcessing);
778 {
779 DEFVARIABLE(i, VariableType::INT32(), Int32(0));
780 Label checkOldToYoung(env);
781 BRANCH(ShouldUpdateRememberSet(glue_, gcPhase), &checkOldToYoung, ¬MarkRSet);
782 Bind(&checkOldToYoung);
783 Jump(&continueLoopHead);
784 LoopBegin(&continueLoopHead);
785 {
786 BRANCH(Int32UnsignedLessThan(*i, slotCount_), &iLessLength, ¬MarkRSet);
787 Bind(&iLessLength);
788 {
789 GateRef offset = PtrMul(ZExtInt32ToPtr(*i), IntPtr(JSTaggedValue::TaggedTypeSize()));
790 GateRef ref = LoadPrimitive(VariableType::JS_ANY(), dstAddr_, offset);
791 BRANCH(TaggedIsHeapObject(ref), &isTaggedObject, &continueLoopEnd);
792 }
793 Bind(&isTaggedObject);
794 MarkRSetCardTable(dstObj_, ¬MarkRSet);
795 Bind(&continueLoopEnd);
796 i = Int32Add(*i, Int32(1));
797 LoopEnd(&continueLoopHead);
798 }
799 }
800 Bind(¬MarkRSet);
801 {
802 DEFVARIABLE(index, VariableType::INT32(), Int32(0));
803 GateRef shouldProcessSATB = ShouldProcessSATB(gcPhase);
804 Jump(¬MarkRSetLoopHead);
805 LoopBegin(¬MarkRSetLoopHead);
806 {
807 BRANCH_LIKELY(Int32UnsignedLessThan(*index, slotCount_), &indexLessLength, &exit);
808 Bind(&indexLessLength);
809 GateRef offset = PtrMul(ZExtInt32ToPtr(*index), IntPtr(JSTaggedValue::TaggedTypeSize()));
810 GateRef ref = LoadPrimitive(VariableType::JS_ANY(), dstAddr_, offset);
811 BRANCH(TaggedIsHeapObject(ref), &RefisTaggedObject, ¬MarkRSetLoopEnd);
812 Bind(&RefisTaggedObject);
813 BRANCH_UNLIKELY(shouldProcessSATB, &markInBuffer, &exit);
814 Bind(&markInBuffer);
815 {
816 ASSERT(RuntimeStubCSigns::Get(RTSTUB_ID(MarkInBuffer))->IsNoTailCall());
817 CallNGCRuntime(glue_, RTSTUB_ID(MarkInBuffer), {ref});
818 Jump(¬MarkRSetLoopEnd);
819 }
820
821 Bind(¬MarkRSetLoopEnd);
822 index = Int32Add(*index, Int32(1));
823 LoopEnd(¬MarkRSetLoopHead);
824 }
825 }
826 }
827 }
828 Bind(&checkNext);
829 {
830 Label handleMark(env);
831 Label handleBitSet(env);
832 Label doReverse(env);
833 BRANCH_NO_WEIGHT(Int32Equal(slotCount_, Int32(0)), &exit, &doReverse);
834 Bind(&doReverse);
835 BRANCH_NO_WEIGHT(InSharedHeap(objectRegion_), &handleMark, &handleBitSet);
836 Bind(&handleBitSet);
837 {
838 DoReverseBarrierInternal();
839 Jump(&handleMark);
840 }
841 Bind(&handleMark);
842 HandleMark();
843 Jump(&exit);
844 }
845 Bind(&exit);
846 env->SubCfgExit();
847 }
848
849
DoReverseBarrierInternal()850 void BarrierStubBuilder::DoReverseBarrierInternal()
851 {
852 auto env = GetEnvironment();
853 Label entry(env);
854 env->SubCfgEntry(&entry);
855 Label exit(env);
856 Label inYoung(env);
857 Label inOld(env);
858 Label copyBitSet(env);
859
860 DEFVARIABLE(bitSetAddr, VariableType::NATIVE_POINTER(), IntPtr(0));
861 DEFVARIABLE(bitSetAddr2, VariableType::NATIVE_POINTER(), IntPtr(0));
862
863 GateRef localToShareOffset = IntPtr(Region::PackedData::GetLocalToShareSetOffset(env->Is32Bit()));
864 GateRef localToShareBitSetAddr =
865 GetBitSetDataAddr(objectRegion_, localToShareOffset, RTSTUB_ID(CreateLocalToShare));
866
867 GateRef localToShareSwapped = IsLocalToShareSwapped(objectRegion_);
868 BRANCH_NO_WEIGHT(InYoungGeneration(objectRegion_), &inYoung, &inOld);
869 Bind(&inYoung);
870 {
871 Label batchBarrier(env);
872 Label reverseLocalToShare(env);
873 BRANCH_UNLIKELY(localToShareSwapped, &batchBarrier, &reverseLocalToShare);
874 Bind(&batchBarrier);
875 {
876 // slowpath, localToShareRSet is swapped, it can't be reversed, just set the bitset bit by bit.
877 BarrierBatchBitSet(LocalToShared);
878 Jump(&exit);
879 }
880 Bind(&reverseLocalToShare);
881 {
882 bitSetAddr = localToShareBitSetAddr;
883 Jump(©BitSet);
884 }
885 }
886 Bind(&inOld);
887 {
888 GateRef oldToNewOffset = IntPtr(Region::PackedData::GetOldToNewSetOffset(env->Is32Bit()));
889 GateRef oldToNewBitSetAddr = GetBitSetDataAddr(objectRegion_, oldToNewOffset, RTSTUB_ID(CreateOldToNew));
890 // CreateOldToNew may change the RSetFlag, so load it again.
891 GateRef oldToNewSwapped = IsOldToNewSwapped(objectRegion_);
892 Label batchBarrier(env);
893 Label tryBatchBarrier(env);
894 Label copyBoth(env);
895 // both localToShareSwapped and oldToNewSwapped are not swapped.
896 BRANCH_LIKELY(BitAnd(BoolNot(localToShareSwapped), BoolNot(oldToNewSwapped)), ©Both, &tryBatchBarrier);
897 Bind(©Both);
898 {
899 bitSetAddr = localToShareBitSetAddr;
900 bitSetAddr2 = oldToNewBitSetAddr;
901 Jump(©BitSet);
902 }
903 Bind(&tryBatchBarrier);
904 {
905 Label reverseOne(env);
906 BRANCH_UNLIKELY(BitAnd(localToShareSwapped, oldToNewSwapped), &batchBarrier, &reverseOne);
907 Bind(&batchBarrier);
908 {
909 // slowpath, localToShareRSet and oldToNewRSet are swapped,
910 // it can't be reversed, just set the bitset bit by bit.
911 BarrierBatchBitSet(LocalToShared | OldToNew);
912 Jump(&exit);
913 }
914 Bind(&reverseOne);
915 {
916 Label localToShareBatchBarrier(env);
917 Label oldToNewBatchBarrier(env);
918 BRANCH_NO_WEIGHT(localToShareSwapped, &localToShareBatchBarrier, &oldToNewBatchBarrier);
919 Bind(&localToShareBatchBarrier);
920 {
921 // slowpath, localToShareRSet is swapped, it can't be reversed, just set the bitset bit by bit.
922 // And copy oldToNewRSet.
923 BarrierBatchBitSet(LocalToShared);
924 bitSetAddr = oldToNewBitSetAddr;
925 Jump(©BitSet);
926 }
927 Bind(&oldToNewBatchBarrier);
928 {
929 // slowpath, oldToNewRSet is swapped, it can't be reversed, just set the bitset bit by bit.
930 // And copy localToShareRSet.
931 BarrierBatchBitSet(OldToNew);
932 bitSetAddr = localToShareBitSetAddr;
933 Jump(©BitSet);
934 }
935 }
936 }
937 }
938 Bind(©BitSet);
939 {
940 GateRef dstBitStartIdx = Int64LSR(Int64Sub(dstAddr_, objectRegion_), Int64(TAGGED_TYPE_SIZE_LOG));
941 Label begin(env);
942 Label endLoop(env);
943 Label next(env);
944 Jump(&begin);
945 LoopBegin(&begin);
946 {
947 BitSetRangeReverse(*bitSetAddr, dstBitStartIdx, ZExtInt32ToInt64(slotCount_));
948 BRANCH_UNLIKELY(IntPtrEqual(*bitSetAddr2, IntPtr(0)), &exit, &next);
949 Bind(&next);
950 {
951 bitSetAddr = *bitSetAddr2;
952 bitSetAddr2 = IntPtr(0);
953 Jump(&endLoop);
954 }
955 }
956 Bind(&endLoop);
957 LoopEnd(&begin);
958 }
959 Bind(&exit);
960 env->SubCfgExit();
961 }
962
BitSetRangeReverse(GateRef bitSet,GateRef startIdx,GateRef length)963 void BarrierStubBuilder::BitSetRangeReverse(GateRef bitSet, GateRef startIdx, GateRef length)
964 {
965 auto env = GetEnvironment();
966 Label entry(env);
967 env->SubCfgEntry(&entry);
968 Label exit(env);
969 // startOf <- startIdx % 64
970 GateRef startOf = Int64And(startIdx, Int64(BIT_PER_QUAD_MASK));
971 // endOf <- (startIdx + len) % 64
972 GateRef endOf = Int64And(Int64Add(startIdx, length), Int64(BIT_PER_QUAD_MASK));
973
974 // startItemIndex <- startIdx / 64
975 GateRef startItemIndex = Int64LSR(startIdx, Int64(BIT_PER_QUAD_LOG2));
976 // endItemIndex <- (startIdx + len - 1) / 64
977 GateRef endItemIndex = Int64LSR(Int64Sub(Int64Add(startIdx, length), Int64(1)), Int64(BIT_PER_QUAD_LOG2));
978
979 GateRef dstOffset = startIdx;
980 // srcOffset <- startIdx + ((64 - endOf) % 64 - startOf)
981 GateRef srcOffset = Int64Add(startIdx,
982 Int64Sub(Int64And(Int64Sub(Int64(BIT_PER_QUAD), endOf), Int64(BIT_PER_QUAD_MASK)), startOf));
983
984 // startByteIndex <- startItemIndex * 8
985 GateRef startByteIndex = Int64LSL(startItemIndex, Int64(BIT_PER_BYTE_LOG2));
986 // endByteIndex <- endItemIndex * 8
987 GateRef endByteIndex = Int64LSL(endItemIndex, Int64(BIT_PER_BYTE_LOG2));
988
989 GateRef startItem = LoadPrimitive(VariableType::INT64(), bitSet, startByteIndex);
990 GateRef endItem = LoadPrimitive(VariableType::INT64(), bitSet, endByteIndex);
991
992 // startMask <- (1 << startOf) - 1
993 GateRef startMask = Int64Sub(Int64LSL(Int64(1), startOf), Int64(1));
994 Label notZero(env);
995 Label next(env);
996 DEFVARIABLE(endMask, VariableType::INT64(), Int64(0));
997 BRANCH_UNLIKELY(Int64Equal(endOf, Int64(0)), &next, ¬Zero);
998 Bind(¬Zero);
999 {
1000 // endMask <- ~((1 << endOf) - 1)
1001 endMask = Int64Not(Int64Sub(Int64LSL(Int64(1), endOf), Int64(1)));
1002 Jump(&next);
1003 }
1004
1005 Bind(&next);
1006 DEFVARIABLE(vstartByteIndex, VariableType::INT64(), startByteIndex);
1007 DEFVARIABLE(vendByteIndex, VariableType::INT64(), endByteIndex);
1008
1009 Label loopBegin1(env);
1010 Label loopEnd1(env);
1011 Label loopExit1(env);
1012 Label startReverse(env);
1013 Jump(&loopBegin1);
1014 LoopBegin(&loopBegin1);
1015 {
1016 BRANCH_LIKELY(Int64LessThanOrEqual(*vstartByteIndex, *vendByteIndex), &startReverse, &loopExit1);
1017 Bind(&startReverse);
1018 {
1019 GateRef item1 = LoadPrimitive(VariableType::INT64(), bitSet, *vstartByteIndex);
1020 GateRef item2 = LoadPrimitive(VariableType::INT64(), bitSet, *vendByteIndex);
1021
1022 // revStart <- Int64BitReverse(bitSet[vstartByteIndex])
1023 // revEnd <- Int64BitReverse(bitSet[vendByteIndex])
1024 GateRef revStart = Int64BitReverse(item1);
1025 GateRef revEnd = Int64BitReverse(item2);
1026
1027 // bitSet[vstartByteIndex] <- revEnd
1028 // bitSet[vendByteIndex] <- revStart
1029 Store(VariableType::INT64(), glue_, bitSet, *vstartByteIndex, revEnd, MemoryAttribute::NoBarrier());
1030 Store(VariableType::INT64(), glue_, bitSet, *vendByteIndex, revStart, MemoryAttribute::NoBarrier());
1031
1032 // vstartByteIndex <- vstartByteIndex + 8
1033 // vendByteIndex <- vendByteIndex - 8
1034 vstartByteIndex = Int64Add(*vstartByteIndex, Int64(BYTE_PER_QUAD));
1035 vendByteIndex = Int64Sub(*vendByteIndex, Int64(BYTE_PER_QUAD));
1036 Jump(&loopEnd1);
1037 }
1038 }
1039 Bind(&loopEnd1);
1040 LoopEnd(&loopBegin1);
1041 Bind(&loopExit1);
1042 BitSetRangeMove(bitSet, bitSet, srcOffset, dstOffset, length);
1043
1044 GateRef neStartMask = Int64Not(startMask);
1045 GateRef neEndMask = Int64Not(*endMask);
1046
1047 // bitSet[startByteIndex] <- (startItem & startMask) | (startItem1 & neStartMask);
1048 GateRef startItem1 = LoadPrimitive(VariableType::INT64(), bitSet, startByteIndex);
1049 GateRef resStartItem = Int64Or(Int64And(startItem, startMask), Int64And(startItem1, neStartMask));
1050 Store(VariableType::INT64(), glue_, bitSet, startByteIndex, resStartItem, MemoryAttribute::NoBarrier());
1051
1052 // bitSet[endByteIndex] <- (resEndItem & endMask) | (endItem1 & neEndMask);
1053 GateRef endItem1 = LoadPrimitive(VariableType::INT64(), bitSet, endByteIndex);
1054 GateRef resEndItem = Int64Or(Int64And(endItem, *endMask), Int64And(endItem1, neEndMask));
1055 Store(VariableType::INT64(), glue_, bitSet, endByteIndex, resEndItem, MemoryAttribute::NoBarrier());
1056 Jump(&exit);
1057 Bind(&exit);
1058 env->SubCfgExit();
1059 }
1060
IsLocalToShareSwapped(GateRef region)1061 GateRef BarrierStubBuilder::IsLocalToShareSwapped(GateRef region)
1062 {
1063 auto env = GetEnvironment();
1064 GateRef RSetSwapFlagOffset = IntPtr(Region::PackedData::GetRSetSwapFlagOffset(env->Is32Bit()));
1065 GateRef RSetFlag = LoadPrimitive(VariableType::INT8(), region, RSetSwapFlagOffset);
1066 return NotEqual(Int8And(RSetFlag, Int8(LOCAL_TO_SHARE_SWAPPED_MASK)), Int8(0));
1067 }
1068
IsOldToNewSwapped(GateRef region)1069 GateRef BarrierStubBuilder::IsOldToNewSwapped(GateRef region)
1070 {
1071 auto env = GetEnvironment();
1072 GateRef RSetSwapFlagOffset = IntPtr(Region::PackedData::GetRSetSwapFlagOffset(env->Is32Bit()));
1073 GateRef RSetFlag = LoadPrimitive(VariableType::INT8(), region, RSetSwapFlagOffset);
1074 return NotEqual(Int8And(RSetFlag, Int8(OLD_TO_NEW_SWAPPED_MASK)), Int8(0));
1075 }
1076 }
1077