1#!/usr/bin/env ruby 2 3# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16include_relative 'common.irt' 17 18function(:EmptyPostWriteBarrier, 19 params: {}, 20 regmap: $full_regmap, 21 regalloc_set: RegMask.new, 22 mode: [:FastPath]) { 23 if Options.arch == :arm32 24 Intrinsic(:UNREACHABLE).Terminator.void 25 next 26 end 27 ReturnVoid().void 28} 29 30def PostInterGenerationalBarrier(obj_num) 31 if obj_num == 0 32 params = {mem: 'ref_uint'} 33 mask = RegMask.new($full_regmap, :arg0, :tmp0, :tmp1) 34 elsif obj_num == 1 35 params = {mem: 'ref_uint', offset: 'word', obj1: 'ref_uint'} 36 mask = RegMask.new($full_regmap, :arg0, :arg1, :tmp0) 37 elsif obj_num == 2 38 params = {mem: 'ref_uint', offset: 'word', obj1: 'ref_uint', obj2: 'ref_uint'} 39 mask = RegMask.new($full_regmap, :arg0, :arg1, :arg2) 40 else 41 raise "Wrong obj_num #{obj_num}" 42 end 43 44 function("PostInterGenerationalBarrier#{obj_num}".to_sym, 45 params: params, 46 regmap: $full_regmap, 47 regalloc_set: mask, 48 mode: [:FastPath]) { 49 if Options.arch == :arm32 50 Intrinsic(:UNREACHABLE).Terminator.void 51 next 52 end 53 54 min_addr := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_MIN_ADDR_OFFSET).word 55 cards := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_ADDR_OFFSET).ptr 56 57 mem_word := Cast(mem).SrcType(Constants::REF_UINT).word 58 card_offset := ShrI(Sub(mem_word, min_addr).word).Imm(Constants::CARD_TABLE_CARD_BITS).word 59 card := Add(cards, card_offset).ptr 60 StoreI(card, Constants::CARD_DIRTY_VALUE).Imm(Constants::CARD_VALUE_OFFSET).u8 61 ReturnVoid().void 62 } 63end 64 65# G1 PostWrite barrier 66# - Checks if mem and obj are in different regions 67# - Checks if GC Card for mem is not marked, then marks it 68# - Pushes Card to G1 LockFreeBuffer concurrently, GC consumer thread will fetch the Card 69def PostInterRegionBarrierMarkSingleFast() 70 params = {mem: 'ref_uint', offset: 'word', obj1: 'ref_uint'} 71 mask = RegMask.new($full_regmap, :arg0, :arg1, :tmp0) 72 73 function("PostInterRegionBarrierMarkSingleFast".to_sym, 74 params: params, 75 regmap: $full_regmap, 76 regalloc_set: mask, 77 mode: [:FastPath]) { 78 if Options.arch == :arm32 79 Intrinsic(:UNREACHABLE).Terminator.void 80 next 81 end 82 83 If(obj1, 0).EQ.Unlikely { 84 Goto(:Done) 85 } 86 87 If(ShrI(Xor(mem, obj1).ref_uint).Imm(Constants::REGION_SIZE_BIT).ref_uint, 0).EQ.Likely { 88 Goto(:Done) 89 } 90 91 ref_addr := Add(mem, offset).ref_uint 92 LiveOut(ref_addr).DstReg(regmap[:arg0]).ref_uint 93 Intrinsic(:TAIL_CALL).AddImm(Constants::POST_INTER_REGION_BARRIER_SLOW).Terminator.void 94 95 Label(:Done) 96 ReturnVoid().void 97 } 98end 99 100# G1 PostWrite barrier for StorePair instruction. 101# Similar to the previous, but: 102# - Checks if the 2nd object's card should be marked; 103# - If not, checks the 1st object, marks it's card if neccessary and exits. 104# - Otherwise, if 1st and 2nd object's cards are the same (i.e. 105# - 2nd store addr isn't aligned on a beggining of a card), marks the card and exits. 106# - Otherwise, marks the 2nd object's card as well as the 1st obj's card. 107def PostInterRegionBarrierMarkPairFast() 108 params = {mem: 'ref_uint', offset: 'word', obj1: 'ref_uint', obj2: 'ref_uint'} 109 mask = RegMask.new($full_regmap, :arg0, :arg1, :arg2) 110 if Options.arch == :x86_64 111 mask += :tmp0 112 end 113 114 function("PostInterRegionBarrierMarkPairFast".to_sym, 115 params: params, 116 regmap: $full_regmap, 117 regalloc_set: mask, 118 mode: [:FastPath]) { 119 if Options.arch == :arm32 120 Intrinsic(:UNREACHABLE).Terminator.void 121 next 122 end 123 124 125 ref_addr_obj1 := Add(mem, offset).ref_uint 126 127 If(obj2, 0).EQ.Unlikely { 128 Goto(:Check1) 129 } 130 ref_addr_obj2 := AddI(ref_addr_obj1.ref_uint).Imm(Constants::REFERENCE_TYPE_SIZE).ref_uint 131 132 If(ShrI(Xor(mem, obj2).ref_uint).Imm(Constants::REGION_SIZE_BIT).ref_uint, 0).NE.Unlikely { 133 IfImm(AndI(ref_addr_obj2).Imm(Constants::CARD_ALIGNMENT_MASK).ref_uint).Imm(0).EQ.Unlikely { 134 LiveOut(ref_addr_obj2).DstReg(regmap[:arg0]).ref_uint 135 Intrinsic(:TAIL_CALL).AddImm(Constants::POST_INTER_REGION_BARRIER_TWO_CARDS_SLOW).Terminator.void 136 } 137 # No need to sub REFERENCE_TYPE_SIZE as the card will be the same: 138 Goto(:Call1) 139 } 140 141 Label(:Check1) 142 If(obj1, 0).EQ.Unlikely { 143 Goto(:Done) 144 } 145 146 If(ShrI(Xor(mem, obj1).ref_uint).Imm(Constants::REGION_SIZE_BIT).ref_uint, 0).EQ.Likely { 147 Goto(:Done) 148 } 149 150 Label(:Call1) 151 phi := Phi(ref_addr_obj2, ref_addr_obj1).ref_uint 152 LiveOut(phi).DstReg(regmap[:arg0]).ref_uint 153 Intrinsic(:TAIL_CALL).AddImm(Constants::POST_INTER_REGION_BARRIER_SLOW).Terminator.void 154 155 Label(:Done) 156 ReturnVoid().void 157 } 158end 159 160scoped_macro(:push_dirty_card_to_buffer) do |cards, min_addr, mem| 161 mem_word := Cast(mem).SrcType(Constants::REF_UINT).word 162 card_offset := ShrI(Sub(mem_word, min_addr).word).Imm(Constants::CARD_TABLE_CARD_BITS).word 163 card := Add(cards, card_offset).ptr 164 card_value := LoadI(card).Imm(Constants::CARD_VALUE_OFFSET).u8 165 166 is_card_clear := Compare(card_value, Constants::CARD_CLEAR_VALUE).EQ.b 167 IfImm(is_card_clear).Imm(0).EQ.Unlikely.b { 168 Goto(:Done) 169 } 170 StoreI(card, Constants::CARD_MARKED_VALUE).Imm(Constants::CARD_VALUE_OFFSET).u8 171 172 buffer := LoadI(%tr).Imm(Constants::MANAGED_THREAD_G1_POST_BARRIER_BUFFER_OFFSET).ptr 173 buffer_data := AddI(buffer).Imm(Constants::G1_LOCK_BUFFER_DATA_OFFSET).ptr 174 tail_index := LoadI(buffer).Imm(Constants::G1_LOCK_BUFFER_TAIL_OFFSET).word 175 tail_offset := ShlI(tail_index).Imm(Constants::POINTER_LOG_SIZE).word 176 next_tail_index := AndI(AddI(tail_index).Imm(1).word).Imm(Constants::G1_LOCK_BUFFER_SIZE_MASK).word 177 178 Label(:TryPushLoop) 179 head_index := LoadI(buffer).Imm(Constants::G1_LOCK_BUFFER_HEAD_OFFSET).Volatile.word 180 If(next_tail_index, head_index).EQ { 181 Goto(:TryPushLoop) 182 } 183 184 Store(buffer_data, tail_offset, card).ptr 185 StoreI(buffer, next_tail_index).Imm(Constants::G1_LOCK_BUFFER_TAIL_OFFSET).Volatile.word 186 187 Label(:Done) 188end 189 190def PostInterRegionBarrierSlow() 191 if Options.arch == :arm64 192 mask = RegMask.new($full_regmap, :arg0, :callee0, :callee1, :callee2, :tmp0, :tmp1) 193 elsif Options.arch == :x86_64 194 mask = RegMask.new($full_regmap, :arg0, :callee0, :caller0, :caller3, :tmp0, :tmp1) 195 else 196 mask = $panda_mask 197 end 198 199 function(:PostInterRegionBarrierSlow, 200 params: {mem: 'ref_uint'}, 201 regmap: $full_regmap, 202 regalloc_set: mask, 203 mode: [:FastPath]) { 204 if Options.arch == :arm32 205 Intrinsic(:UNREACHABLE).Terminator.void 206 next 207 end 208 209 min_addr := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_MIN_ADDR_OFFSET).word 210 cards := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_ADDR_OFFSET).ptr 211 push_dirty_card_to_buffer(cards, min_addr, mem) 212 ReturnVoid().void 213 } 214end 215 216def PostInterRegionBarrierTwoCardsSlow() 217 if Options.arch == :arm64 218 mask = RegMask.new($full_regmap, :arg0, :callee0, :callee1, :callee2, :tmp0, :tmp1) 219 elsif Options.arch == :x86_64 220 mask = RegMask.new($full_regmap, :arg0, :callee0, :caller0, :caller3, :tmp0, :tmp1) 221 else 222 mask = $panda_mask 223 end 224 225 function(:PostInterRegionBarrierTwoCardsSlow, 226 params: {mem: 'ref_uint'}, 227 regmap: $full_regmap, 228 regalloc_set: mask, 229 mode: [:FastPath]) { 230 if Options.arch == :arm32 231 Intrinsic(:UNREACHABLE).Terminator.void 232 next 233 end 234 235 min_addr := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_MIN_ADDR_OFFSET).word 236 cards := LoadI(%tr).Imm(Constants::TLS_CARD_TABLE_ADDR_OFFSET).ptr 237 # `mem` here is a pointer to the 2nd slot of the pair 238 push_dirty_card_to_buffer(cards, min_addr, mem) 239 push_dirty_card_to_buffer(cards, min_addr, SubI(mem).Imm(Constants::REFERENCE_TYPE_SIZE).ref_uint) 240 ReturnVoid().void 241 } 242end 243 244PostInterGenerationalBarrier(0) 245PostInterGenerationalBarrier(1) 246PostInterGenerationalBarrier(2) 247 248PostInterRegionBarrierMarkSingleFast() 249PostInterRegionBarrierMarkPairFast() 250 251PostInterRegionBarrierSlow() 252PostInterRegionBarrierTwoCardsSlow() 253