• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env ruby
2
3# Copyright (c) 2021-2024 Huawei Device Co., Ltd.
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16include_relative 'common.irt'
17
18$available_regs = $temps_mask + :arg0 + :arg1
19# there are 2 temp registers used in the case of arm64 (instead of 3 for
20# x86_64 and arm32) that's why one callee-saved register is added to the mask
21if Options.arch == :arm64
22  $available_regs = $available_regs + :callee1
23end
24
25AllocateObjectTlabValidation = {
26  spills_count_max: { default: 2, arm32: 9999 },  # TODO(msherstennikov): set to 0 once regalloc supports smart temps
27  code_size_max: { arm64: 116, x86_64: 477, arm32: 9999 }
28  # TODO(msherstennikov): revert back code size values, once regalloc supports smart temps
29  # code_size_max: { arm64: 100, x86_64: 125, arm32: 9999 }
30}
31
32AllocateArrayTlabValidation = {
33  spills_count_max: { default: 2, arm32: 9999 },  # TODO(msherstennikov): set to 0 once regalloc supports smart temps
34  code_size_max: { arm64: 136, x86_64: 476, arm32: 9999 }
35  # TODO(msherstennikov): revert back code size values, once regalloc supports smart temps
36  # code_size_max: { arm64: 128, x86_64: 145, arm32: 9999 }
37}
38
39# Increase count of available registers for x86_64
40if Options.arch == :x86_64
41  $available_regs = $panda_mask
42end
43
44function(:AllocateObjectTlab,
45         params: {klass: 'ref', size: 'word'},
46         regmap: $full_regmap,
47         mode: [:FastPath],
48         regalloc_set: $available_regs,
49         validate: AllocateObjectTlabValidation) {
50
51  if Options.arch == :arm32
52    Intrinsic(:UNREACHABLE).Terminator.void
53    ReturnVoid().void
54    next
55  end
56
57  # Load pointer to the TLAB from TLS
58  tlab_ptr := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr
59
60  # Load pointer to the start address of free memory in the TLAB
61  start := LoadI(tlab_ptr).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
62
63  # Load pointer to the end address of free memory in the TLAB
64  tls_end := LoadI(tlab_ptr).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr
65
66  tls_size := Sub(tls_end, start).word
67
68  If(tls_size, size).LT.Unlikely.b {
69    ep_offset = get_entrypoint_offset("CREATE_OBJECT_BY_CLASS_SLOW_PATH")
70    Intrinsic(:SLOW_PATH_ENTRY, klass, size).AddImm(ep_offset).MethodAsImm("CreateObjectByClassBridge").Terminator.ptr
71    Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG
72  } Else {
73    Intrinsic(:WRITE_TLAB_STATS_SAFE, start, size, Cast(-1).u64).void if defines.DEBUG
74    if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__
75      call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, start, size).void
76    end
77    new_start := Add(start, size).ptr
78    store_class(start, klass)
79    addr := Add(tlab_ptr, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
80    StoreI(addr, new_start).Imm(0).Volatile.ptr
81    Return(start).ptr
82  }
83}
84
85# Allow three temps, two arguments registers and one callee saved register
86$available_regs = $available_regs + :callee0
87
88def AllocateArrayTlab(element_size)
89  function("AllocateArrayTlab#{element_size}".to_sym,
90           params: {klass: 'ref', elements_num: 'word'},
91           regmap: $full_regmap,
92           mode: [:FastPath],
93           regalloc_set: $available_regs,
94           validate: AllocateArrayTlabValidation) {
95    if Options.arch == :arm32
96      ReturnVoid().void
97      next
98    end
99    elements_num := And(elements_num, "0x00000000ffffffff").word
100    if element_size == 8
101      size := elements_num
102    elsif element_size == 16
103      size := Shl(elements_num, 1).word
104    elsif element_size == 32
105      size := Shl(elements_num, 2).word
106    elsif element_size == 64
107      size := Shl(elements_num, 3).word
108    else
109      raise "Wrong element size #{element_size}"
110    end
111    # Add sizeof(Array) and do align
112    size := Add(size, "DEFAULT_ALIGNMENT_IN_BYTES - 1 + CORETYPES_ARRAY_CLASS_SIZE").word
113    size := And(size, "(~(DEFAULT_ALIGNMENT_IN_BYTES - 1))").word
114
115    # Load pointer to the TLAB from TLS
116    tlab_ptr := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr
117
118    # Load pointer to the start address of free memory in the TLAB
119    start := LoadI(tlab_ptr).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
120
121    # Load pointer to the end address of free memory in the TLAB
122    tls_end := LoadI(tlab_ptr).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr
123
124    tls_size := Sub(tls_end, start).word
125
126    If(tls_size, size).LT.Unlikely.b {
127      ep_offset = get_entrypoint_offset("CREATE_ARRAY_SLOW_PATH")
128      Intrinsic(:SLOW_PATH_ENTRY, klass, elements_num).AddImm(ep_offset).MethodAsImm("CreateArrayBridge").Terminator.ptr
129      Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG
130    } Else {
131      Intrinsic(:WRITE_TLAB_STATS_SAFE, start, size, Cast(-1).u64).void if defines.DEBUG
132      if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__
133        call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, start, size).void
134      end
135      new_start := Add(start, size).ptr
136      store_class(start, klass)
137      StoreI(start, elements_num).Imm(Constants::ARRAY_LENGTH_OFFSET).word
138      addr := Add(tlab_ptr, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
139      StoreI(addr, new_start).Imm(0).Volatile.ptr
140      Return(start).ptr
141    }
142  }
143end
144
145AllocateArrayTlab(8)
146AllocateArrayTlab(16)
147AllocateArrayTlab(32)
148AllocateArrayTlab(64)
149