1#!/usr/bin/python3 2# 3# Copyright (C) 2018 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18"""Generate intrinsics code.""" 19 20from collections import OrderedDict 21 22import asm_defs 23import json 24import os 25import re 26import sys 27 28# C-level intrinsic calling convention: 29# 1. All arguments are passed using the natural data types: 30# - int8_t passed as one byte argument (on the stack in IA32 mode, in GP register in x86-64 mode) 31# - int32_t passed as 4 bytes argument (on the stack in IA32 mode, in GP register in x86-64 mode) 32# - int64_t is passed as 8 byte argument (on the stack in IA32 mode, in GP register in x86-64 mode) 33# - float is passed as float (on the stack in IA32 mode, in XMM register in x86-64 mode) 34# - double is passed as double (on the stack in IA32 mode, in XMM register in x86-64 mode) 35# - vector formats are passed as pointers to 128bit data structure 36# 2. Return values. 37# - Values are returned as std::tuple. This means that on IA32 it's always returned on stack. 38 39INDENT = ' ' 40AUTOGEN = """\ 41// This file automatically generated by gen_intrinsics.py 42// DO NOT EDIT! 43""" 44 45 46class VecFormat(object): 47 48 def __init__(self, num_elements, element_size, is_unsigned, is_float, index, 49 c_type): 50 self.num_elements = num_elements 51 self.element_size = element_size 52 self.is_unsigned = is_unsigned 53 self.is_float = is_float 54 self.index = index 55 self.c_type = c_type 56 57 58# Vector format defined as: 59# vector_size, element_size, is_unsigned, is_float, index, ir_format, c_type 60# TODO(olonho): make flat numbering after removing legacy macro compat. 61_VECTOR_FORMATS = { 62 'U8x8': VecFormat(8, 1, True, False, 1, 'uint8_t'), 63 'U16x4': VecFormat(4, 2, True, False, 2, 'uint16_t'), 64 'U32x2': VecFormat(2, 4, True, False, 3, 'uint32_t'), 65 'U64x1': VecFormat(1, 8, True, False, 4, 'uint64_t'), 66 'U8x16': VecFormat(16, 1, True, False, 5, 'uint8_t'), 67 'U16x8': VecFormat(8, 2, True, False, 6, 'uint16_t'), 68 'U32x4': VecFormat(4, 4, True, False, 7, 'uint32_t'), 69 'U64x2': VecFormat(2, 8, True, False, 8, 'uint64_t'), 70 'I8x8': VecFormat(8, 1, False, False, 9, 'int8_t'), 71 'I16x4': VecFormat(4, 2, False, False, 10, 'int16_t'), 72 'I32x2': VecFormat(2, 4, False, False, 11, 'int32_t'), 73 'I64x1': VecFormat(1, 8, False, False, 12, 'int64_t'), 74 'I8x16': VecFormat(16, 1, False, False, 13, 'int8_t'), 75 'I16x8': VecFormat(8, 2, False, False, 14, 'int16_t'), 76 'I32x4': VecFormat(4, 4, False, False, 15, 'int32_t'), 77 'I64x2': VecFormat(2, 8, False, False, 16, 'int64_t'), 78 'U8x1': VecFormat(1, 1, True, False, 17, 'uint8_t'), 79 'I8x1': VecFormat(1, 1, False, False, 18, 'int8_t'), 80 'U16x1': VecFormat(1, 2, True, False, 19, 'uint16_t'), 81 'I16x1': VecFormat(1, 2, False, False, 20, 'int16_t'), 82 'U32x1': VecFormat(1, 4, True, False, 21, 'uint32_t'), 83 'I32x1': VecFormat(1, 4, False, False, 22, 'int32_t'), 84 # These vector formats can never intersect with above, so can reuse index. 85 'F32x1': VecFormat(1, 4, False, True, 1, 'Float32'), 86 'F32x2': VecFormat(2, 4, False, True, 2, 'Float32'), 87 'F32x4': VecFormat(4, 4, False, True, 3, 'Float32'), 88 'F64x1': VecFormat(1, 8, False, True, 4, 'Float64'), 89 'F64x2': VecFormat(2, 8, False, True, 5, 'Float64'), 90 # Those vector formats can never intersect with above, so can reuse index. 91 'U8x4': VecFormat(4, 1, True, False, 1, 'uint8_t'), 92 'U16x2': VecFormat(2, 2, True, False, 2, 'uint16_t'), 93 'I8x4': VecFormat(4, 1, False, False, 3, 'int8_t'), 94 'I16x2': VecFormat(2, 2, False, False, 4, 'int16_t'), 95} 96 97 98class VecSize(object): 99 100 def __init__(self, num_elements, index): 101 self.num_elements = num_elements 102 self.index = index 103 104 105_VECTOR_SIZES = {'X64': VecSize(64, 1), 'X128': VecSize(128, 2)} 106 107_ROUNDING_MODES = ['FE_TONEAREST', 'FE_DOWNWARD', 'FE_UPWARD', 'FE_TOWARDZERO', 'FE_TIESAWAY'] 108 109 110def _is_imm_type(arg_type): 111 return 'imm' in arg_type 112 113 114def _is_template_type(arg_type): 115 if not arg_type.startswith('Type'): 116 return False 117 assert isinstance(int(arg_type[4:]), int) 118 return True 119 120 121def _get_imm_c_type(arg_type): 122 return { 123 'imm8' : 'int8_t', 124 'uimm8' : 'uint8_t', 125 'uimm16' : 'uint16_t', 126 'uimm32' : 'uint32_t', 127 }[arg_type] 128 129 130def _get_c_type(arg_type): 131 if (arg_type in ('Float16', 'Float32', 'Float64', 'int8_t', 'uint8_t', 'int16_t', 132 'uint16_t', 'int32_t', 'uint32_t', 'int64_t', 'uint64_t', 133 'volatile uint8_t*', 'volatile uint32_t*') or 134 _is_template_type(arg_type)): 135 return arg_type 136 if arg_type in ('fp_flags', 'fp_control', 'int', 'flag', 'flags', 'vec32'): 137 return 'uint32_t' 138 if _is_imm_type(arg_type): 139 return _get_imm_c_type(arg_type) 140 if arg_type == 'vec': 141 return 'SIMD128Register' 142 if arg_type in _ROUNDING_MODES: 143 return 'int' 144 raise Exception('Type %s not supported' % (arg_type)) 145 146 147def _get_semantic_player_type(arg_type, type_map): 148 if type_map is not None and type_map != False and arg_type in type_map: 149 return type_map[arg_type] 150 if arg_type in ('Float16', 'Float32', 'Float64', 'vec'): 151 return 'SimdRegister' 152 if _is_imm_type(arg_type): 153 return _get_imm_c_type(arg_type) 154 return 'Register' 155 156 157def _gen_scalar_intr_decl(f, name, intr): 158 ins = intr.get('in') 159 outs = intr.get('out') 160 params = [_get_c_type(op) for op in ins] 161 if len(outs) > 0: 162 retval = 'std::tuple<' + ', '.join(_get_c_type(out) for out in outs) + '>' 163 else: 164 retval = 'void' 165 comment = intr.get('comment') 166 if comment: 167 print('// %s.' % (comment), file=f) 168 if intr.get('precise_nans', False): 169 print('template <bool precise_nan_operations_handling, ' 170 'enum PreferredIntrinsicsImplementation = kUseAssemblerImplementationIfPossible>', 171 file=f) 172 print('%s %s(%s);' % (retval, name, ', '.join(params)), file=f) 173 174 175def _gen_template_intr_decl(f, name, intr): 176 ins = intr.get('in') 177 outs = intr.get('out') 178 params = [_get_c_type(op) for op in ins] 179 if len(outs) > 0: 180 retval = 'std::tuple<' + ', '.join(_get_c_type(out) for out in outs) + '>' 181 else: 182 retval = 'void' 183 comment = intr.get('comment') 184 if comment: 185 print('// %s.' % (comment), file=f) 186 print('template <%s>' % _get_template_parameters( 187 intr.get('variants'), intr.get('precise_nans', False)), file=f) 188 print('%s %s(%s);' % (retval, name, ', '.join(params)), file=f) 189 190 191def _get_template_parameters( 192 variants, 193 precise_nans=False, 194 use_type_id=False, 195 extra=['enum PreferredIntrinsicsImplementation = kUseAssemblerImplementationIfPossible']): 196 if use_type_id: 197 typename = 'intrinsics::TemplateTypeId' 198 else: 199 typename = 'typename' 200 template = None 201 for variant in variants: 202 counter = -1 203 def get_counter(): 204 nonlocal counter 205 counter += 1 206 return counter 207 new_template = ', '.join( 208 (["bool kPreciseNaNOperationsHandling"] if precise_nans else []) + 209 ['bool kBool%s' % get_counter() if param.strip() in ('true', 'false') else 210 'int kInt%s' % get_counter() if param.strip() in _ROUNDING_MODES else 211 '%s Type%d' % (typename, get_counter()) if re.search('[_a-zA-Z]', param) else 212 'int kInt%s' % get_counter() 213 for param in variant.split(',')] + extra) 214 assert template is None or template == new_template 215 template = new_template 216 return template 217 218 219def _gen_vector_intr_decl(f, name, intr): 220 ins = intr.get('in') 221 outs = intr.get('out') 222 params = [_get_c_type(op) for op in ins] 223 if len(outs) > 0: 224 retval = 'std::tuple<' + ', '.join(_get_c_type(out) for out in outs) + '>' 225 else: 226 retval = 'void' 227 comment = intr.get('comment') 228 if comment: 229 print('// %s.' % (comment), file=f) 230 if intr.get('precise_nans', False): 231 template_parameters = 'bool precise_nan_operations_handling, ' 232 else: 233 template_parameters = '' 234 if not 'raw' in intr['variants']: 235 template_parameters += 'typename Type, ' 236 template_parameters += 'int size, ' 237 template_parameters += 'enum PreferredIntrinsicsImplementation' 238 template_parameters += ' = kUseAssemblerImplementationIfPossible' 239 print('template <%s>' % template_parameters, file=f) 240 print('%s %s(%s);' % (retval, name, ', '.join(params)), file=f) 241 242 243def _is_vector_class(intr): 244 return intr.get('class') in ('vector_4', 'vector_8', 'vector_16', 245 'vector_8/16', 'vector_8/16/single', 246 'vector_8/single', 'vector_16/single') 247 248 249def _is_simd128_conversion_required(t, type_map=None): 250 return (_get_semantic_player_type(t, type_map) == 'SimdRegister' and 251 _get_c_type(t) != 'SIMD128Register') 252 253 254def _get_semantics_player_hook_result(intr): 255 outs = intr['out'] 256 if len(outs) == 0: 257 return 'void' 258 elif len(outs) == 1: 259 # No tuple for single result. 260 return _get_semantic_player_type(outs[0], intr.get('sem-player-types')) 261 return 'std::tuple<' + ', '.join( 262 _get_semantic_player_type(out, intr.get('sem-player-types')) 263 for out in outs) + '>' 264 265 266def _get_semantics_player_hook_proto_components(name, intr): 267 ins = intr['in'] 268 269 args = [] 270 if _is_vector_class(intr): 271 if 'raw' in intr['variants']: 272 assert len(intr['variants']) == 1, "Unexpected length of variants" 273 args = ["uint8_t size"] 274 else: 275 args = ["uint8_t elem_size", "uint8_t elem_num"] 276 if (_is_signed(intr) and _is_unsigned(intr)): 277 args += ['bool is_signed'] 278 279 args += [ 280 '%s arg%d' % ( 281 _get_semantic_player_type(op, intr.get('sem-player-types')), num) 282 for num, op in enumerate(ins) 283 ] 284 285 result = _get_semantics_player_hook_result(intr) 286 287 return result, name, ', '.join(args) 288 289 290def _get_semantics_player_hook_proto(name, intr, use_type_id=False): 291 result, name, params = _get_semantics_player_hook_proto_components(name, intr) 292 if intr.get('class') == 'template': 293 template_parameters = _get_template_parameters( 294 intr.get('variants'), use_type_id=use_type_id, extra = []) 295 values = '' 296 if use_type_id: 297 spec_arguments = _get_template_spec_arguments(intr.get('variants')) 298 values = ', ' + ', '.join( 299 ['intrinsics::Value<%s>' % argument for argument in spec_arguments]) 300 return 'template<%s>\n%s %s(%s%s)' % ( 301 template_parameters, result, name, params, values) 302 return '%s %s(%s)' % (result, name, params) 303 304 305def _get_interpreter_hook_call_expr(name, intr, desc=None, use_type_id=False): 306 ins = intr['in'] 307 outs = intr['out'] 308 309 call_params = [] 310 for num, op in enumerate(ins): 311 arg = 'arg%d' % (num) 312 semantic_player_type = _get_semantic_player_type( 313 op, intr.get('sem-player-types')) 314 if semantic_player_type == 'FpRegister': 315 if op.startswith('Type') and use_type_id: 316 op = 'intrinsics::TypeFromId<%s>' % op 317 call_params.append('FPRegToFloat<%s>(%s)' % (op, arg)) 318 elif semantic_player_type == 'SimdRegister': 319 call_params.append(_get_cast_from_simd128(arg, op, ptr_bits=64)) 320 elif '*' in _get_c_type(op): 321 call_params.append('berberis::bit_cast<%s>(%s)' % (_get_c_type(op), arg)) 322 else: 323 c_type = _get_c_type(op) 324 if c_type.startswith('Type') and use_type_id: 325 c_type = 'intrinsics::TypeFromId<%s>' % c_type 326 call_params.append('GPRRegToInteger<%s>(%s)' % (c_type, arg)) 327 328 call_expr = 'intrinsics::%s%s(%s)' % ( 329 name, _get_desc_specializations(intr, desc, use_type_id), ', '.join(call_params)) 330 331 if len(outs) == 1: 332 # Unwrap tuple for single result. 333 call_expr = 'std::get<0>(%s)' % call_expr 334 if 'sem-player-types' in intr: 335 out_type = _get_semantic_player_type(outs[0], intr.get('sem-player-types')) 336 if out_type == "FpRegister": 337 call_expr = 'FloatToFPReg(%s)' % call_expr 338 elif out_type != "SimdRegister": 339 assert out_type == "Register" 340 assert not _is_simd128_conversion_required( 341 outs[0], intr.get('sem-player-types')) 342 call_expr = 'IntegerToGPRReg(%s)' % call_expr 343 else: 344 # Currently this kind of mismatch can only happen for single result, so we 345 # can keep simple code here for now. 346 if _is_simd128_conversion_required(outs[0]): 347 out_type = _get_c_type(outs[0]) 348 if out_type in ('Float16', 'Float32', 'Float64'): 349 call_expr = 'FloatToFPReg(%s)' % call_expr 350 else: 351 raise Exception('Type %s is not supported' % (out_type)) 352 else: 353 if any(_is_simd128_conversion_required(out) for out in outs): 354 raise Exception( 355 'Unsupported SIMD128Register conversion with multiple results') 356 357 return call_expr 358 359 360def _get_interpreter_hook_return_stmt(name, intr, desc=None, use_type_id=False): 361 return 'return ' + _get_interpreter_hook_call_expr(name, intr, desc, use_type_id) + ';' 362 363def _get_unused(intr): 364 call_expr = 'UNUSED(%s);' % ', '.join('arg%d' % (num) for num, _ in enumerate(intr['in'])) 365 return call_expr 366 367def _get_placeholder_return_stmt(intr, f): 368 print(INDENT + _get_unused(intr), file=f) 369 outs = intr['out'] 370 if outs: 371 print(INDENT + 'return {};', file=f) 372 373def _get_semantics_player_hook_raw_vector_body(name, intr, get_return_stmt): 374 outs = intr['out'] 375 if (len(outs) == 0): 376 raise Exception('No result raw vector intrinsic is not supported') 377 reg_class = intr.get('class') 378 yield 'switch (size) {' 379 for fmt, desc in _VECTOR_SIZES.items(): 380 if _check_reg_class_size(reg_class, desc.num_elements / 8): 381 yield INDENT + 'case %s:' % desc.num_elements 382 yield 2 * INDENT + get_return_stmt(name, intr, desc) 383 yield INDENT + 'default:' 384 yield 2 * INDENT + 'LOG_ALWAYS_FATAL("Unsupported size");' 385 yield '}' 386 387 388def _is_signed(intr): 389 return any(v.startswith("signed") for v in intr['variants']) 390 391 392def _is_unsigned(intr): 393 return any(v.startswith("unsigned") for v in intr['variants']) 394 395 396def _get_vector_format_init_expr(intr): 397 variants = intr.get('variants') 398 399 if ('Float16' in variants or 'Float32' in variants or 'Float64' in variants): 400 return 'intrinsics::GetVectorFormatFP(elem_size, elem_num)' 401 402 assert _is_signed(intr) or _is_unsigned(intr), "Unexpected intrinsic class" 403 if _is_signed(intr) and _is_unsigned(intr): 404 signed_arg = ', is_signed' 405 else: 406 signed_arg = ', true' if _is_signed(intr) else ', false' 407 return 'intrinsics::GetVectorFormatInt(elem_size, elem_num%s)' % signed_arg 408 409 410def _get_semantics_player_hook_vector_body(name, intr, get_return_stmt): 411 outs = intr['out'] 412 if (len(outs) == 0): 413 raise Exception('No result vector intrinsic is not supported') 414 reg_class = intr.get('class') 415 yield 'auto format = %s;' % _get_vector_format_init_expr(intr) 416 yield 'switch (format) {' 417 for variant in intr.get('variants'): 418 for fmt, desc in _VECTOR_FORMATS.items(): 419 if (_check_reg_class_size(reg_class, 420 desc.element_size * desc.num_elements) and 421 _check_typed_variant(variant, desc)): 422 yield INDENT + 'case intrinsics::kVector%s:' % fmt 423 yield 2 * INDENT + get_return_stmt(name, intr, desc) 424 elif (reg_class in ('vector_8/single', 'vector_8/16/single', 'vector_16/single') and 425 desc.num_elements == 1 and 426 _check_typed_variant(variant, desc)): 427 assert desc.element_size <= 8, "Unexpected element size" 428 yield INDENT + 'case intrinsics::kVector%s:' % fmt 429 yield 2 * INDENT + get_return_stmt(name, intr, desc) 430 yield INDENT + 'default:' 431 yield 2 * INDENT + 'LOG_ALWAYS_FATAL("Unsupported format");' 432 yield '}' 433 434 435# Syntax sugar heavily used in tests. 436def _get_interpreter_hook_vector_body(name, intr): 437 return _get_semantics_player_hook_vector_body( 438 name, intr, _get_interpreter_hook_return_stmt) 439 440 441def _gen_interpreter_hook(f, name, intr, option, use_type_id=False): 442 print('%s const {' % (_get_semantics_player_hook_proto(name, intr, use_type_id)), file=f) 443 444 if _is_vector_class(intr): 445 if 'raw' in intr['variants']: 446 assert len(intr['variants']) == 1, "Unexpected length of variants" 447 lines = _get_semantics_player_hook_raw_vector_body( 448 name, 449 intr, 450 _get_interpreter_hook_return_stmt) 451 else: 452 lines = _get_interpreter_hook_vector_body(name, intr) 453 454 lines = [INDENT + l for l in lines] 455 print('\n'.join(lines), file=f) 456 else: 457 if intr.get('class') == 'template': 458 _gen_template_parameters_verifier(f, intr, use_type_id) 459 # TODO(b/363057506): Add float support and clean up the logic here. 460 arm64_allowlist = ['AmoAdd', 'AmoAnd', 'AmoMax', 'AmoMin', 'AmoOr', 'AmoSwap', 'AmoXor', 'Bclr', 461 'Bclri', 'Bext', 'Bexti', 'Binv', 'Binvi', 'Bset', 'Bseti', 'Div', 'Max', 462 'Min', 'Rem', 'Rev8', 'Rol', 'Ror', 'Sext', 'Sh1add', 'Sh1adduw', 'Sh2add', 463 'Sh2adduw', 'Sh3add', 'Sh3adduw', 'Zext', 'UnboxNan'] 464 if (option == 'arm64') and (name not in arm64_allowlist): 465 _get_placeholder_return_stmt(intr, f) 466 else: 467 print(INDENT + _get_interpreter_hook_return_stmt(name, intr, use_type_id=use_type_id), file=f) 468 469 print('}\n', file=f) 470 471 472def _get_translator_hook_call_expr(name, intr, desc=None, use_type_id=False): 473 desc_spec = _get_desc_specializations(intr, desc, use_type_id) 474 args = [('arg%d' % n) for n, _ in enumerate(intr['in'])] 475 template_params = ['&intrinsics::' + name + desc_spec] 476 template_params += [_get_semantics_player_hook_result(intr)] 477 return 'CallIntrinsic<%s>(%s)' % (', '.join(template_params), ', '.join(args)) 478 479 480def _get_translator_hook_return_stmt(name, intr, desc=None, use_type_id=False): 481 return 'return ' + _get_translator_hook_call_expr(name, intr, desc, use_type_id) + ';' 482 483 484def _gen_translator_hook(f, name, intr, use_type_id=False): 485 print('%s {' % (_get_semantics_player_hook_proto(name, intr, use_type_id)), file=f) 486 487 if _is_vector_class(intr): 488 if 'raw' in intr['variants']: 489 assert len(intr['variants']) == 1, "Unexpected length of variants" 490 lines = _get_semantics_player_hook_raw_vector_body( 491 name, 492 intr, 493 _get_translator_hook_return_stmt) 494 else: 495 lines = _get_semantics_player_hook_vector_body( 496 name, 497 intr, 498 _get_translator_hook_return_stmt) 499 lines = [INDENT + l for l in lines] 500 print('\n'.join(lines), file=f) 501 else: 502 if intr.get('class') == 'template': 503 _gen_template_parameters_verifier(f, intr, use_type_id) 504 print(INDENT + _get_translator_hook_return_stmt(name, intr, use_type_id=use_type_id), file=f) 505 506 print('}\n', file=f) 507 508 509def _gen_mock_semantics_listener_hook(f, name, intr): 510 result, name, params = _get_semantics_player_hook_proto_components(name, intr) 511 if intr.get('class') == 'template': 512 spec_arguments = _get_template_spec_arguments(intr.get('variants')) 513 for use_type_id in [True, False]: 514 template_parameters = _get_template_parameters( 515 intr.get('variants'), use_type_id=use_type_id, extra = []) 516 args = ', '.join( 517 [('arg%d' % n) for n, _ in enumerate(intr['in'])] + 518 [arg 519 if use_type_id or not arg.startswith('Type') else 520 'intrinsics::kIdFromType<%s>' % arg 521 for arg in spec_arguments]) 522 values = '' 523 if use_type_id: 524 values = ', ' + ', '.join( 525 ['intrinsics::Value<%s>' % argument for argument in spec_arguments]) 526 print('template<%s>\n%s %s(%s%s) {\n return %s(%s);\n}' % ( 527 template_parameters, result, name, params, values, name, args), file=f) 528 params = ', '.join( 529 [params] + 530 ['%s %s' % ( 531 { 532 'kBoo': 'bool', 533 'kInt': 'int', 534 'Type': 'intrinsics::TemplateTypeId' 535 }[argument[0:4]], 536 argument) 537 for argument in spec_arguments]) 538 print('MOCK_METHOD((%s), %s, (%s));' % (result, name, params), file=f) 539 540 541def _gen_template_parameters_verifier(f, intr, use_type_id=False): 542 received_params = ', '.join( 543 param 544 if not param.strip().startswith('Type') or use_type_id else 545 f'intrinsics::kIdFromType<{param}>' 546 for param in _get_template_spec_arguments(intr.get('variants'))) 547 print('%sstatic_assert(%s);' % ( 548 INDENT, 549 ' || '.join( 550 'std::tuple{%s} == std::tuple{%s}' % ( 551 received_params, 552 ', '.join( 553 param 554 if param.strip() in ['true', 'false'] + _ROUNDING_MODES or 555 not re.search('[_a-zA-Z]', param) else 556 f'intrinsics::kIdFromType<{param}>' 557 for param in variant.split(','))) 558 for variant in intr.get('variants'))), file=f) 559 560 561def _check_signed_variant(variant, desc): 562 if variant == 'signed': 563 return True 564 if variant == 'signed_32': 565 return desc.element_size == 4 566 if variant == 'signed_64': 567 return desc.element_size == 8 568 if variant == 'signed_16/32': 569 return desc.element_size in (2, 4) 570 if variant == 'signed_8/16/32': 571 return desc.element_size in (1, 2, 4) 572 if variant == 'signed_16/32/64': 573 return desc.element_size in (2, 4, 8) 574 if variant == 'signed_8/16/32/64': 575 return desc.element_size in (1, 2, 4, 8) 576 if variant == 'signed_32/64': 577 return desc.element_size in (4, 8) 578 return False 579 580 581def _check_unsigned_variant(variant, desc): 582 if variant == 'unsigned': 583 return True 584 if variant == 'unsigned_8': 585 return desc.element_size == 1 586 if variant == 'unsigned_16': 587 return desc.element_size == 2 588 if variant == 'unsigned_32': 589 return desc.element_size == 4 590 if variant == 'unsigned_64': 591 return desc.element_size == 8 592 if variant == 'unsigned_8/16': 593 return desc.element_size in (1, 2) 594 if variant == 'unsigned_8/16/32': 595 return desc.element_size in (1, 2, 4) 596 if variant == 'unsigned_16/32/64': 597 return desc.element_size in (2, 4, 8) 598 if variant == 'unsigned_8/16/32/64': 599 return desc.element_size in (1, 2, 4, 8) 600 if variant == 'unsigned_32/64': 601 return desc.element_size in (4, 8) 602 return False 603 604 605def _check_reg_class_size(reg_class, size): 606 # Small vectors are separate namespace. 607 if size == 4 and reg_class == 'vector_4': 608 return True 609 if size == 8 and reg_class in ('vector_8', 'vector_8/16', 'vector_8/16/single', 610 'vector_8/single'): 611 return True 612 if size == 16 and reg_class in ('vector_16', 'vector_8/16', 'vector_8/16/single', 613 'vector_16/single'): 614 return True 615 return False 616 617 618def _check_typed_variant(variant, desc): 619 if desc.is_unsigned and not desc.is_float: 620 return _check_unsigned_variant(variant, desc) 621 if not desc.is_unsigned and not desc.is_float: 622 return _check_signed_variant(variant, desc) 623 if desc.is_float: 624 if desc.element_size == 2: 625 return variant == 'Float16' 626 if desc.element_size == 4: 627 return variant == 'Float32' 628 if desc.element_size == 8: 629 return variant == 'Float64' 630 return False 631 632 633def _get_formats_with_descriptions(intr): 634 reg_class = intr.get('class') 635 for variant in intr.get('variants'): 636 found_fmt = False 637 for fmt, desc in _VECTOR_FORMATS.items(): 638 if (_check_reg_class_size(reg_class, 639 desc.element_size * desc.num_elements) and 640 _check_typed_variant(variant, desc) and 641 (reg_class != 'vector_4' or desc.element_size < 4)): 642 found_fmt = True 643 yield fmt, desc 644 645 if variant == 'raw': 646 for fmt, desc in _VECTOR_SIZES.items(): 647 if _check_reg_class_size(reg_class, desc.num_elements / 8): 648 found_fmt = True 649 yield fmt, desc 650 651 assert found_fmt, 'Couldn\'t expand %s' % reg_class 652 653 654def _get_result_type(outs): 655 result_type = 'void' 656 return_stmt = '' 657 if len(outs) >= 1: 658 result_type = ('std::tuple<' + 659 ', '.join(_get_c_type(out) for out in outs) + '>') 660 return_stmt = 'return ' 661 return result_type, return_stmt 662 663 664def _get_in_params(params): 665 for param_index, param in enumerate(params): 666 yield _get_c_type(param), 'in%d' % (param_index) 667 668 669def _get_out_params(params): 670 for param_index, param in enumerate(params): 671 yield _get_c_type(param), 'out%d' % (param_index) 672 673 674def _get_cast_from_simd128(var, target_type, ptr_bits): 675 if ('*' in target_type): 676 return 'berberis::bit_cast<%s>(%s.Get<uint%d_t>(0))' % (_get_c_type(target_type), var, 677 ptr_bits) 678 679 c_type = _get_c_type(target_type) 680 if c_type in ('Float16', 'Float32', 'Float64'): 681 return 'FPRegToFloat<%s>(%s)' % (c_type, var) 682 683 cast_map = { 684 'int8_t': '.Get<int8_t>(0)', 685 'uint8_t': '.Get<uint8_t>(0)', 686 'int16_t': '.Get<int16_t>(0)', 687 'uint16_t': '.Get<uint16_t>(0)', 688 'int32_t': '.Get<int32_t>(0)', 689 'uint32_t': '.Get<uint32_t>(0)', 690 'int64_t': '.Get<int64_t>(0)', 691 'uint64_t': '.Get<uint64_t>(0)', 692 'SIMD128Register': '' 693 } 694 return '%s%s' % (var, cast_map[c_type]) 695 696 697def _get_desc_specializations(intr, desc=None, use_type_id=False): 698 if intr.get('class') == 'template': 699 spec = _get_template_spec_arguments(intr.get('variants'), use_type_id) 700 elif hasattr(desc, 'c_type'): 701 spec = [desc.c_type, str(desc.num_elements)] 702 elif hasattr(desc, 'num_elements'): 703 spec = [str(desc.num_elements)] 704 else: 705 spec = [] 706 if intr.get('precise_nans', False): 707 spec = ['config::kPreciseNaNOperationsHandling'] + spec 708 if not len(spec): 709 return '' 710 return '<%s>' % ', '.join(spec) 711 712 713def _get_template_spec_arguments(variants, use_type_id=False): 714 spec = None 715 for variant in variants: 716 counter = -1 717 def get_counter(): 718 nonlocal counter 719 counter += 1 720 return counter 721 new_spec = [ 722 'kBool%s' % get_counter() if param.strip() in ('true', 'false') else 723 'kInt%s' % get_counter() if param.strip() in _ROUNDING_MODES else 724 ('intrinsics::TypeFromId<Type%d>' % get_counter() 725 if use_type_id else 726 'Type%d' % get_counter()) 727 if re.search('[_a-zA-Z]', param) else 728 'kInt%s' % get_counter() 729 for param in variant.split(',')] 730 assert spec is None or spec == new_spec 731 spec = new_spec 732 return spec 733 734 735def _intr_has_side_effects(intr, fmt=None): 736 ins = intr.get('in') 737 outs = intr.get('out') 738 # If we have 'has_side_effects' mark in JSON file then we use it "as is". 739 if 'has_side_effects' in intr: 740 return intr.get('has_side_effects') 741 # Otherwise we mark all floating-point related intrinsics as "volatile". 742 # TODO(b/68857496): move that information in HIR/LIR and stop doing that. 743 if 'Float16' in ins or 'Float32' in ins or 'Float64' in ins: 744 return True 745 if 'Float16' in outs or 'Float32' in outs or 'Float64' in outs: 746 return True 747 if fmt is not None and fmt.startswith('F'): 748 return True 749 return False 750 751 752def _gen_intrinsics_inl_h(f, intrs): 753 print(AUTOGEN, file=f) 754 for name, intr in intrs: 755 if intr.get('class') == 'scalar': 756 _gen_scalar_intr_decl(f, name, intr) 757 elif intr.get('class') == 'template': 758 _gen_template_intr_decl(f, name, intr) 759 else: 760 assert intr.get('class').startswith('vector') 761 _gen_vector_intr_decl(f, name, intr) 762 763 764def _gen_semantic_player_types(intrs): 765 for name, intr in intrs: 766 if intr.get('class') == 'template': 767 map = None 768 for variant in intr.get('variants'): 769 counter = -1 770 def get_counter(): 771 nonlocal counter 772 counter += 1 773 return counter 774 new_map = { 775 'Float16': 'FpRegister', 776 'Float32': 'FpRegister', 777 'Float64': 'FpRegister', 778 } 779 for type in filter( 780 lambda param: param.strip() not in ('true', 'false') and 781 re.search('[_a-zA-Z]', param), 782 variant.split(',')): 783 new_map['Type%d' % get_counter()] = ( 784 'FpRegister' if type.strip() in ('Float16', 'Float32', 'Float64') else 785 _get_semantic_player_type(type, None)) 786 if map is None: 787 map = new_map 788 elif map != new_map: 789 # Note: we would use literal `False` as type, which would lead to 790 # compile-time error… that's Ok, because mix of ints and floats may 791 # only happen with vector intrinsics where types used are 792 # never arguments, but just specify type of vector element. 793 # If intrinsics actually have to receive such arguments that such 794 # intrinsics should be split in two. 795 map = False 796 intr['sem-player-types'] = map 797 798 799def _gen_interpreter_intrinsics_hooks_impl_inl_h(f, intrs, option): 800 print(AUTOGEN, file=f) 801 for name, intr in intrs: 802 if intr.get('class') == 'template': 803 _gen_interpreter_hook( 804 f, name, intr, option, use_type_id=True) 805 _gen_interpreter_hook(f, name, intr, option) 806 807 808def _gen_translator_intrinsics_hooks_impl_inl_h(f, intrs): 809 print(AUTOGEN, file=f) 810 for name, intr in intrs: 811 if intr.get('class') == 'template': 812 _gen_translator_hook( 813 f, name, intr, use_type_id=True) 814 _gen_translator_hook(f, name, intr) 815 816 817def _gen_mock_semantics_listener_intrinsics_hooks_impl_inl_h(f, intrs): 818 print(AUTOGEN, file=f) 819 for name, intr in intrs: 820 _gen_mock_semantics_listener_hook(f, name, intr) 821 822 823def _get_reg_operand_info(arg, info_prefix=None): 824 need_tmp = arg['class'] in ('EAX', 'EDX', 'CL', 'ECX') 825 if info_prefix is None: 826 class_info = 'void' 827 else: 828 class_info = '%s::%s' % (info_prefix, arg['class']) 829 if arg['class'] == 'Imm8': 830 return 'ImmArg<%d, int8_t, %s>' % (arg['ir_arg'], class_info) 831 if info_prefix is None: 832 using_info = 'void' 833 else: 834 using_info = '%s::%s' % (info_prefix, { 835 'def': 'Def', 836 'def_early_clobber': 'DefEarlyClobber', 837 'use': 'Use', 838 'use_def': 'UseDef' 839 }[arg['usage']]) 840 if arg['usage'] == 'use': 841 if need_tmp: 842 return 'InTmpArg<%d, %s, %s>' % (arg['ir_arg'], class_info, using_info) 843 return 'InArg<%d, %s, %s>' % (arg['ir_arg'], class_info, using_info) 844 if arg['usage'] in ('def', 'def_early_clobber'): 845 assert 'ir_arg' not in arg 846 if 'ir_res' in arg: 847 if need_tmp: 848 return 'OutTmpArg<%d, %s, %s>' % (arg['ir_res'], class_info, using_info) 849 return 'OutArg<%d, %s, %s>' % (arg['ir_res'], class_info, using_info) 850 return 'TmpArg<%s, %s>' % (class_info, using_info) 851 if arg['usage'] == 'use_def': 852 if 'ir_res' in arg: 853 if need_tmp: 854 return 'InOutTmpArg<%s, %s, %s, %s>' % (arg['ir_arg'], arg['ir_res'], 855 class_info, using_info) 856 return 'InOutArg<%s, %s, %s, %s>' % (arg['ir_arg'], arg['ir_res'], 857 class_info, using_info) 858 return 'InTmpArg<%s, %s, %s>' % (arg['ir_arg'], class_info, using_info) 859 assert False, 'unknown operand usage %s' % (arg['usage']) 860 861 862def _gen_make_intrinsics(f, intrs, archs): 863 print("%s" % AUTOGEN, file=f) 864 callback_lines = [] 865 static_names = [] 866 static_mnemos = [] 867 for line in _gen_c_intrinsics_generator( 868 intrs, _is_interpreter_compatible_assembler, False, static_names, static_mnemos): 869 callback_lines.append(line) 870 print( 871""" 872/* Note: we generate binding names and binding mnemos used by callbacks in ProcessAllBindings 873globally so that ProcessAllBindings can be constexpr. 874 875Once we can use C++23, these can be declared locally in ProcessAllBindings.*/""", file=f) 876 print("namespace process_all_bindings_strings {", file = f) 877 for static_name in static_names: 878 print(" %s" % static_name, file=f) 879 for static_mnemo in static_mnemos: 880 print(" %s" % static_mnemo, file=f) 881 print("} // process_all_bindings_strings", file = f) 882 print(""" 883template <typename MacroAssembler, 884 typename Callback, 885 typename... Args> 886constexpr void ProcessAllBindings([[maybe_unused]] Callback callback, 887 [[maybe_unused]] Args&&... args) { 888 using intrinsics::Float16; 889 using intrinsics::Float32; 890 using intrinsics::Float64; 891 using namespace process_all_bindings_strings;""", 892 file=f) 893 for line in callback_lines: 894 print(line, file=f) 895 print('}', file=f) 896 897def _gen_opcode_generators_f(f, intrs): 898 for line in _gen_opcode_generators(intrs): 899 print(line, file=f) 900 901def _gen_opcode_generators(intrs): 902 opcode_generators = {} 903 for name, intr in intrs: 904 if 'asm' not in intr: 905 continue 906 if 'variants' in intr: 907 variants = _get_formats_with_descriptions(intr) 908 variants = sorted(variants, key=lambda variant: variant[1].index) 909 # Collect intr_asms for all variants of intrinsic. 910 # Note: not all variants are guaranteed to have an asm variant! 911 # If that happens the list of intr_asms for that variant will be empty. 912 variants = [[ 913 intr_asm for intr_asm in _gen_sorted_asms(intr) 914 if fmt in intr_asm['variants'] 915 ] for fmt, _ in variants] 916 # Print intrinsic generator 917 for intr_asms in variants: 918 if len(intr_asms) > 0: 919 for intr_asm in intr_asms: 920 if not _is_translator_compatible_assembler(intr_asm): 921 continue 922 for line in _gen_opcode_generator(intr_asm, opcode_generators): 923 yield line 924 else: 925 for intr_asm in _gen_sorted_asms(intr): 926 if not _is_translator_compatible_assembler(intr_asm): 927 continue 928 for line in _gen_opcode_generator(intr_asm, opcode_generators): 929 yield line 930 931def _gen_opcode_generator(asm, opcode_generators): 932 name = asm['name'] 933 num_mem_args = sum(1 for arg in asm['args'] if arg.get('class').startswith("Mem") and arg.get('usage') == 'def_early_clobber') 934 opcode = 'Undefined' if num_mem_args > 2 else (asm_defs.get_mem_macro_name(asm, '').replace("Mem", "MemBaseDisp")) if num_mem_args > 0 else name 935 936 if name not in opcode_generators: 937 opcode_generators[name] = True 938 yield """ 939// TODO(b/260725458): Pass lambda as template argument after C++20 becomes available. 940class GetOpcode%s { 941 public: 942 template <typename Opcode> 943 constexpr auto operator()() { 944 return Opcode::kMachineOp%s; 945 } 946};""" % (name, opcode) 947 948def _gen_process_bindings(f, intrs, archs): 949 print("%s" % AUTOGEN, file=f) 950 callback_lines = [] 951 static_names = [] 952 static_mnemos = [] 953 for line in _gen_c_intrinsics_generator( 954 intrs, _is_translator_compatible_assembler, True, static_names, static_mnemos): 955 callback_lines.append(line) 956 print( 957""" 958/* Note: we generate binding names and binding mnemos used by callbacks in ProcessBindings 959globally so that ProcessBindings can be constexpr. 960 961Once we can use C++23, these can be declared locally in ProcessBindings.*/""", file=f) 962 print("namespace process_bindings_strings {", file = f) 963 for static_name in static_names: 964 print(" %s" % static_name, file=f) 965 for static_mnemo in static_mnemos: 966 print(" %s" % static_mnemo, file=f) 967 print("} // process_bindings_strings", file = f) 968 _gen_opcode_generators_f(f, intrs) 969 970 print(""" 971template <auto kFunc, 972 typename MacroAssembler, 973 typename Result, 974 typename Callback, 975 typename... Args> 976constexpr Result ProcessBindings(Callback callback, Result def_result, Args&&... args) { 977 using namespace process_bindings_strings;""", 978 file=f) 979 for line in callback_lines: 980 print(line, file=f) 981 print(""" } 982 return std::forward<Result>(def_result); 983}""", file=f) 984 985 986def _gen_c_intrinsics_generator( 987 intrs, check_compatible_assembler, gen_builder, static_names, static_mnemos): 988 string_labels = {} 989 mnemo_idx = [0] 990 for name, intr in intrs: 991 ins = intr.get('in') 992 outs = intr.get('out') 993 params = _get_in_params(ins) 994 formal_args = ', '.join('%s %s' % (type, param) for type, param in params) 995 result_type, _ = _get_result_type(outs) 996 if 'asm' not in intr: 997 continue 998 if 'variants' in intr: 999 variants = _get_formats_with_descriptions(intr) 1000 # Sort by index, to keep order close to what _gen_intrs_enum produces. 1001 variants = sorted(variants, key=lambda variant: variant[1].index) 1002 # Collect intr_asms for all versions of intrinsic. 1003 # Note: not all variants are guaranteed to have asm version! 1004 # If that happens list of intr_asms for that variant would be empty. 1005 variants = [(desc, [ 1006 intr_asm for intr_asm in _gen_sorted_asms(intr) 1007 if fmt in intr_asm['variants'] 1008 ]) for fmt, desc in variants] 1009 # Print intrinsic generator 1010 for desc, intr_asms in variants: 1011 if len(intr_asms) > 0: 1012 if 'raw' in intr['variants']: 1013 spec = '%d' % (desc.num_elements) 1014 else: 1015 spec = '%s, %d' % (desc.c_type, desc.num_elements) 1016 for intr_asm in intr_asms: 1017 for line in _gen_c_intrinsic('%s<%s>' % (name, spec), 1018 intr, 1019 intr_asm, 1020 string_labels, 1021 mnemo_idx, 1022 check_compatible_assembler, 1023 gen_builder, 1024 static_names, 1025 static_mnemos): 1026 yield line 1027 else: 1028 for intr_asm in _gen_sorted_asms(intr): 1029 for line in _gen_c_intrinsic(name, 1030 intr, 1031 intr_asm, 1032 string_labels, 1033 mnemo_idx, 1034 check_compatible_assembler, 1035 gen_builder, 1036 static_names, 1037 static_mnemos): 1038 yield line 1039 1040 1041def _gen_sorted_asms(intr): 1042 return sorted(intr['asm'], 1043 key = lambda intr: 1044 intr.get('nan', '') + 1045 _KNOWN_FEATURES_KEYS.get( 1046 intr.get('feature', ''), intr.get('feature', '')), reverse = True) 1047 1048_KNOWN_FEATURES_KEYS = { 1049 'LZCNT': '001', 1050 'BMI': '002', 1051 'BMI2': '003', 1052 'SSE': '010', 1053 'SSE2': '011', 1054 'SSE3': '012', 1055 'SSSE3': '013', 1056 'SSE4a': '014', 1057 'SSE4_1': '015', 1058 'SSE4_2': '016', 1059 'AVX': '017', 1060 'AVX2': '018', 1061 'AES': '019', 1062 'AESAVX': '020', 1063 'VAES': '021', 1064 'CLMUL': '012', 1065 'CLMULAVX': '023', 1066 'VPCLMULQD': '024', 1067 'F16C': '025', 1068 'FMA': '026', 1069 'FMA4': '027', 1070 'CustomCapability': '999' 1071} 1072 1073 1074def _gen_c_intrinsic(name, 1075 intr, 1076 asm, 1077 string_labels, 1078 mnemo_idx, 1079 check_compatible_assembler, 1080 gen_builder, 1081 static_names, 1082 static_mnemos): 1083 if not check_compatible_assembler(asm): 1084 return 1085 1086 cpuid_restriction = 'intrinsics::bindings::NoCPUIDRestriction' 1087 if 'feature' in asm: 1088 if asm['feature'] == 'AuthenticAMD': 1089 cpuid_restriction = 'intrinsics::bindings::IsAuthenticAMD' 1090 else: 1091 cpuid_restriction = 'intrinsics::bindings::Has%s' % asm['feature'] 1092 1093 nan_restriction = 'intrinsics::bindings::NoNansOperation' 1094 if 'nan' in asm: 1095 nan_restriction = 'intrinsics::bindings::%sNanOperationsHandling' % asm['nan'] 1096 template_arg = 'true' if asm['nan'] == "Precise" else "false" 1097 if '<' in name: 1098 template_pos = name.index('<') 1099 name = name[0:template_pos+1] + template_arg + ", " + name[template_pos+1:] 1100 else: 1101 name += '<' + template_arg + '>' 1102 1103 if name not in string_labels: 1104 name_label = 'BINDING_NAME%d' % len(string_labels) 1105 string_labels[name] = name_label 1106 if check_compatible_assembler == _is_translator_compatible_assembler: 1107 yield ' %s if constexpr (std::is_same_v<FunctionCompareTag<kFunc>,' % ( 1108 '' if name_label == 'BINDING_NAME0' else ' } else' 1109 ) 1110 yield ' FunctionCompareTag<%s>>) {' % name 1111 static_names.append('static constexpr const char %s[] = "%s";' % (name_label, name)) 1112 else: 1113 name_label = string_labels[name] 1114 1115 mnemo = asm['mnemo'] 1116 mnemo_label = 'BINDING_MNEMO%d' % mnemo_idx[0] 1117 mnemo_idx[0] += 1 1118 static_mnemos.append('static constexpr const char %s[] = "%s";' % (mnemo_label, mnemo)) 1119 1120 restriction = [cpuid_restriction, nan_restriction] 1121 1122 if check_compatible_assembler == _is_translator_compatible_assembler: 1123 yield ' if (auto result = callback(' 1124 else: 1125 yield ' callback(' 1126 yield ' intrinsics::bindings::AsmCallInfo<' 1127 yield ' %s>(),' % ( 1128 ',\n '.join( 1129 [name_label, 1130 _get_asm_reference(asm), 1131 mnemo_label, 1132 _get_builder_reference(intr, asm) if gen_builder else 'void', 1133 cpuid_restriction, 1134 nan_restriction, 1135 'true' if _intr_has_side_effects(intr) else 'false', 1136 _get_c_type_tuple(intr['in']), 1137 _get_c_type_tuple(intr['out'])] + 1138 [_get_reg_operand_info(arg, 'intrinsics::bindings') 1139 for arg in asm['args']])) 1140 if check_compatible_assembler == _is_translator_compatible_assembler: 1141 yield ' std::forward<Args>(args)...); result.has_value()) {' 1142 yield ' return *std::move(result);' 1143 yield ' }' 1144 else: 1145 yield ' std::forward<Args>(args)...);' 1146 1147 1148def _get_c_type_tuple(arguments): 1149 return 'std::tuple<%s>' % ', '.join( 1150 _get_c_type(argument) for argument in arguments) 1151 1152 1153def _get_asm_type(asm, prefix=''): 1154 args = filter( 1155 lambda arg: not asm_defs.is_implicit_reg(arg['class']), asm['args']) 1156 return ', '.join(_get_asm_operand_type(arg, prefix) for arg in args) 1157 1158 1159def _get_asm_operand_type(arg, prefix=''): 1160 cls = arg.get('class') 1161 if asm_defs.is_x87reg(cls): 1162 return prefix + 'X87Register' 1163 if asm_defs.is_greg(cls): 1164 return prefix + 'Register' 1165 if asm_defs.is_xreg(cls): 1166 return prefix + 'XMMRegister' 1167 if asm_defs.is_mem_op(cls): 1168 return 'const ' + prefix + 'Operand&' 1169 if asm_defs.is_imm(cls): 1170 if cls == 'Imm2': 1171 return 'int8_t' 1172 return 'int' + cls[3:] + '_t' 1173 assert False 1174 1175 1176def _get_asm_reference(asm): 1177 # Because of misfeature of Itanium C++ ABI we couldn't just use MacroAssembler 1178 # to static cast these references if we want to use them as template argument: 1179 # https://ibob.bg/blog/2018/08/18/a-bug-in-the-cpp-standard/ 1180 1181 # Thankfully there are usually no need to use the same trick for MacroInstructions 1182 # since we may always rename these, except when immediates are involved. 1183 1184 # But for assembler we need to use actual type from where these 1185 # instructions come from! 1186 # 1187 # E.g. LZCNT have to be processed like this: 1188 # static_cast<void (Assembler_common_x86::*)( 1189 # typename Assembler_common_x86::Register, 1190 # typename Assembler_common_x86::Register)>( 1191 # &Assembler_common_x86::Lzcntl) 1192 assembler = 'std::tuple_element_t<%s, MacroAssembler>' % asm['macroassembler'] 1193 return 'static_cast<void (%s::*)(%s)>(%s&%s::%s%s)' % ( 1194 assembler, 1195 _get_asm_type(asm, 'typename %s::' % assembler), 1196 '\n ', 1197 assembler, 1198 'template ' if '<' in asm['asm'] else '', 1199 asm['asm']) 1200 1201def _get_builder_reference(intr, asm): 1202 return 'GetOpcode%s' % (asm['name']) 1203 1204def _load_intrs_def_files(intrs_def_files): 1205 result = {} 1206 for intrs_def in intrs_def_files: 1207 with open(intrs_def) as intrs: 1208 result.update(json.load(intrs)) 1209 result.pop('License', None) 1210 return result 1211 1212 1213def _load_intrs_arch_def(intrs_defs): 1214 json_data = [] 1215 for intrs_def in intrs_defs: 1216 with open(intrs_def) as intrs: 1217 json_array = json.load(intrs) 1218 while isinstance(len(json_array) > 0 and json_array[0], str): 1219 json_array.pop(0) 1220 json_data.extend(json_array) 1221 return json_data 1222 1223 1224def _load_macro_def(intrs, arch_intrs, insns_def, macroassembler): 1225 arch, insns = asm_defs.load_asm_defs(insns_def) 1226 for insn in insns: 1227 insn['macroassembler'] = macroassembler 1228 insns_map = dict((insn['name'], insn) for insn in insns) 1229 unprocessed_intrs = [] 1230 for arch_intr in arch_intrs: 1231 if arch_intr['insn'] in insns_map: 1232 insn = insns_map[arch_intr['insn']] 1233 _add_asm_insn(intrs, arch_intr, insn) 1234 else: 1235 unprocessed_intrs.append(arch_intr) 1236 return arch, unprocessed_intrs 1237 1238 1239def _is_interpreter_compatible_assembler(intr_asm): 1240 if intr_asm.get('usage', '') == 'inline-only': 1241 return False 1242 return True 1243 1244 1245def _is_translator_compatible_assembler(intr_asm): 1246 if intr_asm.get('usage', '') == 'no-inline': 1247 return False 1248 return True 1249 1250 1251 1252def _add_asm_insn(intrs, arch_intr, insn): 1253 name = ','.join(name_part.strip() for name_part in arch_intr['name'].split(',')) 1254 # Sanity checks: MacroInstruction could implement few different intrinsics but 1255 # number of arguments in arch intrinsic and arch-independent intrinsic 1256 # should match. 1257 # 1258 # Note: we allow combining intrinsics with variants and intrinsics without 1259 # variants (e.g. AbsF32 is combined with VectorAbsoluteFP for F32x2 and F32x4), 1260 # but don't allow macroinstructions which would handle different set of 1261 # variants for different intrinsics. 1262 1263 assert 'variants' not in insn or insn['variants'] == arch_intr['variants'] 1264 assert 'feature' not in insn or insn['feature'] == arch_intr['feature'] 1265 assert 'nan' not in insn or insn['nan'] == arch_intr['nan'] 1266 assert 'usage' not in insn or insn['usage'] == arch_intr['usage'] 1267 # Some intrinsics have extra inputs which can be ignored. e,g fpcr could be 1268 # ignored when not needed for precise emulation of NaNs. 1269 # Therefore we check that number inputs to (macro) instruction is less than 1270 # or equal to number of inputs to number of inputs to intrinsic. 1271 assert len(intrs[name]['in']) >= len(arch_intr['in']) 1272 assert len(intrs[name]['out']) == len(arch_intr['out']) 1273 1274 if 'variants' in arch_intr: 1275 insn['variants'] = arch_intr['variants'] 1276 if 'feature' in arch_intr: 1277 insn['feature'] = arch_intr['feature'] 1278 if 'nan' in arch_intr: 1279 insn['nan'] = arch_intr['nan'] 1280 if 'usage' in arch_intr: 1281 insn['usage'] = arch_intr['usage'] 1282 1283 for count, in_arg in enumerate(arch_intr['in']): 1284 # Sanity check: each in argument should only be used once - but if two 1285 # different intrinsics use them same macroinstruction it could be already 1286 # defined... yet it must be defined identically. 1287 assert ('ir_arg' not in insn['args'][in_arg] or 1288 insn['args'][in_arg]['ir_arg'] == count) 1289 insn['args'][in_arg]['ir_arg'] = count 1290 1291 for count, out_arg in enumerate(arch_intr['out']): 1292 # Sanity check: each out argument should only be used once, too. 1293 assert ('ir_res' not in insn['args'][out_arg] or 1294 insn['args'][out_arg]['ir_res'] == count) 1295 insn['args'][out_arg]['ir_res'] = count 1296 1297 # Note: one intrinsic could have more than one implementation (e.g. 1298 # SSE2 vs SSE4.2). 1299 if 'asm' not in intrs[name]: 1300 intrs[name]['asm'] = [] 1301 intrs[name]['asm'].append(insn) 1302 1303 1304def _open_asm_def_files(def_files, arch_def_files, asm_def_files, need_archs=True): 1305 intrs = _load_intrs_def_files(def_files) 1306 expanded_intrs = _expand_template_intrinsics(intrs) 1307 arch_intrs = _load_intrs_arch_def(arch_def_files) 1308 archs = [] 1309 macro_assemblers = 0 1310 for macro_def in asm_def_files: 1311 arch, arch_intrs = _load_macro_def(expanded_intrs, arch_intrs, macro_def, macro_assemblers) 1312 macro_assemblers += 1 1313 # Make sure that all intrinsics were found during processing of arch_intrs. 1314 assert arch_intrs == [] 1315 if need_archs: 1316 return archs, sorted(intrs.items()), sorted(expanded_intrs.items()) 1317 else: 1318 return sorted(intrs.items()) 1319 1320 1321def _expand_template_intrinsics(intrs): 1322 expanded_intrs = {} 1323 for name, intr in intrs.items(): 1324 if intr.get('class') != 'template': 1325 expanded_intrs[name] = intr 1326 else: 1327 for variant in intr.get('variants'): 1328 types = {} 1329 params = [param.strip() for param in variant.split(',')] 1330 for param in params: 1331 if param in ('true', 'false'): 1332 continue 1333 if re.search('[_a-zA-Z]', param): 1334 types['Type'+str(len(types))] = param 1335 new_intr = intr.copy() 1336 del new_intr['variants'] 1337 new_intr['in'] = [types.get(param, param) for param in new_intr.get('in')] 1338 new_intr['out'] = [types.get(param, param) for param in new_intr.get('out')] 1339 expanded_intrs[name+'<'+','.join(params)+'>'] = new_intr 1340 return expanded_intrs 1341 1342 1343def main(argv): 1344 # Usage: 1345 # gen_intrinsics.py --public_headers <intrinsics-inl.h> 1346 # <intrinsics_process_bindings-inl.h> 1347 # <interpreter_intrinsics_hooks-inl.h> 1348 # <translator_intrinsics_hooks-inl.h> 1349 # <mock_semantics_listener_intrinsics_hooks-inl.h> 1350 # <riscv64_to_x86_64/intrinsic_def.json", 1351 # ... 1352 # <riscv64_to_x86_64/machine_ir_intrinsic_binding.json>, 1353 # ... 1354 # <riscv64_to_x86_64/macro_def.json>, 1355 # ... 1356 # gen_intrinsics.py --text_asm_intrinsics_bindings <make_intrinsics-inl.h> 1357 # <riscv64_to_x86_64/intrinsic_def.json", 1358 # ... 1359 # <riscv64_to_x86_64/machine_ir_intrinsic_binding.json>, 1360 # ... 1361 # <riscv64_to_x86_64/macro_def.json>, 1362 # ... 1363 1364 def open_out_file(name): 1365 try: 1366 os.makedirs(os.path.dirname(name)) 1367 except: 1368 pass 1369 return open(name, 'w') 1370 1371 # Temporary special case for riscv64 to arm64. 1372 # TODO(b/362520361): generalize and combine with the below. 1373 option = argv[1] 1374 if option == 'arm64': 1375 mode = argv[2] 1376 out_files_end = 5 1377 def_files_end = out_files_end 1378 while argv[def_files_end].endswith('intrinsic_def.json'): 1379 def_files_end += 1 1380 if (def_files_end == len(argv)): 1381 break 1382 intrs = sorted(_load_intrs_def_files(argv[out_files_end:def_files_end]).items()) 1383 _gen_intrinsics_inl_h(open_out_file(argv[3]), intrs) 1384 _gen_semantic_player_types(intrs) 1385 _gen_interpreter_intrinsics_hooks_impl_inl_h(open_out_file(argv[4]), intrs, option) 1386 return 0 1387 1388 mode = argv[1] 1389 if mode in ('--text_asm_intrinsics_bindings', '--public_headers'): 1390 out_files_end = 3 if mode == '--text_asm_intrinsics_bindings' else 7 1391 def_files_end = out_files_end 1392 while argv[def_files_end].endswith('intrinsic_def.json'): 1393 def_files_end += 1 1394 arch_def_files_end = def_files_end 1395 while argv[arch_def_files_end].endswith('machine_ir_intrinsic_binding.json'): 1396 arch_def_files_end += 1 1397 archs, intrs, expanded_intrs = _open_asm_def_files( 1398 argv[out_files_end:def_files_end], 1399 argv[def_files_end:arch_def_files_end], 1400 argv[arch_def_files_end:], 1401 True) 1402 if mode == '--text_asm_intrinsics_bindings': 1403 _gen_make_intrinsics(open_out_file(argv[2]), expanded_intrs, archs) 1404 else: 1405 _gen_intrinsics_inl_h(open_out_file(argv[2]), intrs) 1406 _gen_process_bindings(open_out_file(argv[3]), expanded_intrs, archs) 1407 _gen_semantic_player_types(intrs) 1408 _gen_interpreter_intrinsics_hooks_impl_inl_h(open_out_file(argv[4]), intrs, '') 1409 _gen_translator_intrinsics_hooks_impl_inl_h( 1410 open_out_file(argv[5]), intrs) 1411 _gen_mock_semantics_listener_intrinsics_hooks_impl_inl_h( 1412 open_out_file(argv[6]), intrs) 1413 else: 1414 assert False, 'unknown option %s' % (mode) 1415 1416 return 0 1417 1418 1419if __name__ == '__main__': 1420 sys.exit(main(sys.argv)) 1421