• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python
2# Copyright 2021 The ANGLE Project Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5#
6# gen_spirv_builder_and_parser.py:
7#   Code generation for SPIR-V instruction builder and parser.
8#   NOTE: don't run this script directly. Run scripts/run_code_generation.py.
9
10import itertools
11import json
12import os
13import sys
14
15# ANGLE uses SPIR-V 1.0 currently, so there's no reason to generate code for newer instructions.
16SPIRV_GRAMMAR_FILE = '../../../third_party/vulkan-deps/spirv-headers/src/include/spirv/1.0/spirv.core.grammar.json'
17
18# The script has two sets of outputs, a header and source file for SPIR-V code generation, and a
19# header and source file for SPIR-V parsing.
20SPIRV_BUILDER_FILE = 'spirv_instruction_builder'
21SPIRV_PARSER_FILE = 'spirv_instruction_parser'
22
23# The types are either defined in spirv_types.h (to use strong types), or are enums that are
24# defined by SPIR-V headers.
25ANGLE_DEFINED_TYPES = [
26    'IdRef', 'IdResult', 'IdResultType', 'IdMemorySemantics', 'IdScope', 'LiteralInteger',
27    'LiteralString', 'LiteralContextDependentNumber', 'LiteralExtInstInteger',
28    'PairLiteralIntegerIdRef', 'PairIdRefLiteralInteger', 'PairIdRefIdRef'
29]
30
31HEADER_TEMPLATE = """// GENERATED FILE - DO NOT EDIT.
32// Generated by {script_name} using data from {data_source_name}.
33//
34// Copyright 2021 The ANGLE Project Authors. All rights reserved.
35// Use of this source code is governed by a BSD-style license that can be
36// found in the LICENSE file.
37//
38// {file_name}_autogen.h:
39//   Functions to {verb} SPIR-V binary for each instruction.
40
41#ifndef COMMON_SPIRV_{file_name_capitalized}AUTOGEN_H_
42#define COMMON_SPIRV_{file_name_capitalized}AUTOGEN_H_
43
44#include <spirv/unified1/spirv.hpp>
45
46#include "spirv_types.h"
47
48namespace angle
49{{
50namespace spirv
51{{
52{prototype_list}
53}}  // namespace spirv
54}}  // namespace angle
55
56#endif // COMMON_SPIRV_{file_name_capitalized}AUTOGEN_H_
57"""
58
59SOURCE_TEMPLATE = """// GENERATED FILE - DO NOT EDIT.
60// Generated by {script_name} using data from {data_source_name}.
61//
62// Copyright 2021 The ANGLE Project Authors. All rights reserved.
63// Use of this source code is governed by a BSD-style license that can be
64// found in the LICENSE file.
65//
66// {file_name}_autogen.cpp:
67//   Functions to {verb} SPIR-V binary for each instruction.
68
69#include "{file_name}_autogen.h"
70
71#include <string.h>
72
73#include "common/debug.h"
74
75namespace angle
76{{
77namespace spirv
78{{
79{helper_functions}
80
81{function_list}
82}}  // namespace spirv
83}}  // namespace angle
84"""
85
86BUILDER_HELPER_FUNCTIONS = """namespace
87{
88uint32_t MakeLengthOp(size_t length, spv::Op op)
89{
90    ASSERT(length <= 0xFFFFu);
91    ASSERT(op <= 0xFFFFu);
92
93    return static_cast<uint32_t>(length) << 16 | op;
94}
95}  // anonymous namespace
96
97void WriteSpirvHeader(std::vector<uint32_t> *blob, uint32_t idCount)
98{
99    // Header:
100    //
101    //  - Magic number
102    //  - Version (1.0)
103    //  - ANGLE's Generator number:
104    //     * 24 for tool id (higher 16 bits)
105    //     * 0 for tool version (lower 16 bits))
106    //  - Bound (idCount)
107    //  - 0 (reserved)
108    constexpr uint32_t kANGLEGeneratorId = 24;
109
110    ASSERT(blob->empty());
111
112    blob->push_back(spv::MagicNumber);
113    blob->push_back(0x00010000);
114    blob->push_back(kANGLEGeneratorId << 16 | 0);
115    blob->push_back(idCount);
116    blob->push_back(0x00000000);
117}
118"""
119
120BUILDER_HELPER_FUNCTION_PROTOTYPE = """
121    void WriteSpirvHeader(std::vector<uint32_t> *blob, uint32_t idCount);
122"""
123
124PARSER_FIXED_FUNCTIONS_PROTOTYPES = """void GetInstructionOpAndLength(const uint32_t *_instruction,
125    spv::Op *opOut, uint32_t *lengthOut);
126"""
127
128PARSER_FIXED_FUNCTIONS = """void GetInstructionOpAndLength(const uint32_t *_instruction,
129    spv::Op *opOut, uint32_t *lengthOut)
130{
131    constexpr uint32_t kOpMask = 0xFFFFu;
132    *opOut = static_cast<spv::Op>(_instruction[0] & kOpMask);
133    *lengthOut = _instruction[0] >> 16;
134}
135"""
136
137TEMPLATE_BUILDER_FUNCTION_PROTOTYPE = """void Write{op}(Blob *blob {param_list})"""
138TEMPLATE_BUILDER_FUNCTION_BODY = """{{
139    const size_t startSize = blob->size();
140    blob->push_back(0);
141    {push_back_lines}
142    (*blob)[startSize] = MakeLengthOp(blob->size() - startSize, spv::Op{op});
143}}
144"""
145
146TEMPLATE_PARSER_FUNCTION_PROTOTYPE = """void Parse{op}(const uint32_t *_instruction {param_list})"""
147TEMPLATE_PARSER_FUNCTION_BODY = """{{
148    spv::Op _op;
149    uint32_t _length;
150    GetInstructionOpAndLength(_instruction, &_op, &_length);
151    ASSERT(_op == spv::Op{op});
152    uint32_t _o = 1;
153    {parse_lines}
154}}
155"""
156
157
158def load_grammar(grammar_file):
159    with open(grammar_file) as grammar_in:
160        grammar = json.loads(grammar_in.read())
161
162    return grammar
163
164
165def remove_chars(string, chars):
166    return filter(lambda c: c not in chars, string)
167
168
169def make_camel_case(name):
170    return name[0].lower() + name[1:]
171
172
173class Writer:
174
175    def __init__(self):
176        self.path_prefix = os.path.dirname(os.path.realpath(__file__)) + os.path.sep
177        self.grammar = load_grammar(self.path_prefix + SPIRV_GRAMMAR_FILE)
178
179        # If an instruction has a parameter of these types, the instruction is ignored
180        self.unsupported_kinds = set(['LiteralSpecConstantOpInteger'])
181        # If an instruction requires a capability of these kinds, the instruction is ignored
182        self.unsupported_capabilities = set(['Kernel', 'Addresses'])
183        # If an instruction requires an extension other than these, the instruction is ignored
184        self.supported_extensions = set([])
185        # List of bit masks.  These have 'Mask' added to their typename in SPIR-V headers.
186        self.bit_mask_types = set([])
187
188        # List of generated instructions builder/parser functions so far.
189        self.instruction_builder_prototypes = [BUILDER_HELPER_FUNCTION_PROTOTYPE]
190        self.instruction_builder_impl = []
191        self.instruction_parser_prototypes = [PARSER_FIXED_FUNCTIONS_PROTOTYPES]
192        self.instruction_parser_impl = [PARSER_FIXED_FUNCTIONS]
193
194    def write_builder_and_parser(self):
195        """Generates four files, a set of header and source files for generating SPIR-V instructions
196        and a set for parsing SPIR-V instructions.  Only Vulkan instructions are processed (and not
197        OpenCL for example), and some assumptions are made based on ANGLE's usage (for example that
198        constants always fit in one 32-bit unit, as GLES doesn't support double or 64-bit types).
199
200        Additionally, enums and other parameter 'kinds' are not parsed from the json file, but
201        rather use the definitions from the SPIR-V headers repository and the spirv_types.h file."""
202
203        # Recurse through capabilities and accumulate ones that depend on unsupported ones.
204        self.accumulate_unsupported_capabilities()
205
206        self.find_bit_mask_types()
207
208        for instruction in self.grammar['instructions']:
209            self.generate_instruction_functions(instruction)
210
211        # Write out the files.
212        data_source_base_name = os.path.basename(SPIRV_GRAMMAR_FILE)
213        builder_template_args = {
214            'script_name': sys.argv[0],
215            'data_source_name': data_source_base_name,
216            'file_name': SPIRV_BUILDER_FILE,
217            'file_name_capitalized': remove_chars(SPIRV_BUILDER_FILE.upper(), '_'),
218            'verb': 'generate',
219            'helper_functions': BUILDER_HELPER_FUNCTIONS,
220            'prototype_list': ''.join(self.instruction_builder_prototypes),
221            'function_list': ''.join(self.instruction_builder_impl)
222        }
223        parser_template_args = {
224            'script_name': sys.argv[0],
225            'data_source_name': data_source_base_name,
226            'file_name': SPIRV_PARSER_FILE,
227            'file_name_capitalized': remove_chars(SPIRV_PARSER_FILE.upper(), '_'),
228            'verb': 'parse',
229            'helper_functions': '',
230            'prototype_list': ''.join(self.instruction_parser_prototypes),
231            'function_list': ''.join(self.instruction_parser_impl)
232        }
233
234        with (open(self.path_prefix + SPIRV_BUILDER_FILE + '_autogen.h', 'w')) as f:
235            f.write(HEADER_TEMPLATE.format(**builder_template_args))
236
237        with (open(self.path_prefix + SPIRV_BUILDER_FILE + '_autogen.cpp', 'w')) as f:
238            f.write(SOURCE_TEMPLATE.format(**builder_template_args))
239
240        with (open(self.path_prefix + SPIRV_PARSER_FILE + '_autogen.h', 'w')) as f:
241            f.write(HEADER_TEMPLATE.format(**parser_template_args))
242
243        with (open(self.path_prefix + SPIRV_PARSER_FILE + '_autogen.cpp', 'w')) as f:
244            f.write(SOURCE_TEMPLATE.format(**parser_template_args))
245
246    def requires_unsupported_capability(self, item):
247        depends = item.get('capabilities', [])
248        if len(depends) == 0:
249            return False
250        return all([dep in self.unsupported_capabilities for dep in depends])
251
252    def requires_unsupported_extension(self, item):
253        extensions = item.get('extensions', [])
254        return any([ext not in self.supported_extensions for ext in extensions])
255
256    def accumulate_unsupported_capabilities(self):
257        operand_kinds = self.grammar['operand_kinds']
258
259        # Find the Capability enum
260        for kind in filter(lambda entry: entry['kind'] == 'Capability', operand_kinds):
261            capabilities = kind['enumerants']
262            for capability in capabilities:
263                name = capability['enumerant']
264                # For each capability, see if any of the capabilities they depend on is unsupported.
265                # If so, add the capability to the list of unsupported ones.
266                if self.requires_unsupported_capability(capability):
267                    self.unsupported_capabilities.add(name)
268                    continue
269                # Do the same for extensions
270                if self.requires_unsupported_extension(capability):
271                    self.unsupported_capabilities.add(name)
272
273    def find_bit_mask_types(self):
274        operand_kinds = self.grammar['operand_kinds']
275
276        # Find the BitEnum categories
277        for bitEnumEntry in filter(lambda entry: entry['category'] == 'BitEnum', operand_kinds):
278            self.bit_mask_types.add(bitEnumEntry['kind'])
279
280    def get_operand_name(self, operand):
281        kind = operand['kind']
282        name = operand.get('name')
283
284        # If no name is given, derive the name from the kind
285        if name is None:
286            assert (kind.find(' ') == -1)
287            return make_camel_case(kind)
288
289        quantifier = operand.get('quantifier', '')
290        name = remove_chars(name, "'")
291
292        # First, a number of special-cases for optional lists
293        if quantifier == '*':
294            suffix = 'List'
295
296            # For IdRefs, change 'Xyz 1', +\n'Xyz 2', +\n...' to xyzList
297            if kind == 'IdRef':
298                if name.find(' ') != -1:
299                    name = name[0:name.find(' ')]
300
301            # Otherwise, if it's a pair in the form of 'Xyz, Abc, ...', change it to xyzAbcPairList
302            elif kind.startswith('Pair'):
303                suffix = 'PairList'
304
305            # Otherwise, it's just a list, so change `xyz abc` to `xyzAbcList
306
307            name = remove_chars(name, " ,.")
308            return make_camel_case(name) + suffix
309
310        # Otherwise, remove invalid characters and make the first letter lower case.
311        name = remove_chars(name, " .,+\n~")
312        name = make_camel_case(name)
313
314        # Make sure the name is not a C++ keyword
315        return 'default_' if name == 'default' else name
316
317    def get_operand_namespace(self, kind):
318        return '' if kind in ANGLE_DEFINED_TYPES else 'spv::'
319
320    def get_operand_type_suffix(self, kind):
321        return 'Mask' if kind in self.bit_mask_types else ''
322
323    def get_kind_cpp_type(self, kind):
324        return self.get_operand_namespace(kind) + kind + self.get_operand_type_suffix(kind)
325
326    def get_operand_type_in_and_out(self, operand):
327        kind = operand['kind']
328        quantifier = operand.get('quantifier', '')
329
330        is_array = quantifier == '*'
331        is_optional = quantifier == '?'
332        cpp_type = self.get_kind_cpp_type(kind)
333
334        if is_array:
335            type_in = 'const ' + cpp_type + 'List &'
336            type_out = cpp_type + 'List *'
337        elif is_optional:
338            type_in = 'const ' + cpp_type + ' *'
339            type_out = cpp_type + ' *'
340        else:
341            type_in = cpp_type
342            type_out = cpp_type + ' *'
343
344        return (type_in, type_out, is_array, is_optional)
345
346    def get_operand_push_back_line(self, operand, operand_name, is_array, is_optional):
347        kind = operand['kind']
348        pre = ''
349        post = ''
350        accessor = '.'
351        item = operand_name
352        item_dereferenced = item
353        if is_optional:
354            # If optional, surround with an if.
355            pre = 'if (' + operand_name + ') {\n'
356            post = '\n}'
357            accessor = '->'
358            item_dereferenced = '*' + item
359        elif is_array:
360            # If array, surround with a loop.
361            pre = 'for (const auto &operand : ' + operand_name + ') {\n'
362            post = '\n}'
363            item = 'operand'
364            item_dereferenced = item
365
366        # Usually the operand is one uint32_t, but it may be pair.  Handle the pairs especially.
367        if kind == 'PairLiteralIntegerIdRef':
368            line = 'blob->push_back(' + item + accessor + 'literal);'
369            line += 'blob->push_back(' + item + accessor + 'id);'
370        elif kind == 'PairIdRefLiteralInteger':
371            line = 'blob->push_back(' + item + accessor + 'id);'
372            line += 'blob->push_back(' + item + accessor + 'literal);'
373        elif kind == 'PairIdRefIdRef':
374            line = 'blob->push_back(' + item + accessor + 'id1);'
375            line += 'blob->push_back(' + item + accessor + 'id2);'
376        elif kind == 'LiteralString':
377            line = '{size_t d = blob->size();'
378            line += 'blob->resize(d + strlen(' + item_dereferenced + ') / 4 + 1, 0);'
379            # We currently don't have any big-endian devices in the list of supported platforms.
380            # Literal strings in SPIR-V are stored little-endian (SPIR-V 1.0 Section 2.2.1, Literal
381            # String), so if a big-endian device is to be supported, the string copy here should
382            # be adjusted.
383            line += 'ASSERT(IsLittleEndian());'
384            line += 'strcpy(reinterpret_cast<char *>(blob->data() + d), ' + item_dereferenced + ');}'
385        else:
386            line = 'blob->push_back(' + item_dereferenced + ');'
387
388        return pre + line + post
389
390    def get_operand_parse_line(self, operand, operand_name, is_array, is_optional):
391        kind = operand['kind']
392        pre = ''
393        post = ''
394        accessor = '->'
395
396        if is_optional:
397            # If optional, surround with an if, both checking if argument is provided, and whether
398            # it exists.
399            pre = 'if (' + operand_name + ' && _o < _length) {\n'
400            post = '\n}'
401        elif is_array:
402            # If array, surround with an if and a loop.
403            pre = 'if (' + operand_name + ') {\n'
404            pre += 'while (_o < _length) {\n'
405            post = '\n}}'
406            accessor = '.'
407
408        # Usually the operand is one uint32_t, but it may be pair.  Handle the pairs especially.
409        if kind == 'PairLiteralIntegerIdRef':
410            if is_array:
411                line = operand_name + '->emplace_back(' + kind + '{LiteralInteger(_instruction[_o]), IdRef(_instruction[_o + 1])});'
412                line += '_o += 2;'
413            else:
414                line = operand_name + '->literal = LiteralInteger(_instruction[_o++]);'
415                line += operand_name + '->id = IdRef(_instruction[_o++]);'
416        elif kind == 'PairIdRefLiteralInteger':
417            if is_array:
418                line = operand_name + '->emplace_back(' + kind + '{IdRef(_instruction[_o]), LiteralInteger(_instruction[_o + 1])});'
419                line += '_o += 2;'
420            else:
421                line = operand_name + '->id = IdRef(_instruction[_o++]);'
422                line += operand_name + '->literal = LiteralInteger(_instruction[_o++]);'
423        elif kind == 'PairIdRefIdRef':
424            if is_array:
425                line = operand_name + '->emplace_back(' + kind + '{IdRef(_instruction[_o]), IdRef(_instruction[_o + 1])});'
426                line += '_o += 2;'
427            else:
428                line = operand_name + '->id1 = IdRef(_instruction[_o++]);'
429                line += operand_name + '->id2 = IdRef(_instruction[_o++]);'
430        elif kind == 'LiteralString':
431            # The length of string in words is ceil((strlen(str) + 1) / 4).  This is equal to
432            # (strlen(str)+1+3) / 4, which is equal to strlen(str)/4+1.
433            assert (not is_array)
434            # See handling of LiteralString in get_operand_push_back_line.
435            line = 'ASSERT(IsLittleEndian());'
436            line += '*' + operand_name + ' = reinterpret_cast<const char *>(&_instruction[_o]);'
437            line += '_o += strlen(*' + operand_name + ') / 4 + 1;'
438        else:
439            if is_array:
440                line = operand_name + '->emplace_back(_instruction[_o++]);'
441            else:
442                line = '*' + operand_name + ' = ' + self.get_kind_cpp_type(
443                    kind) + '(_instruction[_o++]);'
444
445        return pre + line + post
446
447    def process_operand(self, operand, cpp_operands_in, cpp_operands_out, cpp_in_parse_lines,
448                        cpp_out_push_back_lines):
449        operand_name = self.get_operand_name(operand)
450        type_in, type_out, is_array, is_optional = self.get_operand_type_in_and_out(operand)
451
452        # Make the parameter list
453        cpp_operands_in.append(', ' + type_in + ' ' + operand_name)
454        cpp_operands_out.append(', ' + type_out + ' ' + operand_name)
455
456        # Make the builder body lines
457        cpp_out_push_back_lines.append(
458            self.get_operand_push_back_line(operand, operand_name, is_array, is_optional))
459
460        # Make the parser body lines
461        cpp_in_parse_lines.append(
462            self.get_operand_parse_line(operand, operand_name, is_array, is_optional))
463
464    def generate_instruction_functions(self, instruction):
465        name = instruction['opname']
466        assert (name.startswith('Op'))
467        name = name[2:]
468
469        # Skip if the instruction depends on a capability or extension we aren't interested in
470        if self.requires_unsupported_capability(instruction):
471            return
472        if self.requires_unsupported_extension(instruction):
473            return
474
475        operands = instruction.get('operands', [])
476
477        # Skip if any of the operands are not supported
478        if any([operand['kind'] in self.unsupported_kinds for operand in operands]):
479            return
480
481        cpp_operands_in = []
482        cpp_operands_out = []
483        cpp_in_parse_lines = []
484        cpp_out_push_back_lines = []
485
486        for operand in operands:
487            self.process_operand(operand, cpp_operands_in, cpp_operands_out, cpp_in_parse_lines,
488                                 cpp_out_push_back_lines)
489
490            # get_operand_parse_line relies on there only being one array parameter, and it being
491            # the last.
492            assert (operand.get('quantifier') != '*' or len(cpp_in_parse_lines) == len(operands))
493
494            if operand['kind'] == 'Decoration':
495                # Special handling of Op*Decorate instructions with a Decoration operand.  That
496                # operand always comes last, and implies a number of LiteralIntegers to follow.
497                assert (len(cpp_in_parse_lines) == len(operands))
498
499                decoration_operands = {
500                    'name': 'values',
501                    'kind': 'LiteralInteger',
502                    'quantifier': '*'
503                }
504                self.process_operand(decoration_operands, cpp_operands_in, cpp_operands_out,
505                                     cpp_in_parse_lines, cpp_out_push_back_lines)
506
507            elif operand['kind'] == 'ExecutionMode':
508                # Special handling of OpExecutionMode instruction with an ExecutionMode operand.
509                # That operand always comes last, and implies a number of LiteralIntegers to follow.
510                assert (len(cpp_in_parse_lines) == len(operands))
511
512                execution_mode_operands = {
513                    'name': 'operands',
514                    'kind': 'LiteralInteger',
515                    'quantifier': '*'
516                }
517                self.process_operand(execution_mode_operands, cpp_operands_in, cpp_operands_out,
518                                     cpp_in_parse_lines, cpp_out_push_back_lines)
519
520            elif operand['kind'] == 'ImageOperands':
521                # Special handling of OpImage* instructions with an ImageOperands operand.  That
522                # operand always comes last, and implies a number of IdRefs to follow with different
523                # meanings based on the bits set in said operand.
524                assert (len(cpp_in_parse_lines) == len(operands))
525
526                image_operands = {'name': 'imageOperandIds', 'kind': 'IdRef', 'quantifier': '*'}
527                self.process_operand(image_operands, cpp_operands_in, cpp_operands_out,
528                                     cpp_in_parse_lines, cpp_out_push_back_lines)
529
530        # Make the builder prototype body
531        builder_prototype = TEMPLATE_BUILDER_FUNCTION_PROTOTYPE.format(
532            op=name, param_list=''.join(cpp_operands_in))
533        self.instruction_builder_prototypes.append(builder_prototype + ';\n')
534        self.instruction_builder_impl.append(
535            builder_prototype + '\n' + TEMPLATE_BUILDER_FUNCTION_BODY.format(
536                op=name, push_back_lines='\n'.join(cpp_out_push_back_lines)))
537
538        if len(operands) == 0:
539            return
540
541        parser_prototype = TEMPLATE_PARSER_FUNCTION_PROTOTYPE.format(
542            op=name, param_list=''.join(cpp_operands_out))
543        self.instruction_parser_prototypes.append(parser_prototype + ';\n')
544        self.instruction_parser_impl.append(
545            parser_prototype + '\n' + TEMPLATE_PARSER_FUNCTION_BODY.format(
546                op=name, parse_lines='\n'.join(cpp_in_parse_lines)))
547
548
549def main():
550
551    # auto_script parameters.
552    if len(sys.argv) > 1:
553        if sys.argv[1] == 'inputs':
554            print(SPIRV_GRAMMAR_FILE)
555        elif sys.argv[1] == 'outputs':
556            output_files_base = [SPIRV_BUILDER_FILE, SPIRV_PARSER_FILE]
557            output_files = [
558                '_autogen.'.join(pair)
559                for pair in itertools.product(output_files_base, ['h', 'cpp'])
560            ]
561            print(','.join(output_files))
562        else:
563            print('Invalid script parameters')
564            return 1
565        return 0
566
567    writer = Writer()
568    writer.write_builder_and_parser()
569
570    return 0
571
572
573if __name__ == '__main__':
574    sys.exit(main())
575