• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python3
2#
3# Copyright (C) 2020 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import argparse
19import json
20import os.path
21import subprocess
22from collections import defaultdict
23from datetime import datetime
24
25from pyclibrary import CParser
26
27LICENSE_HEADER = """/*
28 * Copyright (C) 2020 The Android Open Source Project
29 *
30 * Licensed under the Apache License, Version 2.0 (the "License");
31 * you may not use this file except in compliance with the License.
32 * You may obtain a copy of the License at
33 *
34 *      http://www.apache.org/licenses/LICENSE-2.0
35 *
36 * Unless required by applicable law or agreed to in writing, software
37 * distributed under the License is distributed on an "AS IS" BASIS,
38 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
39 * See the License for the specific language governing permissions and
40 * limitations under the License.
41 */
42"""
43
44# Paths for output, relative to system/chre
45CHPP_PARSER_INCLUDE_PATH = "chpp/include/chpp/common"
46CHPP_PARSER_SOURCE_PATH = "chpp/common"
47
48
49def system_chre_abs_path():
50    """Gets the absolute path to the system/chre directory containing this script."""
51    script_dir = os.path.dirname(os.path.realpath(__file__))
52    # Assuming we're at system/chre/chpp/api_parser (i.e. up 2 to get to system/chre)
53    chre_project_base_dir = os.path.normpath(script_dir + "/../..")
54    return chre_project_base_dir
55
56
57class CodeGenerator:
58    """Given an ApiParser object, generates a header file with structure definitions in CHPP format.
59    """
60
61    def __init__(self, api, commit_hash):
62        """
63        :param api: ApiParser object
64        """
65        self.api = api
66        self.json = api.json
67        # Turn "chre_api/include/chre_api/chre/wwan.h" into "wwan"
68        self.service_name = self.json['filename'].split('/')[-1].split('.')[0]
69        self.capitalized_service_name = self.service_name[0].upper() + self.service_name[1:]
70        self.commit_hash = commit_hash
71
72    # ----------------------------------------------------------------------------------------------
73    # Header generation methods (plus some methods shared with encoder generation)
74    # ----------------------------------------------------------------------------------------------
75
76    def _autogen_notice(self):
77        out = []
78        out.append("// This file was automatically generated by {}\n".format(
79            os.path.basename(__file__)))
80        out.append("// Date: {} UTC\n".format(datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')))
81        out.append("// Source: {} @ commit {}\n\n".format(self.json['filename'], self.commit_hash))
82        out.append("// DO NOT modify this file directly, as those changes will be lost the next\n")
83        out.append("// time the script is executed\n\n")
84        return out
85
86    def _dump_to_file(self, output_filename, contents, dry_run, skip_clang_fomat):
87        """Outputs contents to output_filename, or prints contents if dry_run is True"""
88        if dry_run:
89            print("---- {} ----".format(output_filename))
90            print(contents)
91            print("---- end of {} ----\n".format(output_filename))
92        else:
93            with open(output_filename, 'w') as f:
94                f.write(contents)
95
96            if not skip_clang_fomat:
97                clang_format_path = os.path.normpath(
98                    "../../../../prebuilts/clang/host/linux-x86/clang-stable/bin/clang-format")
99                args = [clang_format_path, '-i', output_filename]
100                result = subprocess.run(args)
101                result.check_returncode()
102
103    def _is_array_type(self, type_info):
104        # If this is an array type, declarators will be a tuple containing a list of
105        # a single int element giving the size of the array
106        return len(type_info.declarators) == 1 and isinstance(type_info.declarators[0], list)
107
108    def _get_array_len(self, type_info):
109        return type_info.declarators[0][0]
110
111    def _get_chpp_type_from_chre(self, chre_type):
112        """Given 'chreWwanCellInfo' returns 'struct ChppWwanCellInfo', etc."""
113        prefix = self._get_struct_or_union_prefix(chre_type)
114
115        # First see if we have an explicit name override (e.g. for anonymous types)
116        for annotation in self.api.annotations[chre_type]["."]:
117            if annotation['annotation'] == "rename_type":
118                return prefix + annotation['type_override']
119
120        # Otherwise, use the existing type name, just replace the "chre" prefix with "Chpp"
121        if chre_type.startswith('chre'):
122            return prefix + 'Chpp' + chre_type[4:]
123        else:
124            raise RuntimeError("Couldn't figure out new type name for {}".format(chre_type))
125
126    def _get_chre_type_with_prefix(self, chre_type):
127        """Given 'chreWwanCellInfo' returns 'struct chreWwanCellInfo', etc."""
128        return self._get_struct_or_union_prefix(chre_type) + chre_type
129
130    def _get_chpp_header_type_from_chre(self, chre_type):
131        """Given 'chreWwanCellInfo' returns 'struct ChppWwanCellInfoWithHeader', etc."""
132        return self._get_chpp_type_from_chre(chre_type) + 'WithHeader'
133
134    def _get_member_comment(self, member_info):
135        for annotation in member_info['annotations']:
136            if annotation['annotation'] == "fixed_value":
137                return "  // Input ignored; always set to {}".format(annotation['value'])
138            elif annotation['annotation'] == "var_len_array":
139                return "  // References {} instances of {}".format(
140                    annotation['length_field'], self._get_member_type(member_info))
141        return ""
142
143    def _get_member_type(self, member_info, underlying_vla_type=False):
144        """Gets the CHPP type specification prefix for a struct/union member.
145
146        :param member_info: a dict element from self.api.structs_and_unions[struct]['members']
147        :param underlying_vla_type: (used only for var-len array types) False to output
148            'struct ChppOffset', and True to output the type that ChppOffset references
149        :return: type specification string that prefixes the field name, e.g. 'uint8_t'
150        """
151        # 4 cases to handle:
152        #   1) Annotation gives explicit type that we should use
153        #   2) Annotation says this is a variable length array (so use ChppOffset if
154        #      underlying_vla_type is False)
155        #   3) This is a struct/union type, so use the renamed (CHPP) type name
156        #   4) Regular type, e.g. uint32_t, so just use the type spec as-is
157        for annotation in member_info['annotations']:
158            if annotation['annotation'] == "rewrite_type":
159                return annotation['type_override']
160            elif not underlying_vla_type and annotation['annotation'] in ["var_len_array", "string"]:
161                return "struct ChppOffset"
162
163        if not underlying_vla_type and len(member_info['type'].declarators) > 0 and \
164                member_info['type'].declarators[0] == "*":
165            # This case should either be handled by rewrite_type (e.g. to uint32_t as
166            # opaque/ignored), or var_len_array
167            raise RuntimeError("Pointer types require annotation\n{}".format(
168                member_info))
169
170        if member_info['is_nested_type']:
171            return self._get_chpp_type_from_chre(member_info['nested_type_name'])
172
173        return member_info['type'].type_spec
174
175    def _get_member_type_suffix(self, member_info):
176        if self._is_array_type(member_info['type']):
177            return "[{}]".format(self._get_array_len(member_info['type']))
178        return ""
179
180    def _get_struct_or_union_prefix(self, chre_type):
181        return 'struct ' if not self.api.structs_and_unions[chre_type]['is_union'] else 'union '
182
183    def _gen_header_includes(self):
184        """Generates #include directives for use in <service>_types.h"""
185        out = ["#include <stdbool.h>\n#include <stddef.h>\n#include <stdint.h>\n\n"]
186
187        includes = ["chpp/app.h", "chpp/macros.h", "chre_api/chre/version.h"]
188        includes.extend(self.json['output_includes'])
189        for incl in sorted(includes):
190            out.append("#include \"{}\"\n".format(incl))
191        out.append("\n")
192        return out
193
194    def _gen_struct_or_union(self, name):
195        """Generates the definition for a single struct/union type"""
196        out = []
197        if not name.startswith('anon'):
198            out.append("//! See {{@link {}}} for details\n".format(name))
199        out.append("{} {{\n".format(self._get_chpp_type_from_chre(name)))
200        for member_info in self.api.structs_and_unions[name]['members']:
201            out.append("  {} {}{};{}\n".format(self._get_member_type(member_info),
202                                               member_info['name'],
203                                               self._get_member_type_suffix(member_info),
204                                               self._get_member_comment(member_info)))
205
206        out.append("} CHPP_PACKED_ATTR;\n\n")
207        return out
208
209    def _gen_header_struct(self, chre_type):
210        """Generates the definition for the type with header (WithHeader)"""
211        out = []
212        out.append("//! CHPP app header plus {}\n".format(
213            self._get_chpp_header_type_from_chre(chre_type)))
214
215        out.append("{} {{\n".format(self._get_chpp_header_type_from_chre(chre_type)))
216        out.append("  struct ChppAppHeader header;\n")
217        out.append("  {} payload;\n".format(self._get_chpp_type_from_chre(chre_type)))
218        out.append("} CHPP_PACKED_ATTR;\n\n")
219
220        return out
221
222    def _gen_structs_and_unions(self):
223        """Generates definitions for all struct/union types required for the root structs."""
224        out = []
225        out.append("CHPP_PACKED_START\n\n")
226
227        sorted_structs = self._sorted_structs(self.json['root_structs'])
228        for type_name in sorted_structs:
229            out.extend(self._gen_struct_or_union(type_name))
230
231        for chre_type in self.json['root_structs']:
232            out.extend(self._gen_header_struct(chre_type))
233
234        out.append("CHPP_PACKED_END\n\n")
235        return out
236
237    def _sorted_structs(self, root_nodes):
238        """Implements a topological sort on self.api.structs_and_unions.
239
240        Elements are ordered by definition dependency, i.e. if A includes a field of type B,
241        then B will appear before A in the returned list.
242        :return: list of keys in self.api.structs_and_unions, sorted by dependency order
243        """
244        result = []
245        visited = set()
246
247        def sort_helper(collection, key):
248            for dep in sorted(collection[key]['dependencies']):
249                if dep not in visited:
250                    visited.add(dep)
251                    sort_helper(collection, dep)
252            result.append(key)
253
254        for node in sorted(root_nodes):
255            sort_helper(self.api.structs_and_unions, node)
256        return result
257
258    # ----------------------------------------------------------------------------------------------
259    # Encoder function generation methods (CHRE --> CHPP)
260    # ----------------------------------------------------------------------------------------------
261
262    def _get_chpp_member_sizeof_call(self, member_info):
263        """Returns invocation used to determine the size of the provided member when encoded.
264
265        Will be either sizeof(<type in CHPP struct>) or a function call if the member contains a VLA
266        :param member_info: a dict element from self.api.structs_and_unions[struct]['members']
267        :return: string
268        """
269        type_name = None
270        if member_info['is_nested_type']:
271            chre_type = member_info['nested_type_name']
272            if self.api.structs_and_unions[chre_type]['has_vla_member']:
273                return "{}(in->{})".format(self._get_chpp_sizeof_function_name(chre_type),
274                                           member_info['name'])
275            else:
276                type_name = self._get_chpp_type_from_chre(chre_type)
277        else:
278            type_name = member_info['type'].type_spec
279        return "sizeof({})".format(type_name)
280
281    def _gen_chpp_sizeof_function(self, chre_type):
282        """Generates a function to determine the encoded size of the CHRE struct, if necessary."""
283        out = []
284
285        # Note that this function *should* work with unions as well, but at the time of writing
286        # it'll only be used with structs, so names, etc. are written with that in mind
287        struct_info = self.api.structs_and_unions[chre_type]
288        if not struct_info['has_vla_member']:
289            # No codegen necessary, just sizeof on the CHPP structure name is sufficient
290            return out
291
292        core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type)
293        parameter_name = core_type_name[0].lower() + core_type_name[1:]
294        chpp_type_name = self._get_chpp_header_type_from_chre(chre_type)
295        out.append("//! @return number of bytes required to represent the given\n"
296                   "//! {} along with the CHPP header as\n"
297                   "//! {}\n"
298                   .format(chre_type, chpp_type_name))
299        out.append("static size_t {}(\n        const {}{} *{}) {{\n"
300                   .format(self._get_chpp_sizeof_function_name(chre_type),
301                           self._get_struct_or_union_prefix(chre_type), chre_type,
302                           parameter_name))
303
304        # sizeof(this struct)
305        out.append("  size_t encodedSize = sizeof({});\n".format(chpp_type_name))
306
307        # Plus count * sizeof(type) for each var-len array included in this struct
308        for member_info in self.api.structs_and_unions[chre_type]['members']:
309            for annotation in member_info['annotations']:
310                if annotation['annotation'] == "var_len_array":
311                    # If the VLA field itself contains a VLA, then we'd need to generate a for
312                    # loop to calculate the size of each element individually - I don't think we
313                    # have any of these in the CHRE API today, so leaving this functionality out.
314                    # Also note that to support that case we'd also want to recursively call this
315                    # function to generate sizeof functions for nested fields.
316                    if member_info['is_nested_type'] and \
317                            self.api.structs_and_unions[member_info['nested_type_name']][
318                                'has_vla_member']:
319                        raise RuntimeError(
320                            "Nested variable-length arrays is not currently supported ({} "
321                            "in {})".format(member_info['name'], chre_type))
322
323                    out.append("  encodedSize += {}->{} * sizeof({});\n".format(
324                        parameter_name, annotation['length_field'],
325                        self._get_member_type(member_info, True)))
326                elif annotation['annotation'] == "string":
327                    out.append("  if ({}->{} != NULL) {{".format(
328                        parameter_name, annotation['field']))
329                    out.append("    encodedSize += strlen({}->{}) + 1;\n".format(
330                        parameter_name, annotation['field']))
331                    out.append("  }\n")
332
333        out.append("  return encodedSize;\n}\n\n")
334        return out
335
336    def _gen_chpp_sizeof_functions(self):
337        """For each root struct, generate necessary functions to determine their encoded size."""
338        out = []
339        for struct in self.json['root_structs']:
340            out.extend(self._gen_chpp_sizeof_function(struct))
341        return out
342
343    def _gen_conversion_includes(self):
344        """Generates #include directives for the conversion source file"""
345        out = ["#include \"chpp/macros.h\"\n"
346               "#include \"chpp/memory.h\"\n"
347               "#include \"chpp/common/{}_types.h\"\n\n".format(self.service_name)]
348        out.append("#include <stddef.h>\n#include <stdint.h>\n#include <string.h>\n\n")
349        return out
350
351    def _get_chpp_sizeof_function_name(self, chre_struct):
352        """Function name used to compute the encoded size of the given struct at runtime"""
353        core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_struct)
354        return "chpp{}SizeOf{}FromChre".format(self.capitalized_service_name, core_type_name)
355
356    def _get_encoding_function_name(self, chre_type):
357        core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type)
358        return "chpp{}Convert{}FromChre".format(self.capitalized_service_name, core_type_name)
359
360    def _gen_encoding_function_signature(self, chre_type):
361        out = []
362        out.append("void {}(\n".format(self._get_encoding_function_name(chre_type)))
363        out.append("    const {}{} *in,\n".format(
364            self._get_struct_or_union_prefix(chre_type), chre_type))
365        out.append("    {} *out".format(self._get_chpp_type_from_chre(chre_type)))
366        if self.api.structs_and_unions[chre_type]['has_vla_member']:
367            out.append(",\n")
368            out.append("    uint8_t *payload,\n")
369            out.append("    size_t payloadSize,\n")
370            out.append("    uint16_t *vlaOffset")
371        out.append(")")
372        return out
373
374    def _gen_string_encoding(self, member_info, annotation):
375        out = []
376        # Might want to revisit this if we ever end up supporting NULL strings
377        # in our API. We can assert here since there's currently no API that
378        # does so.
379        member_name = member_info['name']
380        out.append("  if (in->{} != NULL) {{\n".format(member_name))
381        out.append("    size_t strSize = strlen(in->{}) + 1;\n".format(member_name))
382        out.append("    memcpy(&payload[*vlaOffset], in->{}, strSize);\n".format(
383            member_name))
384        out.append("    out->{}.length = (uint16_t)(strSize);\n".format(
385            member_name))
386        out.append("    out->{}.offset = *vlaOffset;\n".format(member_name))
387        out.append("    *vlaOffset += out->{}.length;\n".format(member_name))
388        out.append("  } else {\n")
389        out.append("    out->{}.length = 0;\n".format(member_name))
390        out.append("    out->{}.offset = 0;\n".format(member_name))
391        out.append("  }\n\n")
392
393        return out
394
395    def _gen_vla_encoding(self, member_info, annotation):
396        out = []
397
398        variable_name = member_info['name']
399        chpp_type = self._get_member_type(member_info, True)
400
401        if member_info['is_nested_type']:
402            out.append("\n  {} *{} = ({} *) &payload[*vlaOffset];\n".format(
403                chpp_type, variable_name, chpp_type))
404
405        out.append("  out->{}.length = (uint16_t)(in->{} * {});\n".format(
406            member_info['name'], annotation['length_field'],
407            self._get_chpp_member_sizeof_call(member_info)))
408
409        out.append("  CHPP_ASSERT((size_t)(*vlaOffset + out->{}.length) <= payloadSize);\n".format(
410            member_info['name']))
411
412        out.append("  if (out->{}.length > 0 &&\n"
413                   "      *vlaOffset + out->{}.length <= payloadSize) {{\n".format(
414            member_info['name'], member_info['name']))
415
416        if member_info['is_nested_type']:
417            out.append("    for (size_t i = 0; i < in->{}; i++) {{\n".format(
418                annotation['length_field']))
419            out.append("      {}".format(
420                self._get_assignment_statement_for_field(member_info, in_vla_loop=True)))
421            out.append("    }\n")
422        else:
423            out.append("memcpy(&payload[*vlaOffset], in->{}, in->{} * sizeof({}));\n".format(
424                member_info['name'], annotation['length_field'], chpp_type))
425
426        out.append("    out->{}.offset = *vlaOffset;\n".format(member_info['name']))
427        out.append("    *vlaOffset += out->{}.length;\n".format(member_info['name']))
428
429        out.append("  } else {\n")
430        out.append("    out->{}.offset = 0;\n".format(member_info['name']))
431        out.append("  }\n");
432
433        return out
434
435    # ----------------------------------------------------------------------------------------------
436    # Encoder / decoder function generation methods (CHRE <--> CHPP)
437    # ----------------------------------------------------------------------------------------------
438
439    def _get_assignment_statement_for_field(self, member_info,
440                                            in_vla_loop=False,
441                                            containing_field_name=None,
442                                            decode_mode=False):
443        """Returns a statement to assign the provided member
444
445        :param member_info:
446        :param in_vla_loop: True if we're currently inside a loop and should append [i]
447        :param containing_field_name: Additional member name to use to access the target field, or
448        None; for example the normal case is "out->field = in->field", but if we're generating
449        assignments in the parent conversion function (e.g. as used for union variants), we need to
450        do "out->nested_field.field = in->nested_field.field"
451        :param decode_mode: True converts from CHPP to CHRE. False from CHRE to CHPP
452        :return: assignment statement as a string
453        """
454        array_index = "[i]" if in_vla_loop else ""
455        output_accessor = "" if in_vla_loop else "out->"
456        containing_field = containing_field_name + "." if containing_field_name is not None else ""
457
458        output_variable = "{}{}{}{}".format(output_accessor, containing_field, member_info['name'],
459                                            array_index)
460        input_variable = "in->{}{}{}".format(containing_field, member_info['name'], array_index)
461
462        if decode_mode and in_vla_loop:
463            output_variable = "{}Out{}".format(member_info['name'], array_index)
464            input_variable = "{}In{}".format(member_info['name'], array_index)
465
466        if member_info['is_nested_type']:
467            chre_type = member_info['nested_type_name']
468            has_vla_member = self.api.structs_and_unions[chre_type]['has_vla_member']
469            if decode_mode:
470                # Use decoding function
471                vla_params = ", inSize" if has_vla_member else ""
472                out = "if (!{}(&{}, &{}{})) {{\n".format(
473                    self._get_decoding_function_name(chre_type), input_variable,
474                    output_variable, vla_params)
475                if has_vla_member:
476                    out += "  CHPP_FREE_AND_NULLIFY({}Out);\n".format(member_info['name'])
477                out += "  return false;\n"
478                out += "}\n"
479                return out
480            else:
481                # Use encoding function
482                vla_params = ", payload, payloadSize, vlaOffset" if has_vla_member else ""
483                return "{}(&{}, &{}{});\n".format(
484                    self._get_encoding_function_name(chre_type), input_variable, output_variable,
485                    vla_params)
486        elif self._is_array_type(member_info['type']):
487            # Array of primitive type (e.g. uint32_t[8]) - use memcpy
488            return "memcpy({}, {}, sizeof({}));\n".format(output_variable, input_variable,
489                                                          output_variable)
490        else:
491            # Regular assignment
492            return "{} = {};\n".format(output_variable, input_variable)
493
494    def _gen_union_variant_conversion_code(self, member_info, annotation, decode_mode):
495        """Generates a switch statement to encode the "active"/"used" field within a union.
496
497        Handles cases where a union has multiple types, but there's another peer/adjacent field
498        which tells you which field in the union is to be used. Outputs code like this:
499        switch (in->{discriminator field}) {
500            case {first discriminator value associated with a fields}:
501                {conversion code for the field associated with this discriminator value}
502                ...
503        :param chre_type: CHRE type of the union
504        :param annotation: Reference to JSON annotation data with the discriminator mapping data
505        :param decode_mode: False encodes from CHRE to CHPP. True decodes from CHPP to CHRE
506        :return: list of strings
507        """
508        out = []
509        chre_type = member_info['nested_type_name']
510        struct_info = self.api.structs_and_unions[chre_type]
511
512        # Start off by zeroing out the union field so any padding is set to a consistent value
513        out.append("  memset(&out->{}, 0, sizeof(out->{}));\n".format(member_info['name'],
514                                                                      member_info['name']))
515
516        # Next, generate the switch statement that will copy over the proper values
517        out.append("  switch (in->{}) {{\n".format(annotation['discriminator']))
518        for value, field_name in annotation['mapping']:
519            out.append("    case {}:\n".format(value))
520
521            found = False
522            for nested_member_info in struct_info['members']:
523                if nested_member_info['name'] == field_name:
524                    out.append("      {}".format(
525                        self._get_assignment_statement_for_field(
526                            nested_member_info,
527                            containing_field_name=member_info['name'],
528                            decode_mode=decode_mode)))
529                    found = True
530                    break
531
532            if not found:
533                raise RuntimeError("Invalid mapping - couldn't find target field {} in struct {}"
534                                   .format(field_name, chre_type))
535
536            out.append("      break;\n")
537
538        out.append("    default:\n"
539                   "      CHPP_ASSERT(false);\n"
540                   "  }\n")
541
542        return out
543
544    def _gen_conversion_function(self, chre_type, already_generated, decode_mode):
545        out = []
546
547        struct_info = self.api.structs_and_unions[chre_type]
548        for dependency in sorted(struct_info['dependencies']):
549            if dependency not in already_generated:
550                out.extend(
551                    self._gen_conversion_function(dependency, already_generated, decode_mode))
552
553        # Skip if we've already generated code for this type, or if it's a union (in which case we
554        # handle the assignment in the parent structure to enable support for discrimination of
555        # which field in the union to use)
556        if chre_type in already_generated or struct_info['is_union']:
557            return out
558        already_generated.add(chre_type)
559
560        out.append("static ")
561        if decode_mode:
562            out.extend(self._gen_decoding_function_signature(chre_type))
563        else:
564            out.extend(self._gen_encoding_function_signature(chre_type))
565        out.append(" {\n")
566
567        for member_info in self.api.structs_and_unions[chre_type]['members']:
568            generated_by_annotation = False
569            for annotation in member_info['annotations']:
570                if annotation['annotation'] == "fixed_value":
571                    if self._is_array_type(member_info['type']):
572                        out.append("  memset(&out->{}, {}, sizeof(out->{}));\n".format(
573                            member_info['name'], annotation['value'], member_info['name']))
574                    else:
575                        out.append("  out->{} = {};\n".format(member_info['name'],
576                                                              annotation['value']))
577                    generated_by_annotation = True
578                    break
579                elif annotation['annotation'] == "enum":
580                    # TODO: generate range verification code?
581                    pass
582                elif annotation['annotation'] == "var_len_array":
583                    if decode_mode:
584                        out.extend(self._gen_vla_decoding(member_info, annotation))
585                    else:
586                        out.extend(self._gen_vla_encoding(member_info, annotation))
587                    generated_by_annotation = True
588                    break
589                elif annotation['annotation'] == "string":
590                    if decode_mode:
591                        out.extend(self._gen_string_decoding(member_info, annotation))
592                    else:
593                        out.extend(self._gen_string_encoding(member_info, annotation))
594                    generated_by_annotation = True
595                    break
596                elif annotation['annotation'] == "union_variant":
597                    out.extend(self._gen_union_variant_conversion_code(
598                        member_info, annotation, decode_mode))
599                    generated_by_annotation = True
600                    break
601
602            if not generated_by_annotation:
603                out.append("  {}".format(
604                    self._get_assignment_statement_for_field(member_info, decode_mode=decode_mode)))
605
606        if decode_mode:
607            out.append("\n  return true;\n")
608
609        out.append("}\n\n")
610        return out
611
612    def _gen_conversion_functions(self, decode_mode):
613        out = []
614        already_generated = set()
615        for struct in self.json['root_structs']:
616            out.extend(self._gen_conversion_function(struct, already_generated, decode_mode))
617        return out
618
619    def _strip_prefix_and_service_from_chre_struct_name(self, struct):
620        """Strip 'chre' and service prefix, e.g. 'chreWwanCellResultInfo' -> 'CellResultInfo'"""
621        chre_stripped = struct[4:]
622        upcased_service_name = self.service_name[0].upper() + self.service_name[1:]
623        if not struct.startswith('chre') or not chre_stripped.startswith(upcased_service_name):
624            # If this happens, we need to update the script to handle it. Right we assume struct
625            # naming follows the pattern "chre<Service_name><Thing_name>"
626            raise RuntimeError("Unexpected structure name {}".format(struct))
627
628        return chre_stripped[len(self.service_name):]
629
630    # ----------------------------------------------------------------------------------------------
631    # Memory allocation generation methods
632    # ----------------------------------------------------------------------------------------------
633
634    def _get_chpp_sizeof_call(self, chre_type):
635        """Returns invocation used to determine the size of the provided CHRE struct (with the CHPP
636        app header) after encoding.
637
638        Like _get_chpp_member_sizeof_call(), except for a top-level type assigned to the variable
639        "in" rather than a member within a structure (e.g. a VLA field)
640        :param chre_type: CHRE type name
641        :return: string
642        """
643        if self.api.structs_and_unions[chre_type]['has_vla_member']:
644            return "{}(in)".format(self._get_chpp_sizeof_function_name(chre_type))
645        else:
646            return "sizeof({})".format(self._get_chpp_header_type_from_chre(chre_type))
647
648    def _get_encode_allocation_function_name(self, chre_type):
649        core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type)
650        return "chpp{}{}FromChre".format(self.capitalized_service_name, core_type_name)
651
652    def _gen_encode_allocation_function_signature(self, chre_type, gen_docs=False):
653        out = []
654        if gen_docs:
655            out.append("/**\n"
656                       " * Converts from given CHRE structure to serialized CHPP type.\n"
657                       " *\n"
658                       " * @param in Fully-formed CHRE structure.\n"
659                       " * @param out Upon success, will point to a buffer allocated with "
660                       "chppMalloc().\n"
661                       " * It is the responsibility of the caller to set the values of the CHPP "
662                       "app layer header, and to free the buffer when it is no longer needed via "
663                       "chppFree() or CHPP_FREE_AND_NULLIFY().\n"
664                       " * @param outSize Upon success, will be set to the size of the output "
665                       "buffer, in bytes.\n"
666                       " *\n"
667                       " * @return true on success, false if memory allocation failed.\n"
668                       " */\n")
669        out.append("bool {}(\n".format(self._get_encode_allocation_function_name(chre_type)))
670        out.append("    const {}{} *in,\n".format(
671            self._get_struct_or_union_prefix(chre_type), chre_type))
672        out.append("    {} **out,\n".format(self._get_chpp_header_type_from_chre(chre_type)))
673        out.append("    size_t *outSize)")
674        return out
675
676    def _gen_encode_allocation_function(self, chre_type):
677        out = []
678        out.extend(self._gen_encode_allocation_function_signature(chre_type))
679        out.append(" {\n")
680        out.append("  CHPP_NOT_NULL(out);\n")
681        out.append("  CHPP_NOT_NULL(outSize);\n\n")
682        out.append("  size_t payloadSize = {};\n".format(
683            self._get_chpp_sizeof_call(chre_type)))
684        out.append("  *out = chppMalloc(payloadSize);\n")
685
686        out.append("  if (*out != NULL) {\n")
687
688        struct_info = self.api.structs_and_unions[chre_type]
689        if struct_info['has_vla_member']:
690            out.append("    uint8_t *payload = (uint8_t *) &(*out)->payload;\n")
691            out.append("    uint16_t vlaOffset = sizeof({});\n".format(
692                self._get_chpp_type_from_chre(chre_type)))
693
694        out.append("    {}(in, &(*out)->payload".format(
695            self._get_encoding_function_name(chre_type)))
696        if struct_info['has_vla_member']:
697            out.append(", payload, payloadSize, &vlaOffset")
698        out.append(");\n")
699        out.append("    *outSize = payloadSize;\n")
700        out.append("    return true;\n")
701        out.append("  }\n")
702
703        out.append("  return false;\n}\n\n")
704        return out
705
706    def _gen_encode_allocation_functions(self):
707        out = []
708        for chre_type in self.json['root_structs']:
709            out.extend(self._gen_encode_allocation_function(chre_type))
710        return out
711
712    def _gen_encode_allocation_function_signatures(self):
713        out = []
714        for chre_type in self.json['root_structs']:
715            out.extend(self._gen_encode_allocation_function_signature(chre_type, True))
716            out.append(";\n\n")
717        return out
718
719    # ----------------------------------------------------------------------------------------------
720    # Decoder function generation methods (CHPP --> CHRE)
721    # ----------------------------------------------------------------------------------------------
722
723    def _get_decoding_function_name(self, chre_type):
724        core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type)
725        return "chpp{}Convert{}ToChre".format(self.capitalized_service_name, core_type_name)
726
727    def _gen_decoding_function_signature(self, chre_type):
728        out = []
729        out.append("bool {}(\n".format(self._get_decoding_function_name(chre_type)))
730        out.append("    const {} *in,\n".format(self._get_chpp_type_from_chre(chre_type)))
731        out.append("    {} *out".format(self._get_chre_type_with_prefix(chre_type)))
732        if self.api.structs_and_unions[chre_type]['has_vla_member']:
733            out.append(",\n")
734            out.append("    size_t inSize")
735        out.append(")")
736        return out
737
738    def _gen_string_decoding(self, member_info, annotation):
739        out = []
740        variable_name = member_info['name']
741        out.append("\n")
742        out.append("  if (in->{}.length == 0) {{\n".format(variable_name))
743        out.append("    out->{} = NULL;\n".format(variable_name))
744        out.append("  } else {\n")
745        out.append("    char *{}Out = chppMalloc(in->{}.length);\n".format(
746            variable_name, variable_name))
747        out.append("    if ({}Out == NULL) {{\n".format(variable_name))
748        out.append("      return false;\n")
749        out.append("    }\n\n")
750        out.append("    memcpy({}Out, &((const uint8_t *)in)[in->{}.offset],\n".format(
751            variable_name, variable_name))
752        out.append("      in->{}.length);\n".format(variable_name))
753        out.append("    out->{} = {}Out;\n".format(variable_name, variable_name))
754        out.append("  }\n")
755
756        return out
757
758
759    def _gen_vla_decoding(self, member_info, annotation):
760        out = []
761
762        variable_name = member_info['name']
763        chpp_type = self._get_member_type(member_info, True)
764        if member_info['is_nested_type']:
765            chre_type = self._get_chre_type_with_prefix(member_info['nested_type_name'])
766        else:
767            chre_type = chpp_type
768
769        out.append("\n")
770        out.append("  if (in->{}.length == 0) {{\n".format(variable_name))
771        out.append("    out->{} = NULL;\n".format(variable_name))
772        out.append("  }\n")
773        out.append("  else {\n")
774        out.append("    if (in->{}.offset + in->{}.length > inSize ||\n".format(
775            variable_name, variable_name))
776        out.append("        in->{}.length != in->{} * sizeof({})) {{\n".format(
777            variable_name, annotation['length_field'], chpp_type))
778
779        out.append("      return false;\n")
780        out.append("    }\n\n")
781
782        if member_info['is_nested_type']:
783            out.append("    const {} *{}In =\n".format(chpp_type, variable_name))
784            out.append("        (const {} *) &((const uint8_t *)in)[in->{}.offset];\n\n".format(
785                chpp_type, variable_name))
786
787        out.append("    {} *{}Out = chppMalloc(in->{} * sizeof({}));\n".format(
788            chre_type, variable_name, annotation['length_field'], chre_type))
789        out.append("    if ({}Out == NULL) {{\n".format(variable_name))
790        out.append("      return false;\n")
791        out.append("    }\n\n")
792
793        if member_info['is_nested_type']:
794            out.append("    for (size_t i = 0; i < in->{}; i++) {{\n".format(
795                annotation['length_field'], variable_name))
796            out.append("      {}".format(self._get_assignment_statement_for_field(
797                member_info, in_vla_loop=True, decode_mode=True)))
798            out.append("    }\n")
799        else:
800            out.append("    memcpy({}Out, &((const uint8_t *)in)[in->{}.offset],\n".format(
801                variable_name, variable_name))
802            out.append("      in->{} * sizeof({}));\n".format(
803                annotation['length_field'], chre_type))
804
805        out.append("    out->{} = {}Out;\n".format(variable_name, variable_name))
806        out.append("  }\n\n")
807
808        return out
809
810    def _get_decode_allocation_function_name(self, chre_type):
811        core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type)
812        return "chpp{}{}ToChre".format(self.capitalized_service_name, core_type_name)
813
814    def _gen_decode_allocation_function_signature(self, chre_type, gen_docs=False):
815        out = []
816        if gen_docs:
817            out.append("/**\n"
818                       " * Converts from serialized CHPP structure to a CHRE type.\n"
819                       " *\n"
820                       " * @param in Fully-formed CHPP structure.\n"
821                       " * @param in Size of the CHPP structure in bytes.\n"
822                       " *\n"
823                       " * @return If successful, a pointer to a CHRE structure allocated with "
824                       "chppMalloc(). If unsuccessful, null.\n"
825                       " * It is the responsibility of the caller to free the buffer when it is no "
826                       "longer needed via chppFree() or CHPP_FREE_AND_NULLIFY().\n"
827                       " */\n")
828
829        out.append("{} *{}(\n".format(
830            self._get_chre_type_with_prefix(chre_type),
831            self._get_decode_allocation_function_name(chre_type)))
832        out.append("    const {} *in,\n".format(self._get_chpp_type_from_chre(chre_type)))
833        out.append("    size_t inSize)")
834        return out
835
836    def _gen_decode_allocation_function(self, chre_type):
837        out = []
838
839        out.extend(self._gen_decode_allocation_function_signature(chre_type))
840        out.append(" {\n")
841
842        out.append("  {} *out = NULL;\n\n".format(
843            self._get_chre_type_with_prefix(chre_type)))
844
845        out.append("  if (inSize >= sizeof({})) {{\n".format(
846            self._get_chpp_type_from_chre(chre_type)))
847
848        out.append("    out = chppMalloc(sizeof({}));\n".format(
849            self._get_chre_type_with_prefix(chre_type)))
850        out.append("    if (out != NULL) {\n")
851
852        struct_info = self.api.structs_and_unions[chre_type]
853
854        out.append("      if (!{}(in, out".format(self._get_decoding_function_name(chre_type)))
855        if struct_info['has_vla_member']:
856            out.append(", inSize")
857        out.append(")) {")
858        out.append("        CHPP_FREE_AND_NULLIFY(out);\n")
859        out.append("      }\n")
860
861        out.append("    }\n")
862        out.append("  }\n\n")
863        out.append("  return out;\n")
864        out.append("}\n")
865        return out
866
867    def _gen_decode_allocation_functions(self):
868        out = []
869        for chre_type in self.json['root_structs']:
870            out.extend(self._gen_decode_allocation_function(chre_type))
871        return out
872
873    def _gen_decode_allocation_function_signatures(self):
874        out = []
875        for chre_type in self.json['root_structs']:
876            out.extend(self._gen_decode_allocation_function_signature(chre_type, True))
877            out.append(";\n\n")
878        return out
879
880    # ----------------------------------------------------------------------------------------------
881    # Public methods
882    # ----------------------------------------------------------------------------------------------
883
884    def generate_header_file(self, dry_run=False, skip_clang_format=False):
885        """Creates a C header file for this API and writes it to the file indicated in the JSON."""
886        filename = self.service_name + "_types.h"
887        if not dry_run:
888            print("Generating {} ... ".format(filename), end='', flush=True)
889        output_file = os.path.join(system_chre_abs_path(), CHPP_PARSER_INCLUDE_PATH, filename)
890        header = self.generate_header_string()
891        self._dump_to_file(output_file, header, dry_run, skip_clang_format)
892        if not dry_run:
893            print("done")
894
895    def generate_header_string(self):
896        """Returns a C header with structure definitions for this API."""
897        # To defer concatenation (speed things up), build the file as a list of strings then only
898        # concatenate once at the end
899        out = [LICENSE_HEADER]
900
901        header_guard = "CHPP_{}_TYPES_H_".format(self.service_name.upper())
902
903        out.append("#ifndef {}\n#define {}\n\n".format(header_guard, header_guard))
904        out.extend(self._autogen_notice())
905        out.extend(self._gen_header_includes())
906        out.append("#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n")
907        out.extend(self._gen_structs_and_unions())
908
909        out.append("\n// Encoding functions (CHRE --> CHPP)\n\n")
910        out.extend(self._gen_encode_allocation_function_signatures())
911
912        out.append("\n// Decoding functions (CHPP --> CHRE)\n\n")
913        out.extend(self._gen_decode_allocation_function_signatures())
914
915        out.append("#ifdef __cplusplus\n}\n#endif\n\n")
916        out.append("#endif  // {}\n".format(header_guard))
917        return ''.join(out)
918
919    def generate_conversion_file(self, dry_run=False, skip_clang_format=False):
920        """Generates a .c file with functions for encoding CHRE structs into CHPP and vice versa."""
921        filename = self.service_name + "_convert.c"
922        if not dry_run:
923            print("Generating {} ... ".format(filename), end='', flush=True)
924        contents = self.generate_conversion_string()
925        output_file = os.path.join(system_chre_abs_path(), CHPP_PARSER_SOURCE_PATH, filename)
926        self._dump_to_file(output_file, contents, dry_run, skip_clang_format)
927        if not dry_run:
928            print("done")
929
930    def generate_conversion_string(self):
931        """Returns C code for encoding CHRE structs into CHPP and vice versa."""
932        out = [LICENSE_HEADER, "\n"]
933
934        out.extend(self._autogen_notice())
935        out.extend(self._gen_conversion_includes())
936
937        out.append("\n// Encoding (CHRE --> CHPP) size functions\n\n")
938        out.extend(self._gen_chpp_sizeof_functions())
939        out.append("\n// Encoding (CHRE --> CHPP) conversion functions\n\n")
940        out.extend(self._gen_conversion_functions(False))
941        out.append("\n// Encoding (CHRE --> CHPP) top-level functions\n\n")
942        out.extend(self._gen_encode_allocation_functions())
943
944        out.append("\n// Decoding (CHPP --> CHRE) conversion functions\n\n")
945        out.extend(self._gen_conversion_functions(True))
946        out.append("\n// Decoding (CHPP --> CHRE) top-level functions\n\n")
947        out.extend(self._gen_decode_allocation_functions())
948
949        return ''.join(out)
950
951
952class ApiParser:
953    """Given a file-specific set of annotations (extracted from JSON annotations file), parses a
954    single API header file into data structures suitable for use with code generation.
955    """
956
957    def __init__(self, json_obj):
958        """Initialize and parse the API file described in the provided JSON-derived object.
959
960        :param json_obj: Extracted file-specific annotations from JSON
961        """
962        self.json = json_obj
963        self.structs_and_unions = {}
964        self._parse_annotations()
965        self._parse_api()
966
967    def _parse_annotations(self):
968        # Convert annotations list to a more usable data structure: dict keyed by structure name,
969        # containing a dict keyed by field name, containing a list of annotations (as they
970        # appear in the JSON). In other words, we can easily get all of the annotations for the
971        # "version" field in "chreWwanCellInfoResult" via
972        # annotations['chreWwanCellInfoResult']['version']. This is also a defaultdict, so it's safe
973        # to access if there are no annotations for this structure + field; it'll just give you
974        # an empty list in that case.
975        self.annotations = defaultdict(lambda: defaultdict(list))
976        for struct_info in self.json['struct_info']:
977            for annotation in struct_info['annotations']:
978                self.annotations[struct_info['name']][annotation['field']].append(annotation)
979
980    def _files_to_parse(self):
981        """Returns a list of files to supply as input to CParser"""
982        # Input paths for CParser are stored in JSON relative to <android_root>/system/chre
983        # Reformulate these to absolute paths, and add in some default includes that we always
984        # supply
985        chre_project_base_dir = system_chre_abs_path()
986        default_includes = ["chpp/api_parser/parser_defines.h",
987                            "chre_api/include/chre_api/chre/version.h"]
988        files = default_includes + self.json['includes'] + [self.json['filename']]
989        return [os.path.join(chre_project_base_dir, file) for file in files]
990
991    def _parse_structs_and_unions(self):
992        # Starting with the root structures (i.e. those that will appear at the top-level in one
993        # or more CHPP messages), build a data structure containing all of the information we'll
994        # need to emit the CHPP structure definition and conversion code.
995        structs_and_unions_to_parse = self.json['root_structs'].copy()
996        while len(structs_and_unions_to_parse) > 0:
997            type_name = structs_and_unions_to_parse.pop()
998            if type_name in self.structs_and_unions:
999                continue
1000
1001            entry = {
1002                'appears_in': set(),  # Other types this type is nested within
1003                'dependencies': set(),  # Types that are nested in this type
1004                'has_vla_member': False,  # True if this type or any dependency has a VLA member
1005                'members': [],  # Info about each member of this type
1006            }
1007            if type_name in self.parser.defs['structs']:
1008                defs = self.parser.defs['structs'][type_name]
1009                entry['is_union'] = False
1010            elif type_name in self.parser.defs['unions']:
1011                defs = self.parser.defs['unions'][type_name]
1012                entry['is_union'] = True
1013            else:
1014                raise RuntimeError("Couldn't find {} in parsed structs/unions".format(type_name))
1015
1016            for member_name, member_type, _ in defs['members']:
1017                member_info = {
1018                    'name': member_name,
1019                    'type': member_type,
1020                    'annotations': self.annotations[type_name][member_name],
1021                    'is_nested_type': False,
1022                }
1023
1024                if member_type.type_spec.startswith('struct ') or \
1025                        member_type.type_spec.startswith('union '):
1026                    member_info['is_nested_type'] = True
1027                    member_type_name = member_type.type_spec.split(' ')[1]
1028                    member_info['nested_type_name'] = member_type_name
1029                    entry['dependencies'].add(member_type_name)
1030                    structs_and_unions_to_parse.append(member_type_name)
1031
1032                entry['members'].append(member_info)
1033
1034                # Flip a flag if this structure has at least one variable-length array member, which
1035                # means that the encoded size can only be computed at runtime
1036                if not entry['has_vla_member']:
1037                    for annotation in self.annotations[type_name][member_name]:
1038                        if annotation['annotation'] == "var_len_array":
1039                            entry['has_vla_member'] = True
1040
1041            self.structs_and_unions[type_name] = entry
1042
1043        # Build reverse linkage of dependency chain (i.e. lookup between a type and the other types
1044        # it appears in)
1045        for type_name, type_info in self.structs_and_unions.items():
1046            for dependency in type_info['dependencies']:
1047                self.structs_and_unions[dependency]['appears_in'].add(type_name)
1048
1049        # Bubble up "has_vla_member" to types each type it appears in, i.e. if this flag is set to
1050        # True on a leaf node, then all its ancestors should also have the flag set to True
1051        for type_name, type_info in self.structs_and_unions.items():
1052            if type_info['has_vla_member']:
1053                types_to_mark = list(type_info['appears_in'])
1054                while len(types_to_mark) > 0:
1055                    type_to_mark = types_to_mark.pop()
1056                    self.structs_and_unions[type_to_mark]['has_vla_member'] = True
1057                    types_to_mark.extend(list(self.structs_and_unions[type_to_mark]['appears_in']))
1058
1059    def _parse_api(self):
1060        file_to_parse = self._files_to_parse()
1061        self.parser = CParser(file_to_parse, cache='parser_cache')
1062        self._parse_structs_and_unions()
1063
1064
1065def run(args):
1066    with open('chre_api_annotations.json') as f:
1067        js = json.load(f)
1068
1069    commit_hash = subprocess.getoutput("git describe --always --long --dirty --exclude '*'")
1070    for file in js:
1071        if args.file_filter:
1072            matched = False
1073            for matcher in args.file_filter:
1074                if matcher in file['filename']:
1075                    matched = True
1076                    break
1077            if not matched:
1078                print("Skipping {} - doesn't match filter(s) {}".format(file['filename'],
1079                                                                        args.file_filter))
1080                continue
1081        print("Parsing {} ... ".format(file['filename']), end='', flush=True)
1082        api_parser = ApiParser(file)
1083        code_gen = CodeGenerator(api_parser, commit_hash)
1084        print("done")
1085        code_gen.generate_header_file(args.dry_run, args.skip_clang_format)
1086        code_gen.generate_conversion_file(args.dry_run, args.skip_clang_format)
1087
1088
1089if __name__ == "__main__":
1090    parser = argparse.ArgumentParser(description='Generate CHPP serialization code from CHRE APIs.')
1091    parser.add_argument('-n', dest='dry_run', action='store_true',
1092                        help='Print the output instead of writing to a file')
1093    parser.add_argument('--skip-clang-format', dest='skip_clang_format', action='store_true',
1094                        help='Skip running clang-format on the output files (doesn\'t apply to dry '
1095                             'runs)')
1096    parser.add_argument('file_filter', nargs='*',
1097                        help='Filters the input files (filename field in the JSON) to generate a '
1098                             'subset of the typical output, e.g. "wifi" to just generate conversion'
1099                             ' routines for wifi.h')
1100    args = parser.parse_args()
1101    run(args)
1102