1#!/usr/bin/python3 2# 3# Copyright (C) 2020 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import argparse 19import json 20import os.path 21import subprocess 22from collections import defaultdict 23from datetime import datetime 24 25from pyclibrary import CParser 26 27LICENSE_HEADER = """/* 28 * Copyright (C) 2020 The Android Open Source Project 29 * 30 * Licensed under the Apache License, Version 2.0 (the "License"); 31 * you may not use this file except in compliance with the License. 32 * You may obtain a copy of the License at 33 * 34 * http://www.apache.org/licenses/LICENSE-2.0 35 * 36 * Unless required by applicable law or agreed to in writing, software 37 * distributed under the License is distributed on an "AS IS" BASIS, 38 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 39 * See the License for the specific language governing permissions and 40 * limitations under the License. 41 */ 42""" 43 44# Paths for output, relative to system/chre 45CHPP_PARSER_INCLUDE_PATH = "chpp/include/chpp/common" 46CHPP_PARSER_SOURCE_PATH = "chpp/common" 47 48 49def system_chre_abs_path(): 50 """Gets the absolute path to the system/chre directory containing this script.""" 51 script_dir = os.path.dirname(os.path.realpath(__file__)) 52 # Assuming we're at system/chre/chpp/api_parser (i.e. up 2 to get to system/chre) 53 chre_project_base_dir = os.path.normpath(script_dir + "/../..") 54 return chre_project_base_dir 55 56 57class CodeGenerator: 58 """Given an ApiParser object, generates a header file with structure definitions in CHPP format. 59 """ 60 61 def __init__(self, api, commit_hash): 62 """ 63 :param api: ApiParser object 64 """ 65 self.api = api 66 self.json = api.json 67 # Turn "chre_api/include/chre_api/chre/wwan.h" into "wwan" 68 self.service_name = self.json['filename'].split('/')[-1].split('.')[0] 69 self.capitalized_service_name = self.service_name[0].upper() + self.service_name[1:] 70 self.commit_hash = commit_hash 71 72 # ---------------------------------------------------------------------------------------------- 73 # Header generation methods (plus some methods shared with encoder generation) 74 # ---------------------------------------------------------------------------------------------- 75 76 def _autogen_notice(self): 77 out = [] 78 out.append("// This file was automatically generated by {}\n".format( 79 os.path.basename(__file__))) 80 out.append("// Date: {} UTC\n".format(datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))) 81 out.append("// Source: {} @ commit {}\n\n".format(self.json['filename'], self.commit_hash)) 82 out.append("// DO NOT modify this file directly, as those changes will be lost the next\n") 83 out.append("// time the script is executed\n\n") 84 return out 85 86 def _dump_to_file(self, output_filename, contents, dry_run, skip_clang_fomat): 87 """Outputs contents to output_filename, or prints contents if dry_run is True""" 88 if dry_run: 89 print("---- {} ----".format(output_filename)) 90 print(contents) 91 print("---- end of {} ----\n".format(output_filename)) 92 else: 93 with open(output_filename, 'w') as f: 94 f.write(contents) 95 96 if not skip_clang_fomat: 97 clang_format_path = os.path.normpath( 98 "../../../../prebuilts/clang/host/linux-x86/clang-stable/bin/clang-format") 99 args = [clang_format_path, '-i', output_filename] 100 result = subprocess.run(args) 101 result.check_returncode() 102 103 def _is_array_type(self, type_info): 104 # If this is an array type, declarators will be a tuple containing a list of 105 # a single int element giving the size of the array 106 return len(type_info.declarators) == 1 and isinstance(type_info.declarators[0], list) 107 108 def _get_array_len(self, type_info): 109 return type_info.declarators[0][0] 110 111 def _get_chpp_type_from_chre(self, chre_type): 112 """Given 'chreWwanCellInfo' returns 'struct ChppWwanCellInfo', etc.""" 113 prefix = self._get_struct_or_union_prefix(chre_type) 114 115 # First see if we have an explicit name override (e.g. for anonymous types) 116 for annotation in self.api.annotations[chre_type]["."]: 117 if annotation['annotation'] == "rename_type": 118 return prefix + annotation['type_override'] 119 120 # Otherwise, use the existing type name, just replace the "chre" prefix with "Chpp" 121 if chre_type.startswith('chre'): 122 return prefix + 'Chpp' + chre_type[4:] 123 else: 124 raise RuntimeError("Couldn't figure out new type name for {}".format(chre_type)) 125 126 def _get_chre_type_with_prefix(self, chre_type): 127 """Given 'chreWwanCellInfo' returns 'struct chreWwanCellInfo', etc.""" 128 return self._get_struct_or_union_prefix(chre_type) + chre_type 129 130 def _get_chpp_header_type_from_chre(self, chre_type): 131 """Given 'chreWwanCellInfo' returns 'struct ChppWwanCellInfoWithHeader', etc.""" 132 return self._get_chpp_type_from_chre(chre_type) + 'WithHeader' 133 134 def _get_member_comment(self, member_info): 135 for annotation in member_info['annotations']: 136 if annotation['annotation'] == "fixed_value": 137 return " // Input ignored; always set to {}".format(annotation['value']) 138 elif annotation['annotation'] == "var_len_array": 139 return " // References {} instances of {}".format( 140 annotation['length_field'], self._get_member_type(member_info)) 141 return "" 142 143 def _get_member_type(self, member_info, underlying_vla_type=False): 144 """Gets the CHPP type specification prefix for a struct/union member. 145 146 :param member_info: a dict element from self.api.structs_and_unions[struct]['members'] 147 :param underlying_vla_type: (used only for var-len array types) False to output 148 'struct ChppOffset', and True to output the type that ChppOffset references 149 :return: type specification string that prefixes the field name, e.g. 'uint8_t' 150 """ 151 # 4 cases to handle: 152 # 1) Annotation gives explicit type that we should use 153 # 2) Annotation says this is a variable length array (so use ChppOffset if 154 # underlying_vla_type is False) 155 # 3) This is a struct/union type, so use the renamed (CHPP) type name 156 # 4) Regular type, e.g. uint32_t, so just use the type spec as-is 157 for annotation in member_info['annotations']: 158 if annotation['annotation'] == "rewrite_type": 159 return annotation['type_override'] 160 elif not underlying_vla_type and annotation['annotation'] in ["var_len_array", "string"]: 161 return "struct ChppOffset" 162 163 if not underlying_vla_type and len(member_info['type'].declarators) > 0 and \ 164 member_info['type'].declarators[0] == "*": 165 # This case should either be handled by rewrite_type (e.g. to uint32_t as 166 # opaque/ignored), or var_len_array 167 raise RuntimeError("Pointer types require annotation\n{}".format( 168 member_info)) 169 170 if member_info['is_nested_type']: 171 return self._get_chpp_type_from_chre(member_info['nested_type_name']) 172 173 return member_info['type'].type_spec 174 175 def _get_member_type_suffix(self, member_info): 176 if self._is_array_type(member_info['type']): 177 return "[{}]".format(self._get_array_len(member_info['type'])) 178 return "" 179 180 def _get_struct_or_union_prefix(self, chre_type): 181 return 'struct ' if not self.api.structs_and_unions[chre_type]['is_union'] else 'union ' 182 183 def _gen_header_includes(self): 184 """Generates #include directives for use in <service>_types.h""" 185 out = ["#include <stdbool.h>\n#include <stddef.h>\n#include <stdint.h>\n\n"] 186 187 includes = ["chpp/app.h", "chpp/macros.h", "chre_api/chre/version.h"] 188 includes.extend(self.json['output_includes']) 189 for incl in sorted(includes): 190 out.append("#include \"{}\"\n".format(incl)) 191 out.append("\n") 192 return out 193 194 def _gen_struct_or_union(self, name): 195 """Generates the definition for a single struct/union type""" 196 out = [] 197 if not name.startswith('anon'): 198 out.append("//! See {{@link {}}} for details\n".format(name)) 199 out.append("{} {{\n".format(self._get_chpp_type_from_chre(name))) 200 for member_info in self.api.structs_and_unions[name]['members']: 201 out.append(" {} {}{};{}\n".format(self._get_member_type(member_info), 202 member_info['name'], 203 self._get_member_type_suffix(member_info), 204 self._get_member_comment(member_info))) 205 206 out.append("} CHPP_PACKED_ATTR;\n\n") 207 return out 208 209 def _gen_header_struct(self, chre_type): 210 """Generates the definition for the type with header (WithHeader)""" 211 out = [] 212 out.append("//! CHPP app header plus {}\n".format( 213 self._get_chpp_header_type_from_chre(chre_type))) 214 215 out.append("{} {{\n".format(self._get_chpp_header_type_from_chre(chre_type))) 216 out.append(" struct ChppAppHeader header;\n") 217 out.append(" {} payload;\n".format(self._get_chpp_type_from_chre(chre_type))) 218 out.append("} CHPP_PACKED_ATTR;\n\n") 219 220 return out 221 222 def _gen_structs_and_unions(self): 223 """Generates definitions for all struct/union types required for the root structs.""" 224 out = [] 225 out.append("CHPP_PACKED_START\n\n") 226 227 sorted_structs = self._sorted_structs(self.json['root_structs']) 228 for type_name in sorted_structs: 229 out.extend(self._gen_struct_or_union(type_name)) 230 231 for chre_type in self.json['root_structs']: 232 out.extend(self._gen_header_struct(chre_type)) 233 234 out.append("CHPP_PACKED_END\n\n") 235 return out 236 237 def _sorted_structs(self, root_nodes): 238 """Implements a topological sort on self.api.structs_and_unions. 239 240 Elements are ordered by definition dependency, i.e. if A includes a field of type B, 241 then B will appear before A in the returned list. 242 :return: list of keys in self.api.structs_and_unions, sorted by dependency order 243 """ 244 result = [] 245 visited = set() 246 247 def sort_helper(collection, key): 248 for dep in sorted(collection[key]['dependencies']): 249 if dep not in visited: 250 visited.add(dep) 251 sort_helper(collection, dep) 252 result.append(key) 253 254 for node in sorted(root_nodes): 255 sort_helper(self.api.structs_and_unions, node) 256 return result 257 258 # ---------------------------------------------------------------------------------------------- 259 # Encoder function generation methods (CHRE --> CHPP) 260 # ---------------------------------------------------------------------------------------------- 261 262 def _get_chpp_member_sizeof_call(self, member_info): 263 """Returns invocation used to determine the size of the provided member when encoded. 264 265 Will be either sizeof(<type in CHPP struct>) or a function call if the member contains a VLA 266 :param member_info: a dict element from self.api.structs_and_unions[struct]['members'] 267 :return: string 268 """ 269 type_name = None 270 if member_info['is_nested_type']: 271 chre_type = member_info['nested_type_name'] 272 if self.api.structs_and_unions[chre_type]['has_vla_member']: 273 return "{}(in->{})".format(self._get_chpp_sizeof_function_name(chre_type), 274 member_info['name']) 275 else: 276 type_name = self._get_chpp_type_from_chre(chre_type) 277 else: 278 type_name = member_info['type'].type_spec 279 return "sizeof({})".format(type_name) 280 281 def _gen_chpp_sizeof_function(self, chre_type): 282 """Generates a function to determine the encoded size of the CHRE struct, if necessary.""" 283 out = [] 284 285 # Note that this function *should* work with unions as well, but at the time of writing 286 # it'll only be used with structs, so names, etc. are written with that in mind 287 struct_info = self.api.structs_and_unions[chre_type] 288 if not struct_info['has_vla_member']: 289 # No codegen necessary, just sizeof on the CHPP structure name is sufficient 290 return out 291 292 core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type) 293 parameter_name = core_type_name[0].lower() + core_type_name[1:] 294 chpp_type_name = self._get_chpp_header_type_from_chre(chre_type) 295 out.append("//! @return number of bytes required to represent the given\n" 296 "//! {} along with the CHPP header as\n" 297 "//! {}\n" 298 .format(chre_type, chpp_type_name)) 299 out.append("static size_t {}(\n const {}{} *{}) {{\n" 300 .format(self._get_chpp_sizeof_function_name(chre_type), 301 self._get_struct_or_union_prefix(chre_type), chre_type, 302 parameter_name)) 303 304 # sizeof(this struct) 305 out.append(" size_t encodedSize = sizeof({});\n".format(chpp_type_name)) 306 307 # Plus count * sizeof(type) for each var-len array included in this struct 308 for member_info in self.api.structs_and_unions[chre_type]['members']: 309 for annotation in member_info['annotations']: 310 if annotation['annotation'] == "var_len_array": 311 # If the VLA field itself contains a VLA, then we'd need to generate a for 312 # loop to calculate the size of each element individually - I don't think we 313 # have any of these in the CHRE API today, so leaving this functionality out. 314 # Also note that to support that case we'd also want to recursively call this 315 # function to generate sizeof functions for nested fields. 316 if member_info['is_nested_type'] and \ 317 self.api.structs_and_unions[member_info['nested_type_name']][ 318 'has_vla_member']: 319 raise RuntimeError( 320 "Nested variable-length arrays is not currently supported ({} " 321 "in {})".format(member_info['name'], chre_type)) 322 323 out.append(" encodedSize += {}->{} * sizeof({});\n".format( 324 parameter_name, annotation['length_field'], 325 self._get_member_type(member_info, True))) 326 elif annotation['annotation'] == "string": 327 out.append(" if ({}->{} != NULL) {{".format( 328 parameter_name, annotation['field'])) 329 out.append(" encodedSize += strlen({}->{}) + 1;\n".format( 330 parameter_name, annotation['field'])) 331 out.append(" }\n") 332 333 out.append(" return encodedSize;\n}\n\n") 334 return out 335 336 def _gen_chpp_sizeof_functions(self): 337 """For each root struct, generate necessary functions to determine their encoded size.""" 338 out = [] 339 for struct in self.json['root_structs']: 340 out.extend(self._gen_chpp_sizeof_function(struct)) 341 return out 342 343 def _gen_conversion_includes(self): 344 """Generates #include directives for the conversion source file""" 345 out = ["#include \"chpp/macros.h\"\n" 346 "#include \"chpp/memory.h\"\n" 347 "#include \"chpp/common/{}_types.h\"\n\n".format(self.service_name)] 348 out.append("#include <stddef.h>\n#include <stdint.h>\n#include <string.h>\n\n") 349 return out 350 351 def _get_chpp_sizeof_function_name(self, chre_struct): 352 """Function name used to compute the encoded size of the given struct at runtime""" 353 core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_struct) 354 return "chpp{}SizeOf{}FromChre".format(self.capitalized_service_name, core_type_name) 355 356 def _get_encoding_function_name(self, chre_type): 357 core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type) 358 return "chpp{}Convert{}FromChre".format(self.capitalized_service_name, core_type_name) 359 360 def _gen_encoding_function_signature(self, chre_type): 361 out = [] 362 out.append("void {}(\n".format(self._get_encoding_function_name(chre_type))) 363 out.append(" const {}{} *in,\n".format( 364 self._get_struct_or_union_prefix(chre_type), chre_type)) 365 out.append(" {} *out".format(self._get_chpp_type_from_chre(chre_type))) 366 if self.api.structs_and_unions[chre_type]['has_vla_member']: 367 out.append(",\n") 368 out.append(" uint8_t *payload,\n") 369 out.append(" size_t payloadSize,\n") 370 out.append(" uint16_t *vlaOffset") 371 out.append(")") 372 return out 373 374 def _gen_string_encoding(self, member_info, annotation): 375 out = [] 376 # Might want to revisit this if we ever end up supporting NULL strings 377 # in our API. We can assert here since there's currently no API that 378 # does so. 379 member_name = member_info['name'] 380 out.append(" if (in->{} != NULL) {{\n".format(member_name)) 381 out.append(" size_t strSize = strlen(in->{}) + 1;\n".format(member_name)) 382 out.append(" memcpy(&payload[*vlaOffset], in->{}, strSize);\n".format( 383 member_name)) 384 out.append(" out->{}.length = (uint16_t)(strSize);\n".format( 385 member_name)) 386 out.append(" out->{}.offset = *vlaOffset;\n".format(member_name)) 387 out.append(" *vlaOffset += out->{}.length;\n".format(member_name)) 388 out.append(" } else {\n") 389 out.append(" out->{}.length = 0;\n".format(member_name)) 390 out.append(" out->{}.offset = 0;\n".format(member_name)) 391 out.append(" }\n\n") 392 393 return out 394 395 def _gen_vla_encoding(self, member_info, annotation): 396 out = [] 397 398 variable_name = member_info['name'] 399 chpp_type = self._get_member_type(member_info, True) 400 401 if member_info['is_nested_type']: 402 out.append("\n {} *{} = ({} *) &payload[*vlaOffset];\n".format( 403 chpp_type, variable_name, chpp_type)) 404 405 out.append(" out->{}.length = (uint16_t)(in->{} * {});\n".format( 406 member_info['name'], annotation['length_field'], 407 self._get_chpp_member_sizeof_call(member_info))) 408 409 out.append(" CHPP_ASSERT((size_t)(*vlaOffset + out->{}.length) <= payloadSize);\n".format( 410 member_info['name'])) 411 412 out.append(" if (out->{}.length > 0 &&\n" 413 " *vlaOffset + out->{}.length <= payloadSize) {{\n".format( 414 member_info['name'], member_info['name'])) 415 416 if member_info['is_nested_type']: 417 out.append(" for (size_t i = 0; i < in->{}; i++) {{\n".format( 418 annotation['length_field'])) 419 out.append(" {}".format( 420 self._get_assignment_statement_for_field(member_info, in_vla_loop=True))) 421 out.append(" }\n") 422 else: 423 out.append("memcpy(&payload[*vlaOffset], in->{}, in->{} * sizeof({}));\n".format( 424 member_info['name'], annotation['length_field'], chpp_type)) 425 426 out.append(" out->{}.offset = *vlaOffset;\n".format(member_info['name'])) 427 out.append(" *vlaOffset += out->{}.length;\n".format(member_info['name'])) 428 429 out.append(" } else {\n") 430 out.append(" out->{}.offset = 0;\n".format(member_info['name'])) 431 out.append(" }\n"); 432 433 return out 434 435 # ---------------------------------------------------------------------------------------------- 436 # Encoder / decoder function generation methods (CHRE <--> CHPP) 437 # ---------------------------------------------------------------------------------------------- 438 439 def _get_assignment_statement_for_field(self, member_info, 440 in_vla_loop=False, 441 containing_field_name=None, 442 decode_mode=False): 443 """Returns a statement to assign the provided member 444 445 :param member_info: 446 :param in_vla_loop: True if we're currently inside a loop and should append [i] 447 :param containing_field_name: Additional member name to use to access the target field, or 448 None; for example the normal case is "out->field = in->field", but if we're generating 449 assignments in the parent conversion function (e.g. as used for union variants), we need to 450 do "out->nested_field.field = in->nested_field.field" 451 :param decode_mode: True converts from CHPP to CHRE. False from CHRE to CHPP 452 :return: assignment statement as a string 453 """ 454 array_index = "[i]" if in_vla_loop else "" 455 output_accessor = "" if in_vla_loop else "out->" 456 containing_field = containing_field_name + "." if containing_field_name is not None else "" 457 458 output_variable = "{}{}{}{}".format(output_accessor, containing_field, member_info['name'], 459 array_index) 460 input_variable = "in->{}{}{}".format(containing_field, member_info['name'], array_index) 461 462 if decode_mode and in_vla_loop: 463 output_variable = "{}Out{}".format(member_info['name'], array_index) 464 input_variable = "{}In{}".format(member_info['name'], array_index) 465 466 if member_info['is_nested_type']: 467 chre_type = member_info['nested_type_name'] 468 has_vla_member = self.api.structs_and_unions[chre_type]['has_vla_member'] 469 if decode_mode: 470 # Use decoding function 471 vla_params = ", inSize" if has_vla_member else "" 472 out = "if (!{}(&{}, &{}{})) {{\n".format( 473 self._get_decoding_function_name(chre_type), input_variable, 474 output_variable, vla_params) 475 if has_vla_member: 476 out += " CHPP_FREE_AND_NULLIFY({}Out);\n".format(member_info['name']) 477 out += " return false;\n" 478 out += "}\n" 479 return out 480 else: 481 # Use encoding function 482 vla_params = ", payload, payloadSize, vlaOffset" if has_vla_member else "" 483 return "{}(&{}, &{}{});\n".format( 484 self._get_encoding_function_name(chre_type), input_variable, output_variable, 485 vla_params) 486 elif self._is_array_type(member_info['type']): 487 # Array of primitive type (e.g. uint32_t[8]) - use memcpy 488 return "memcpy({}, {}, sizeof({}));\n".format(output_variable, input_variable, 489 output_variable) 490 else: 491 # Regular assignment 492 return "{} = {};\n".format(output_variable, input_variable) 493 494 def _gen_union_variant_conversion_code(self, member_info, annotation, decode_mode): 495 """Generates a switch statement to encode the "active"/"used" field within a union. 496 497 Handles cases where a union has multiple types, but there's another peer/adjacent field 498 which tells you which field in the union is to be used. Outputs code like this: 499 switch (in->{discriminator field}) { 500 case {first discriminator value associated with a fields}: 501 {conversion code for the field associated with this discriminator value} 502 ... 503 :param chre_type: CHRE type of the union 504 :param annotation: Reference to JSON annotation data with the discriminator mapping data 505 :param decode_mode: False encodes from CHRE to CHPP. True decodes from CHPP to CHRE 506 :return: list of strings 507 """ 508 out = [] 509 chre_type = member_info['nested_type_name'] 510 struct_info = self.api.structs_and_unions[chre_type] 511 512 # Start off by zeroing out the union field so any padding is set to a consistent value 513 out.append(" memset(&out->{}, 0, sizeof(out->{}));\n".format(member_info['name'], 514 member_info['name'])) 515 516 # Next, generate the switch statement that will copy over the proper values 517 out.append(" switch (in->{}) {{\n".format(annotation['discriminator'])) 518 for value, field_name in annotation['mapping']: 519 out.append(" case {}:\n".format(value)) 520 521 found = False 522 for nested_member_info in struct_info['members']: 523 if nested_member_info['name'] == field_name: 524 out.append(" {}".format( 525 self._get_assignment_statement_for_field( 526 nested_member_info, 527 containing_field_name=member_info['name'], 528 decode_mode=decode_mode))) 529 found = True 530 break 531 532 if not found: 533 raise RuntimeError("Invalid mapping - couldn't find target field {} in struct {}" 534 .format(field_name, chre_type)) 535 536 out.append(" break;\n") 537 538 out.append(" default:\n" 539 " CHPP_ASSERT(false);\n" 540 " }\n") 541 542 return out 543 544 def _gen_conversion_function(self, chre_type, already_generated, decode_mode): 545 out = [] 546 547 struct_info = self.api.structs_and_unions[chre_type] 548 for dependency in sorted(struct_info['dependencies']): 549 if dependency not in already_generated: 550 out.extend( 551 self._gen_conversion_function(dependency, already_generated, decode_mode)) 552 553 # Skip if we've already generated code for this type, or if it's a union (in which case we 554 # handle the assignment in the parent structure to enable support for discrimination of 555 # which field in the union to use) 556 if chre_type in already_generated or struct_info['is_union']: 557 return out 558 already_generated.add(chre_type) 559 560 out.append("static ") 561 if decode_mode: 562 out.extend(self._gen_decoding_function_signature(chre_type)) 563 else: 564 out.extend(self._gen_encoding_function_signature(chre_type)) 565 out.append(" {\n") 566 567 for member_info in self.api.structs_and_unions[chre_type]['members']: 568 generated_by_annotation = False 569 for annotation in member_info['annotations']: 570 if annotation['annotation'] == "fixed_value": 571 if self._is_array_type(member_info['type']): 572 out.append(" memset(&out->{}, {}, sizeof(out->{}));\n".format( 573 member_info['name'], annotation['value'], member_info['name'])) 574 else: 575 out.append(" out->{} = {};\n".format(member_info['name'], 576 annotation['value'])) 577 generated_by_annotation = True 578 break 579 elif annotation['annotation'] == "enum": 580 # TODO: generate range verification code? 581 pass 582 elif annotation['annotation'] == "var_len_array": 583 if decode_mode: 584 out.extend(self._gen_vla_decoding(member_info, annotation)) 585 else: 586 out.extend(self._gen_vla_encoding(member_info, annotation)) 587 generated_by_annotation = True 588 break 589 elif annotation['annotation'] == "string": 590 if decode_mode: 591 out.extend(self._gen_string_decoding(member_info, annotation)) 592 else: 593 out.extend(self._gen_string_encoding(member_info, annotation)) 594 generated_by_annotation = True 595 break 596 elif annotation['annotation'] == "union_variant": 597 out.extend(self._gen_union_variant_conversion_code( 598 member_info, annotation, decode_mode)) 599 generated_by_annotation = True 600 break 601 602 if not generated_by_annotation: 603 out.append(" {}".format( 604 self._get_assignment_statement_for_field(member_info, decode_mode=decode_mode))) 605 606 if decode_mode: 607 out.append("\n return true;\n") 608 609 out.append("}\n\n") 610 return out 611 612 def _gen_conversion_functions(self, decode_mode): 613 out = [] 614 already_generated = set() 615 for struct in self.json['root_structs']: 616 out.extend(self._gen_conversion_function(struct, already_generated, decode_mode)) 617 return out 618 619 def _strip_prefix_and_service_from_chre_struct_name(self, struct): 620 """Strip 'chre' and service prefix, e.g. 'chreWwanCellResultInfo' -> 'CellResultInfo'""" 621 chre_stripped = struct[4:] 622 upcased_service_name = self.service_name[0].upper() + self.service_name[1:] 623 if not struct.startswith('chre') or not chre_stripped.startswith(upcased_service_name): 624 # If this happens, we need to update the script to handle it. Right we assume struct 625 # naming follows the pattern "chre<Service_name><Thing_name>" 626 raise RuntimeError("Unexpected structure name {}".format(struct)) 627 628 return chre_stripped[len(self.service_name):] 629 630 # ---------------------------------------------------------------------------------------------- 631 # Memory allocation generation methods 632 # ---------------------------------------------------------------------------------------------- 633 634 def _get_chpp_sizeof_call(self, chre_type): 635 """Returns invocation used to determine the size of the provided CHRE struct (with the CHPP 636 app header) after encoding. 637 638 Like _get_chpp_member_sizeof_call(), except for a top-level type assigned to the variable 639 "in" rather than a member within a structure (e.g. a VLA field) 640 :param chre_type: CHRE type name 641 :return: string 642 """ 643 if self.api.structs_and_unions[chre_type]['has_vla_member']: 644 return "{}(in)".format(self._get_chpp_sizeof_function_name(chre_type)) 645 else: 646 return "sizeof({})".format(self._get_chpp_header_type_from_chre(chre_type)) 647 648 def _get_encode_allocation_function_name(self, chre_type): 649 core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type) 650 return "chpp{}{}FromChre".format(self.capitalized_service_name, core_type_name) 651 652 def _gen_encode_allocation_function_signature(self, chre_type, gen_docs=False): 653 out = [] 654 if gen_docs: 655 out.append("/**\n" 656 " * Converts from given CHRE structure to serialized CHPP type.\n" 657 " *\n" 658 " * @param in Fully-formed CHRE structure.\n" 659 " * @param out Upon success, will point to a buffer allocated with " 660 "chppMalloc().\n" 661 " * It is the responsibility of the caller to set the values of the CHPP " 662 "app layer header, and to free the buffer when it is no longer needed via " 663 "chppFree() or CHPP_FREE_AND_NULLIFY().\n" 664 " * @param outSize Upon success, will be set to the size of the output " 665 "buffer, in bytes.\n" 666 " *\n" 667 " * @return true on success, false if memory allocation failed.\n" 668 " */\n") 669 out.append("bool {}(\n".format(self._get_encode_allocation_function_name(chre_type))) 670 out.append(" const {}{} *in,\n".format( 671 self._get_struct_or_union_prefix(chre_type), chre_type)) 672 out.append(" {} **out,\n".format(self._get_chpp_header_type_from_chre(chre_type))) 673 out.append(" size_t *outSize)") 674 return out 675 676 def _gen_encode_allocation_function(self, chre_type): 677 out = [] 678 out.extend(self._gen_encode_allocation_function_signature(chre_type)) 679 out.append(" {\n") 680 out.append(" CHPP_NOT_NULL(out);\n") 681 out.append(" CHPP_NOT_NULL(outSize);\n\n") 682 out.append(" size_t payloadSize = {};\n".format( 683 self._get_chpp_sizeof_call(chre_type))) 684 out.append(" *out = chppMalloc(payloadSize);\n") 685 686 out.append(" if (*out != NULL) {\n") 687 688 struct_info = self.api.structs_and_unions[chre_type] 689 if struct_info['has_vla_member']: 690 out.append(" uint8_t *payload = (uint8_t *) &(*out)->payload;\n") 691 out.append(" uint16_t vlaOffset = sizeof({});\n".format( 692 self._get_chpp_type_from_chre(chre_type))) 693 694 out.append(" {}(in, &(*out)->payload".format( 695 self._get_encoding_function_name(chre_type))) 696 if struct_info['has_vla_member']: 697 out.append(", payload, payloadSize, &vlaOffset") 698 out.append(");\n") 699 out.append(" *outSize = payloadSize;\n") 700 out.append(" return true;\n") 701 out.append(" }\n") 702 703 out.append(" return false;\n}\n\n") 704 return out 705 706 def _gen_encode_allocation_functions(self): 707 out = [] 708 for chre_type in self.json['root_structs']: 709 out.extend(self._gen_encode_allocation_function(chre_type)) 710 return out 711 712 def _gen_encode_allocation_function_signatures(self): 713 out = [] 714 for chre_type in self.json['root_structs']: 715 out.extend(self._gen_encode_allocation_function_signature(chre_type, True)) 716 out.append(";\n\n") 717 return out 718 719 # ---------------------------------------------------------------------------------------------- 720 # Decoder function generation methods (CHPP --> CHRE) 721 # ---------------------------------------------------------------------------------------------- 722 723 def _get_decoding_function_name(self, chre_type): 724 core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type) 725 return "chpp{}Convert{}ToChre".format(self.capitalized_service_name, core_type_name) 726 727 def _gen_decoding_function_signature(self, chre_type): 728 out = [] 729 out.append("bool {}(\n".format(self._get_decoding_function_name(chre_type))) 730 out.append(" const {} *in,\n".format(self._get_chpp_type_from_chre(chre_type))) 731 out.append(" {} *out".format(self._get_chre_type_with_prefix(chre_type))) 732 if self.api.structs_and_unions[chre_type]['has_vla_member']: 733 out.append(",\n") 734 out.append(" size_t inSize") 735 out.append(")") 736 return out 737 738 def _gen_string_decoding(self, member_info, annotation): 739 out = [] 740 variable_name = member_info['name'] 741 out.append("\n") 742 out.append(" if (in->{}.length == 0) {{\n".format(variable_name)) 743 out.append(" out->{} = NULL;\n".format(variable_name)) 744 out.append(" } else {\n") 745 out.append(" char *{}Out = chppMalloc(in->{}.length);\n".format( 746 variable_name, variable_name)) 747 out.append(" if ({}Out == NULL) {{\n".format(variable_name)) 748 out.append(" return false;\n") 749 out.append(" }\n\n") 750 out.append(" memcpy({}Out, &((const uint8_t *)in)[in->{}.offset],\n".format( 751 variable_name, variable_name)) 752 out.append(" in->{}.length);\n".format(variable_name)) 753 out.append(" out->{} = {}Out;\n".format(variable_name, variable_name)) 754 out.append(" }\n") 755 756 return out 757 758 759 def _gen_vla_decoding(self, member_info, annotation): 760 out = [] 761 762 variable_name = member_info['name'] 763 chpp_type = self._get_member_type(member_info, True) 764 if member_info['is_nested_type']: 765 chre_type = self._get_chre_type_with_prefix(member_info['nested_type_name']) 766 else: 767 chre_type = chpp_type 768 769 out.append("\n") 770 out.append(" if (in->{}.length == 0) {{\n".format(variable_name)) 771 out.append(" out->{} = NULL;\n".format(variable_name)) 772 out.append(" }\n") 773 out.append(" else {\n") 774 out.append(" if (in->{}.offset + in->{}.length > inSize ||\n".format( 775 variable_name, variable_name)) 776 out.append(" in->{}.length != in->{} * sizeof({})) {{\n".format( 777 variable_name, annotation['length_field'], chpp_type)) 778 779 out.append(" return false;\n") 780 out.append(" }\n\n") 781 782 if member_info['is_nested_type']: 783 out.append(" const {} *{}In =\n".format(chpp_type, variable_name)) 784 out.append(" (const {} *) &((const uint8_t *)in)[in->{}.offset];\n\n".format( 785 chpp_type, variable_name)) 786 787 out.append(" {} *{}Out = chppMalloc(in->{} * sizeof({}));\n".format( 788 chre_type, variable_name, annotation['length_field'], chre_type)) 789 out.append(" if ({}Out == NULL) {{\n".format(variable_name)) 790 out.append(" return false;\n") 791 out.append(" }\n\n") 792 793 if member_info['is_nested_type']: 794 out.append(" for (size_t i = 0; i < in->{}; i++) {{\n".format( 795 annotation['length_field'], variable_name)) 796 out.append(" {}".format(self._get_assignment_statement_for_field( 797 member_info, in_vla_loop=True, decode_mode=True))) 798 out.append(" }\n") 799 else: 800 out.append(" memcpy({}Out, &((const uint8_t *)in)[in->{}.offset],\n".format( 801 variable_name, variable_name)) 802 out.append(" in->{} * sizeof({}));\n".format( 803 annotation['length_field'], chre_type)) 804 805 out.append(" out->{} = {}Out;\n".format(variable_name, variable_name)) 806 out.append(" }\n\n") 807 808 return out 809 810 def _get_decode_allocation_function_name(self, chre_type): 811 core_type_name = self._strip_prefix_and_service_from_chre_struct_name(chre_type) 812 return "chpp{}{}ToChre".format(self.capitalized_service_name, core_type_name) 813 814 def _gen_decode_allocation_function_signature(self, chre_type, gen_docs=False): 815 out = [] 816 if gen_docs: 817 out.append("/**\n" 818 " * Converts from serialized CHPP structure to a CHRE type.\n" 819 " *\n" 820 " * @param in Fully-formed CHPP structure.\n" 821 " * @param in Size of the CHPP structure in bytes.\n" 822 " *\n" 823 " * @return If successful, a pointer to a CHRE structure allocated with " 824 "chppMalloc(). If unsuccessful, null.\n" 825 " * It is the responsibility of the caller to free the buffer when it is no " 826 "longer needed via chppFree() or CHPP_FREE_AND_NULLIFY().\n" 827 " */\n") 828 829 out.append("{} *{}(\n".format( 830 self._get_chre_type_with_prefix(chre_type), 831 self._get_decode_allocation_function_name(chre_type))) 832 out.append(" const {} *in,\n".format(self._get_chpp_type_from_chre(chre_type))) 833 out.append(" size_t inSize)") 834 return out 835 836 def _gen_decode_allocation_function(self, chre_type): 837 out = [] 838 839 out.extend(self._gen_decode_allocation_function_signature(chre_type)) 840 out.append(" {\n") 841 842 out.append(" {} *out = NULL;\n\n".format( 843 self._get_chre_type_with_prefix(chre_type))) 844 845 out.append(" if (inSize >= sizeof({})) {{\n".format( 846 self._get_chpp_type_from_chre(chre_type))) 847 848 out.append(" out = chppMalloc(sizeof({}));\n".format( 849 self._get_chre_type_with_prefix(chre_type))) 850 out.append(" if (out != NULL) {\n") 851 852 struct_info = self.api.structs_and_unions[chre_type] 853 854 out.append(" if (!{}(in, out".format(self._get_decoding_function_name(chre_type))) 855 if struct_info['has_vla_member']: 856 out.append(", inSize") 857 out.append(")) {") 858 out.append(" CHPP_FREE_AND_NULLIFY(out);\n") 859 out.append(" }\n") 860 861 out.append(" }\n") 862 out.append(" }\n\n") 863 out.append(" return out;\n") 864 out.append("}\n") 865 return out 866 867 def _gen_decode_allocation_functions(self): 868 out = [] 869 for chre_type in self.json['root_structs']: 870 out.extend(self._gen_decode_allocation_function(chre_type)) 871 return out 872 873 def _gen_decode_allocation_function_signatures(self): 874 out = [] 875 for chre_type in self.json['root_structs']: 876 out.extend(self._gen_decode_allocation_function_signature(chre_type, True)) 877 out.append(";\n\n") 878 return out 879 880 # ---------------------------------------------------------------------------------------------- 881 # Public methods 882 # ---------------------------------------------------------------------------------------------- 883 884 def generate_header_file(self, dry_run=False, skip_clang_format=False): 885 """Creates a C header file for this API and writes it to the file indicated in the JSON.""" 886 filename = self.service_name + "_types.h" 887 if not dry_run: 888 print("Generating {} ... ".format(filename), end='', flush=True) 889 output_file = os.path.join(system_chre_abs_path(), CHPP_PARSER_INCLUDE_PATH, filename) 890 header = self.generate_header_string() 891 self._dump_to_file(output_file, header, dry_run, skip_clang_format) 892 if not dry_run: 893 print("done") 894 895 def generate_header_string(self): 896 """Returns a C header with structure definitions for this API.""" 897 # To defer concatenation (speed things up), build the file as a list of strings then only 898 # concatenate once at the end 899 out = [LICENSE_HEADER] 900 901 header_guard = "CHPP_{}_TYPES_H_".format(self.service_name.upper()) 902 903 out.append("#ifndef {}\n#define {}\n\n".format(header_guard, header_guard)) 904 out.extend(self._autogen_notice()) 905 out.extend(self._gen_header_includes()) 906 out.append("#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n") 907 out.extend(self._gen_structs_and_unions()) 908 909 out.append("\n// Encoding functions (CHRE --> CHPP)\n\n") 910 out.extend(self._gen_encode_allocation_function_signatures()) 911 912 out.append("\n// Decoding functions (CHPP --> CHRE)\n\n") 913 out.extend(self._gen_decode_allocation_function_signatures()) 914 915 out.append("#ifdef __cplusplus\n}\n#endif\n\n") 916 out.append("#endif // {}\n".format(header_guard)) 917 return ''.join(out) 918 919 def generate_conversion_file(self, dry_run=False, skip_clang_format=False): 920 """Generates a .c file with functions for encoding CHRE structs into CHPP and vice versa.""" 921 filename = self.service_name + "_convert.c" 922 if not dry_run: 923 print("Generating {} ... ".format(filename), end='', flush=True) 924 contents = self.generate_conversion_string() 925 output_file = os.path.join(system_chre_abs_path(), CHPP_PARSER_SOURCE_PATH, filename) 926 self._dump_to_file(output_file, contents, dry_run, skip_clang_format) 927 if not dry_run: 928 print("done") 929 930 def generate_conversion_string(self): 931 """Returns C code for encoding CHRE structs into CHPP and vice versa.""" 932 out = [LICENSE_HEADER, "\n"] 933 934 out.extend(self._autogen_notice()) 935 out.extend(self._gen_conversion_includes()) 936 937 out.append("\n// Encoding (CHRE --> CHPP) size functions\n\n") 938 out.extend(self._gen_chpp_sizeof_functions()) 939 out.append("\n// Encoding (CHRE --> CHPP) conversion functions\n\n") 940 out.extend(self._gen_conversion_functions(False)) 941 out.append("\n// Encoding (CHRE --> CHPP) top-level functions\n\n") 942 out.extend(self._gen_encode_allocation_functions()) 943 944 out.append("\n// Decoding (CHPP --> CHRE) conversion functions\n\n") 945 out.extend(self._gen_conversion_functions(True)) 946 out.append("\n// Decoding (CHPP --> CHRE) top-level functions\n\n") 947 out.extend(self._gen_decode_allocation_functions()) 948 949 return ''.join(out) 950 951 952class ApiParser: 953 """Given a file-specific set of annotations (extracted from JSON annotations file), parses a 954 single API header file into data structures suitable for use with code generation. 955 """ 956 957 def __init__(self, json_obj): 958 """Initialize and parse the API file described in the provided JSON-derived object. 959 960 :param json_obj: Extracted file-specific annotations from JSON 961 """ 962 self.json = json_obj 963 self.structs_and_unions = {} 964 self._parse_annotations() 965 self._parse_api() 966 967 def _parse_annotations(self): 968 # Convert annotations list to a more usable data structure: dict keyed by structure name, 969 # containing a dict keyed by field name, containing a list of annotations (as they 970 # appear in the JSON). In other words, we can easily get all of the annotations for the 971 # "version" field in "chreWwanCellInfoResult" via 972 # annotations['chreWwanCellInfoResult']['version']. This is also a defaultdict, so it's safe 973 # to access if there are no annotations for this structure + field; it'll just give you 974 # an empty list in that case. 975 self.annotations = defaultdict(lambda: defaultdict(list)) 976 for struct_info in self.json['struct_info']: 977 for annotation in struct_info['annotations']: 978 self.annotations[struct_info['name']][annotation['field']].append(annotation) 979 980 def _files_to_parse(self): 981 """Returns a list of files to supply as input to CParser""" 982 # Input paths for CParser are stored in JSON relative to <android_root>/system/chre 983 # Reformulate these to absolute paths, and add in some default includes that we always 984 # supply 985 chre_project_base_dir = system_chre_abs_path() 986 default_includes = ["chpp/api_parser/parser_defines.h", 987 "chre_api/include/chre_api/chre/version.h"] 988 files = default_includes + self.json['includes'] + [self.json['filename']] 989 return [os.path.join(chre_project_base_dir, file) for file in files] 990 991 def _parse_structs_and_unions(self): 992 # Starting with the root structures (i.e. those that will appear at the top-level in one 993 # or more CHPP messages), build a data structure containing all of the information we'll 994 # need to emit the CHPP structure definition and conversion code. 995 structs_and_unions_to_parse = self.json['root_structs'].copy() 996 while len(structs_and_unions_to_parse) > 0: 997 type_name = structs_and_unions_to_parse.pop() 998 if type_name in self.structs_and_unions: 999 continue 1000 1001 entry = { 1002 'appears_in': set(), # Other types this type is nested within 1003 'dependencies': set(), # Types that are nested in this type 1004 'has_vla_member': False, # True if this type or any dependency has a VLA member 1005 'members': [], # Info about each member of this type 1006 } 1007 if type_name in self.parser.defs['structs']: 1008 defs = self.parser.defs['structs'][type_name] 1009 entry['is_union'] = False 1010 elif type_name in self.parser.defs['unions']: 1011 defs = self.parser.defs['unions'][type_name] 1012 entry['is_union'] = True 1013 else: 1014 raise RuntimeError("Couldn't find {} in parsed structs/unions".format(type_name)) 1015 1016 for member_name, member_type, _ in defs['members']: 1017 member_info = { 1018 'name': member_name, 1019 'type': member_type, 1020 'annotations': self.annotations[type_name][member_name], 1021 'is_nested_type': False, 1022 } 1023 1024 if member_type.type_spec.startswith('struct ') or \ 1025 member_type.type_spec.startswith('union '): 1026 member_info['is_nested_type'] = True 1027 member_type_name = member_type.type_spec.split(' ')[1] 1028 member_info['nested_type_name'] = member_type_name 1029 entry['dependencies'].add(member_type_name) 1030 structs_and_unions_to_parse.append(member_type_name) 1031 1032 entry['members'].append(member_info) 1033 1034 # Flip a flag if this structure has at least one variable-length array member, which 1035 # means that the encoded size can only be computed at runtime 1036 if not entry['has_vla_member']: 1037 for annotation in self.annotations[type_name][member_name]: 1038 if annotation['annotation'] == "var_len_array": 1039 entry['has_vla_member'] = True 1040 1041 self.structs_and_unions[type_name] = entry 1042 1043 # Build reverse linkage of dependency chain (i.e. lookup between a type and the other types 1044 # it appears in) 1045 for type_name, type_info in self.structs_and_unions.items(): 1046 for dependency in type_info['dependencies']: 1047 self.structs_and_unions[dependency]['appears_in'].add(type_name) 1048 1049 # Bubble up "has_vla_member" to types each type it appears in, i.e. if this flag is set to 1050 # True on a leaf node, then all its ancestors should also have the flag set to True 1051 for type_name, type_info in self.structs_and_unions.items(): 1052 if type_info['has_vla_member']: 1053 types_to_mark = list(type_info['appears_in']) 1054 while len(types_to_mark) > 0: 1055 type_to_mark = types_to_mark.pop() 1056 self.structs_and_unions[type_to_mark]['has_vla_member'] = True 1057 types_to_mark.extend(list(self.structs_and_unions[type_to_mark]['appears_in'])) 1058 1059 def _parse_api(self): 1060 file_to_parse = self._files_to_parse() 1061 self.parser = CParser(file_to_parse, cache='parser_cache') 1062 self._parse_structs_and_unions() 1063 1064 1065def run(args): 1066 with open('chre_api_annotations.json') as f: 1067 js = json.load(f) 1068 1069 commit_hash = subprocess.getoutput("git describe --always --long --dirty --exclude '*'") 1070 for file in js: 1071 if args.file_filter: 1072 matched = False 1073 for matcher in args.file_filter: 1074 if matcher in file['filename']: 1075 matched = True 1076 break 1077 if not matched: 1078 print("Skipping {} - doesn't match filter(s) {}".format(file['filename'], 1079 args.file_filter)) 1080 continue 1081 print("Parsing {} ... ".format(file['filename']), end='', flush=True) 1082 api_parser = ApiParser(file) 1083 code_gen = CodeGenerator(api_parser, commit_hash) 1084 print("done") 1085 code_gen.generate_header_file(args.dry_run, args.skip_clang_format) 1086 code_gen.generate_conversion_file(args.dry_run, args.skip_clang_format) 1087 1088 1089if __name__ == "__main__": 1090 parser = argparse.ArgumentParser(description='Generate CHPP serialization code from CHRE APIs.') 1091 parser.add_argument('-n', dest='dry_run', action='store_true', 1092 help='Print the output instead of writing to a file') 1093 parser.add_argument('--skip-clang-format', dest='skip_clang_format', action='store_true', 1094 help='Skip running clang-format on the output files (doesn\'t apply to dry ' 1095 'runs)') 1096 parser.add_argument('file_filter', nargs='*', 1097 help='Filters the input files (filename field in the JSON) to generate a ' 1098 'subset of the typical output, e.g. "wifi" to just generate conversion' 1099 ' routines for wifi.h') 1100 args = parser.parse_args() 1101 run(args) 1102