1#!/usr/bin/env python3 2# 3# Copyright 2021 Google Inc. All rights reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import filecmp 18import glob 19import shutil 20import subprocess 21import generate_grpc_examples 22from pathlib import Path 23from util import flatc, root_path, tests_path, args, flatc_path 24 25# Specify the other paths that will be referenced 26swift_code_gen = Path(root_path, "tests/swift/tests/CodeGenerationTests") 27ts_code_gen = Path(root_path, "tests/ts") 28samples_path = Path(root_path, "samples") 29reflection_path = Path(root_path, "reflection") 30 31 32# Generate the code for flatbuffers reflection schema 33def flatc_reflection(options, location, target): 34 full_options = ["--no-prefix"] + options 35 temp_dir = ".tmp" 36 flatc( 37 full_options, 38 prefix=temp_dir, 39 schema="reflection.fbs", 40 cwd=reflection_path, 41 ) 42 new_reflection_path = Path(reflection_path, temp_dir, target) 43 original_reflection_path = Path(root_path, location, target) 44 if not filecmp.cmp(str(new_reflection_path), str(original_reflection_path)): 45 shutil.rmtree(str(original_reflection_path), ignore_errors=True) 46 shutil.move(str(new_reflection_path), str(original_reflection_path)) 47 shutil.rmtree(str(Path(reflection_path, temp_dir))) 48 49 50def flatc_annotate(schema, file, include=None, cwd=tests_path): 51 cmd = [str(flatc_path)] 52 if include: 53 cmd += ["-I"] + [include] 54 cmd += ["--annotate", schema, file] 55 result = subprocess.run(cmd, cwd=str(cwd), check=True) 56 57 58# Glob a pattern relative to file path 59def glob(path, pattern): 60 return [str(p) for p in path.glob(pattern)] 61 62 63# flatc options that are shared 64BASE_OPTS = ["--reflect-names", "--gen-mutable", "--gen-object-api"] 65NO_INCL_OPTS = BASE_OPTS + ["--no-includes"] 66 67# Language specific options 68CS_OPTS = ["--csharp", "--cs-gen-json-serializer"] 69CPP_OPTS = [ 70 "--cpp", 71 "--gen-compare", 72] + (["--cpp-std", "c++0x"] if args.cpp_0x else []) 73 74CPP_17_OPTS = NO_INCL_OPTS + [ 75 "--cpp", 76 "--cpp-std", 77 "c++17", 78 "--cpp-static-reflection", 79 "--gen-object-api", 80] 81RUST_OPTS = BASE_OPTS + [ 82 "--rust", 83 "--gen-all", 84 "--gen-name-strings", 85 "--rust-module-root-file", 86] 87RUST_SERIALIZE_OPTS = BASE_OPTS + [ 88 "--rust", 89 "--gen-all", 90 "--gen-name-strings", 91 "--rust-serialize", 92 "--rust-module-root-file", 93] 94TS_OPTS = ["--ts", "--gen-name-strings"] 95LOBSTER_OPTS = ["--lobster"] 96SWIFT_OPTS = ["--swift", "--gen-json-emit", "--bfbs-filenames", str(tests_path)] 97SWIFT_OPTS_CODE_GEN = [ 98 "--swift", 99 "--gen-json-emit", 100 "--bfbs-filenames", 101 str(swift_code_gen), 102] 103JAVA_OPTS = ["--java"] 104KOTLIN_OPTS = ["--kotlin"] 105PHP_OPTS = ["--php"] 106DART_OPTS = ["--dart"] 107PYTHON_OPTS = ["--python", "--python-typing"] 108BINARY_OPTS = ["-b", "--schema", "--bfbs-comments", "--bfbs-builtins"] 109PROTO_OPTS = ["--proto"] 110 111# Basic Usage 112 113flatc( 114 NO_INCL_OPTS 115 + CPP_OPTS 116 + CS_OPTS 117 + [ 118 "--binary", 119 "--java", 120 "--kotlin", 121 "--dart", 122 "--go", 123 "--lobster", 124 "--php", 125 ], 126 schema="monster_test.fbs", 127 include="include_test", 128 data="monsterdata_test.json", 129) 130 131flatc( 132 NO_INCL_OPTS + DART_OPTS, 133 schema="include_test/include_test1.fbs", 134 include="include_test/sub", 135) 136 137flatc( 138 NO_INCL_OPTS + DART_OPTS, 139 schema="include_test/sub/include_test2.fbs", 140 include="include_test", 141) 142 143flatc( 144 NO_INCL_OPTS + TS_OPTS, 145 cwd=ts_code_gen, 146 schema="../monster_test.fbs", 147 include="../include_test", 148 data="../monsterdata_test.json", 149) 150 151flatc( 152 ["--lua", "--bfbs-filenames", str(tests_path)], 153 schema="monster_test.fbs", 154 include="include_test", 155) 156 157flatc( 158 NO_INCL_OPTS + CPP_OPTS + ["--grpc"], 159 schema="monster_test.fbs", 160 include="include_test", 161 data="monsterdata_test.json", 162) 163 164flatc( 165 RUST_OPTS, 166 schema="monster_test.fbs", 167 include="include_test", 168 prefix="monster_test", 169 data="monsterdata_test.json", 170) 171 172flatc( 173 RUST_SERIALIZE_OPTS, 174 schema="monster_test.fbs", 175 include="include_test", 176 prefix="monster_test_serialize", 177 data="monsterdata_test.json", 178) 179 180flatc( 181 options=BASE_OPTS + ["--python"], 182 schema="monster_test.fbs", 183 include="include_test", 184 data="monsterdata_test.json", 185) 186 187flatc( 188 options=BASE_OPTS + ["--python", "--gen-onefile"], 189 schema="monster_test.fbs", 190 include="include_test", 191 data="monsterdata_test.json", 192) 193 194flatc( 195 PROTO_OPTS, 196 schema="prototest/test.proto", 197) 198 199# For Rust we currently generate two independent schemas, with namespace_test2 200# duplicating the types in namespace_test1 201flatc( 202 RUST_OPTS + CS_OPTS, 203 prefix="namespace_test", 204 schema=[ 205 "namespace_test/namespace_test1.fbs", 206 "namespace_test/namespace_test2.fbs", 207 ], 208) 209 210flatc( 211 [ 212 "--cpp", 213 "--reflect-names", 214 "--no-includes", 215 "--gen-mutable", 216 "--gen-object-api", 217 "--gen-compare", 218 "--gen-name-strings", 219 ], 220 prefix="namespace_test", 221 schema=[ 222 "namespace_test/namespace_test1.fbs", 223 "namespace_test/namespace_test2.fbs", 224 ], 225) 226 227flatc( 228 BASE_OPTS + CPP_OPTS + CS_OPTS + JAVA_OPTS + KOTLIN_OPTS + PHP_OPTS, 229 prefix="union_vector", 230 schema="union_vector/union_vector.fbs", 231) 232 233flatc( 234 BASE_OPTS + TS_OPTS, 235 cwd=ts_code_gen, 236 prefix="union_vector", 237 schema="../union_vector/union_vector.fbs", 238) 239 240flatc( 241 BASE_OPTS + TS_OPTS + ["--gen-name-strings", "--gen-mutable"], 242 cwd=ts_code_gen, 243 include="../include_test", 244 schema="../monster_test.fbs", 245) 246 247flatc( 248 BASE_OPTS + TS_OPTS + ["-b"], 249 cwd=ts_code_gen, 250 include="../include_test", 251 schema="../monster_test.fbs", 252 data="../unicode_test.json", 253) 254 255flatc( 256 BASE_OPTS + TS_OPTS + ["--gen-name-strings"], 257 cwd=ts_code_gen, 258 prefix="union_vector", 259 schema="../union_vector/union_vector.fbs", 260) 261 262flatc( 263 RUST_OPTS, 264 prefix="include_test1", 265 include="include_test", 266 schema="include_test/include_test1.fbs", 267) 268 269flatc( 270 RUST_OPTS, 271 prefix="include_test2", 272 include="include_test", 273 schema="include_test/sub/include_test2.fbs", 274) 275 276flatc( 277 BINARY_OPTS + ["--bfbs-filenames", str(tests_path)], 278 include="include_test", 279 schema="monster_test.fbs", 280) 281 282# Generate the annotated binary of the monster_test binary schema. 283flatc_annotate( 284 schema="../reflection/reflection.fbs", 285 file="monster_test.bfbs", 286 include="include_test", 287) 288 289flatc_annotate( 290 schema="monster_test.fbs", file="monsterdata_test.mon", include="include_test" 291) 292 293flatc( 294 CPP_OPTS 295 + NO_INCL_OPTS 296 + [ 297 "--bfbs-comments", 298 "--bfbs-builtins", 299 "--bfbs-gen-embed", 300 "--bfbs-filenames", 301 str(tests_path), 302 ], 303 include="include_test", 304 schema="monster_test.fbs", 305) 306 307flatc( 308 BINARY_OPTS + ["--bfbs-filenames", str(tests_path)], 309 include="include_test", 310 schema="arrays_test.fbs", 311) 312 313flatc( 314 ["--jsonschema", "--schema"], 315 include="include_test", 316 schema="monster_test.fbs", 317) 318 319if not args.skip_monster_extra: 320 flatc( 321 CPP_OPTS + CS_OPTS + NO_INCL_OPTS + JAVA_OPTS + KOTLIN_OPTS + PYTHON_OPTS, 322 schema="monster_extra.fbs", 323 data="monsterdata_extra.json", 324 ) 325 326 flatc( 327 DART_OPTS + ["--gen-object-api"], 328 schema="monster_extra.fbs", 329 ) 330 331flatc( 332 CPP_OPTS + CS_OPTS + NO_INCL_OPTS + JAVA_OPTS + ["--jsonschema", "--scoped-enums"], 333 schema="arrays_test.fbs", 334) 335 336flatc( 337 RUST_OPTS, 338 prefix="arrays_test", 339 schema="arrays_test.fbs", 340) 341 342flatc( 343 RUST_OPTS, 344 prefix="rust_namer_test", 345 schema="rust_namer_test.fbs", 346) 347 348flatc( 349 BASE_OPTS + PYTHON_OPTS, 350 schema="arrays_test.fbs", 351) 352 353 354flatc( 355 BASE_OPTS + PYTHON_OPTS, 356 schema="nested_union_test.fbs", 357) 358 359 360# Optional Scalars 361optional_scalars_schema = "optional_scalars.fbs" 362flatc(["--java", "--kotlin", "--lobster"], schema=optional_scalars_schema) 363flatc(TS_OPTS, cwd=ts_code_gen, schema="../optional_scalars.fbs") 364 365flatc(["--csharp", "--python", "--gen-object-api"], schema=optional_scalars_schema) 366 367flatc(RUST_OPTS, prefix="optional_scalars", schema=optional_scalars_schema) 368 369flatc(NO_INCL_OPTS + CPP_OPTS, schema=optional_scalars_schema) 370 371# Type / field collsion 372type_field_collsion_schema = "type_field_collsion.fbs" 373 374flatc(["--csharp", "--gen-object-api"], schema=type_field_collsion_schema) 375 376# Union / value collision 377flatc( 378 CS_OPTS + ["--gen-object-api", "--gen-onefile"], 379 prefix="union_value_collsion", 380 schema="union_value_collision.fbs", 381) 382 383# Generate string/vector default code for tests 384flatc(RUST_OPTS, prefix="more_defaults", schema="more_defaults.fbs") 385 386# Generate the schema evolution tests 387flatc( 388 CPP_OPTS + ["--scoped-enums"], 389 prefix="evolution_test", 390 schema=glob(tests_path, "evolution_test/evolution_v*.fbs"), 391) 392 393# Generate the keywords tests 394flatc(BASE_OPTS + CS_OPTS, schema="keyword_test.fbs") 395flatc(RUST_OPTS, prefix="keyword_test", schema="keyword_test.fbs") 396flatc( 397 BASE_OPTS + CS_OPTS + ["--cs-global-alias", "--gen-onefile"], 398 prefix="nested_namespace_test", 399 schema=glob(tests_path, "nested_namespace_test/nested_namespace_test*.fbs"), 400) 401flatc(BASE_OPTS + DART_OPTS, prefix="../dart/test/", schema="keyword_test.fbs") 402 403# Field key lookup with default value test 404dictionary_lookup_schema = "dictionary_lookup.fbs" 405flatc(["--java", "--kotlin"], schema=dictionary_lookup_schema) 406 407# Swift Tests 408swift_prefix = "swift/tests/Tests/FlatBuffers.Test.SwiftTests" 409flatc( 410 SWIFT_OPTS + BASE_OPTS + ["--grpc"], 411 schema="monster_test.fbs", 412 include="include_test", 413 prefix=swift_prefix, 414) 415flatc( 416 SWIFT_OPTS + BASE_OPTS, 417 schema="union_vector/union_vector.fbs", 418 prefix=swift_prefix, 419) 420flatc(SWIFT_OPTS, schema="optional_scalars.fbs", prefix=swift_prefix) 421flatc(SWIFT_OPTS, schema="vector_has_test.fbs", prefix=swift_prefix) 422flatc(SWIFT_OPTS, schema="nan_inf_test.fbs", prefix=swift_prefix) 423flatc( 424 SWIFT_OPTS + ["--gen-object-api"], 425 schema="more_defaults.fbs", 426 prefix=swift_prefix, 427) 428flatc( 429 SWIFT_OPTS + BASE_OPTS, 430 schema="MutatingBool.fbs", 431 prefix=swift_prefix, 432) 433 434flatc( 435 SWIFT_OPTS_CODE_GEN + BASE_OPTS + ["--grpc", "--swift-implementation-only"], 436 schema="test_import.fbs", 437 cwd=swift_code_gen, 438) 439 440flatc( 441 SWIFT_OPTS_CODE_GEN + NO_INCL_OPTS + ["--grpc"], 442 schema="test_no_include.fbs", 443 cwd=swift_code_gen, 444) 445 446# Swift Wasm Tests 447swift_Wasm_prefix = "swift/Wasm.tests/Tests/FlatBuffers.Test.Swift.WasmTests" 448flatc( 449 SWIFT_OPTS + BASE_OPTS, 450 schema="monster_test.fbs", 451 include="include_test", 452 prefix=swift_Wasm_prefix, 453) 454 455# Nim Tests 456NIM_OPTS = BASE_OPTS + ["--nim"] 457flatc(NIM_OPTS, schema="monster_test.fbs", include="include_test") 458flatc(NIM_OPTS, schema="optional_scalars.fbs") 459flatc(NIM_OPTS, schema="more_defaults.fbs") 460flatc(NIM_OPTS, schema="MutatingBool.fbs") 461 462# --filename-suffix and --filename-ext tests 463flatc( 464 CPP_OPTS + NO_INCL_OPTS + ["--grpc", "--filename-ext", "hpp"], 465 include="include_test", 466 prefix="monster_test_suffix/ext_only", 467 schema="monster_test.fbs", 468) 469flatc( 470 CPP_OPTS + NO_INCL_OPTS + ["--grpc", "--filename-suffix", "_suffix"], 471 include="include_test", 472 prefix="monster_test_suffix/filesuffix_only", 473 schema="monster_test.fbs", 474) 475flatc( 476 CPP_OPTS 477 + NO_INCL_OPTS 478 + ["--grpc", "--filename-suffix", "_suffix", "--filename-ext", "hpp"], 479 include="include_test", 480 prefix="monster_test_suffix", 481 schema="monster_test.fbs", 482) 483 484# Flag c++17 requires Clang6, GCC7, MSVC2017 (_MSC_VER >= 1914) or higher. 485cpp_17_prefix = "cpp17/generated_cpp17" 486flatc( 487 CPP_17_OPTS, 488 schema="monster_test.fbs", 489 include="include_test", 490 prefix=cpp_17_prefix, 491) 492flatc( 493 CPP_17_OPTS, 494 schema="optional_scalars.fbs", 495 prefix=cpp_17_prefix, 496) 497flatc( 498 CPP_17_OPTS, 499 schema="union_vector/union_vector.fbs", 500 prefix=cpp_17_prefix, 501) 502 503# Private annotations 504annotations_test_schema = "private_annotation_test.fbs" 505 506flatc( 507 RUST_OPTS + ["--no-leak-private-annotation", "--gen-object-api"], 508 prefix="private_annotation_test", 509 schema=annotations_test_schema, 510) 511 512# Sample files 513samples_schema = "monster.fbs" 514flatc(BASE_OPTS + CPP_OPTS + LOBSTER_OPTS, schema=samples_schema, cwd=samples_path) 515flatc(RUST_OPTS, prefix="rust_generated", schema=samples_schema, cwd=samples_path) 516flatc( 517 BINARY_OPTS + ["--bfbs-filenames", str(samples_path)], 518 schema=samples_schema, 519 cwd=samples_path, 520) 521 522# Reflection 523 524# Skip generating the reflection if told too, as we run this script after 525# building flatc which uses the reflection_generated.h itself. 526if not args.skip_gen_reflection: 527 # C++ Reflection 528 flatc_reflection( 529 ["-c", "--cpp-std", "c++0x"], "include/flatbuffers", "reflection_generated.h" 530 ) 531 532# Python Reflection 533flatc_reflection(["-p"], "python/flatbuffers", "reflection") 534 535# Java Reflection 536flatc_reflection( 537 ["-j", "--java-package-prefix", "com.google.flatbuffers"], 538 "java/src/main/java", 539 "com/google/flatbuffers/reflection", 540) 541 542# Annotation 543 544 545def flatc_annotate(schema, include=None, data=None, cwd=tests_path): 546 cmd = [str(flatc_path)] 547 if include: 548 cmd += ["-I"] + [include] 549 cmd += ["--annotate", schema] 550 if data: 551 cmd += [data] if isinstance(data, str) else data 552 subprocess.run(cmd, cwd=str(cwd), check=True) 553 554 555flatc_annotate( 556 schema="monster_test.fbs", include="include_test", data="monsterdata_test.mon" 557) 558 559# Run the generate_grpc_examples script 560generate_grpc_examples.GenerateGRPCExamples() 561