• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2
3# Copyright 2023 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from __future__ import print_function
18
19import binascii
20import collections
21import ctypes
22import datetime
23import json
24import math
25import os
26import re
27import sys
28
29import yaml
30
31with open("src/core/lib/config/config_vars.yaml") as f:
32    attrs = yaml.safe_load(f.read())
33
34error = False
35today = datetime.date.today()
36two_quarters_from_now = today + datetime.timedelta(days=180)
37for attr in attrs:
38    if "name" not in attr:
39        print("config has no name: %r" % attr)
40        error = True
41        continue
42    if "experiment" in attr["name"] and attr["name"] != "experiments":
43        print("use experiment system for experiments")
44        error = True
45    if "description" not in attr:
46        print("no description for %s" % attr["name"])
47        error = True
48    if "default" not in attr:
49        print("no default for %s" % attr["name"])
50        error = True
51
52if error:
53    sys.exit(1)
54
55
56def c_str(s, encoding="ascii"):
57    if s is None:
58        return '""'
59    if isinstance(s, str):
60        s = s.encode(encoding)
61    result = ""
62    for c in s:
63        c = chr(c) if isinstance(c, int) else c
64        if not (32 <= ord(c) < 127) or c in ("\\", '"'):
65            result += "\\%03o" % ord(c)
66        else:
67            result += c
68    return '"' + result + '"'
69
70
71def snake_to_pascal(s):
72    return "".join(x.capitalize() for x in s.split("_"))
73
74
75# utility: print a big comment block into a set of files
76def put_banner(files, banner):
77    for f in files:
78        for line in banner:
79            print("// %s" % line, file=f)
80        print(file=f)
81
82
83def put_copyright(file):
84    # copy-paste copyright notice from this file
85    with open(sys.argv[0]) as my_source:
86        copyright = []
87        for line in my_source:
88            if line[0] != "#":
89                break
90        for line in my_source:
91            if line[0] == "#":
92                copyright.append(line)
93                break
94        for line in my_source:
95            if line[0] != "#":
96                break
97            copyright.append(line)
98        put_banner([file], [line[2:].rstrip() for line in copyright])
99
100
101RETURN_TYPE = {
102    "int": "int32_t",
103    "string": "absl::string_view",
104    "comma_separated_string": "absl::string_view",
105    "bool": "bool",
106}
107
108MEMBER_TYPE = {
109    "int": "int32_t",
110    "string": "std::string",
111    "comma_separated_string": "std::string",
112    "bool": "bool",
113}
114
115FLAG_TYPE = {
116    "int": "absl::optional<int32_t>",
117    "string": "absl::optional<std::string>",
118    "comma_separated_string": "std::vector<std::string>",
119    "bool": "absl::optional<bool>",
120}
121
122PROTO_TYPE = {
123    "int": "int32",
124    "string": "string",
125    "comma_separated_string": "string",
126    "bool": "bool",
127}
128
129SORT_ORDER_FOR_PACKING = {
130    "int": 0,
131    "bool": 1,
132    "string": 2,
133    "comma_separated_string": 3,
134}
135
136
137def bool_default_value(x, name):
138    if x == True:
139        return "true"
140    if x == False:
141        return "false"
142    if isinstance(x, str) and x.startswith("$"):
143        return x[1:]
144    return x
145
146
147def int_default_value(x, name):
148    if isinstance(x, str) and x.startswith("$"):
149        return x[1:]
150    return x
151
152
153def string_default_value(x, name):
154    if x is None:
155        return '""'
156    if x.startswith("$"):
157        return x[1:]
158    return c_str(x)
159
160
161DEFAULT_VALUE = {
162    "int": int_default_value,
163    "bool": bool_default_value,
164    "string": string_default_value,
165    "comma_separated_string": string_default_value,
166}
167
168TO_STRING = {
169    "int": "$",
170    "bool": '$?"true":"false"',
171    "string": '"\\"", absl::CEscape($), "\\""',
172    "comma_separated_string": '"\\"", absl::CEscape($), "\\""',
173}
174
175attrs_in_packing_order = sorted(
176    attrs, key=lambda a: SORT_ORDER_FOR_PACKING[a["type"]]
177)
178
179with open("test/core/util/fuzz_config_vars.proto", "w") as P:
180    put_copyright(P)
181
182    put_banner(
183        [P],
184        [
185            "",
186            "Automatically generated by tools/codegen/core/gen_config_vars.py",
187            "",
188        ],
189    )
190
191    print('syntax = "proto3";', file=P)
192    print(file=P)
193    print("package grpc.testing;", file=P)
194    print(file=P)
195    print("message FuzzConfigVars {", file=P)
196    for attr in attrs_in_packing_order:
197        if attr.get("fuzz", False) == False:
198            continue
199        print(
200            "  optional %s %s = %d;"
201            % (
202                attr.get("fuzz_type", PROTO_TYPE[attr["type"]]),
203                attr["name"],
204                binascii.crc32(attr["name"].encode("ascii")) & 0x1FFFFFFF,
205            ),
206            file=P,
207        )
208    print("};", file=P)
209
210with open("test/core/util/fuzz_config_vars.h", "w") as H:
211    put_copyright(H)
212
213    put_banner(
214        [H],
215        [
216            "",
217            "Automatically generated by tools/codegen/core/gen_config_vars.py",
218            "",
219        ],
220    )
221
222    print("#ifndef GRPC_TEST_CORE_UTIL_FUZZ_CONFIG_VARS_H", file=H)
223    print("#define GRPC_TEST_CORE_UTIL_FUZZ_CONFIG_VARS_H", file=H)
224    print(file=H)
225    print("#include <grpc/support/port_platform.h>", file=H)
226    print(file=H)
227    print('#include "test/core/util/fuzz_config_vars.pb.h"', file=H)
228    print('#include "src/core/lib/config/config_vars.h"', file=H)
229    print(file=H)
230    print("namespace grpc_core {", file=H)
231    print(file=H)
232    print(
233        (
234            "ConfigVars::Overrides OverridesFromFuzzConfigVars(const"
235            " grpc::testing::FuzzConfigVars& vars);"
236        ),
237        file=H,
238    )
239    print(
240        "void ApplyFuzzConfigVars(const grpc::testing::FuzzConfigVars& vars);",
241        file=H,
242    )
243    print(file=H)
244    print("}  // namespace grpc_core", file=H)
245    print(file=H)
246    print("#endif  // GRPC_TEST_CORE_UTIL_FUZZ_CONFIG_VARS_H", file=H)
247
248with open("test/core/util/fuzz_config_vars.cc", "w") as C:
249    put_copyright(C)
250
251    put_banner(
252        [C],
253        [
254            "",
255            "Automatically generated by tools/codegen/core/gen_config_vars.py",
256            "",
257        ],
258    )
259
260    print('#include "test/core/util/fuzz_config_vars.h"', file=C)
261    print('#include "test/core/util/fuzz_config_vars_helpers.h"', file=C)
262    print(file=C)
263    print("namespace grpc_core {", file=C)
264    print(file=C)
265    print(
266        (
267            "ConfigVars::Overrides OverridesFromFuzzConfigVars(const"
268            " grpc::testing::FuzzConfigVars& vars) {"
269        ),
270        file=C,
271    )
272    print("  ConfigVars::Overrides overrides;", file=C)
273    for attr in attrs_in_packing_order:
274        fuzz = attr.get("fuzz", False)
275        if not fuzz:
276            continue
277        print("  if (vars.has_%s()) {" % attr["name"], file=C)
278        if isinstance(fuzz, str):
279            print(
280                "    overrides.%s = %s(vars.%s());"
281                % (attr["name"], fuzz, attr["name"]),
282                file=C,
283            )
284        else:
285            print(
286                "    overrides.%s = vars.%s();" % (attr["name"], attr["name"]),
287                file=C,
288            )
289        print("  }", file=C)
290    print("  return overrides;", file=C)
291    print("}", file=C)
292    print(
293        "void ApplyFuzzConfigVars(const grpc::testing::FuzzConfigVars& vars) {",
294        file=C,
295    )
296    print(
297        "  ConfigVars::SetOverrides(OverridesFromFuzzConfigVars(vars));", file=C
298    )
299    print("}", file=C)
300    print(file=C)
301    print("}  // namespace grpc_core", file=C)
302
303with open("src/core/lib/config/config_vars.h", "w") as H:
304    put_copyright(H)
305
306    put_banner(
307        [H],
308        [
309            "",
310            "Automatically generated by tools/codegen/core/gen_config_vars.py",
311            "",
312        ],
313    )
314
315    print("#ifndef GRPC_SRC_CORE_LIB_CONFIG_CONFIG_VARS_H", file=H)
316    print("#define GRPC_SRC_CORE_LIB_CONFIG_CONFIG_VARS_H", file=H)
317    print(file=H)
318    print("#include <grpc/support/port_platform.h>", file=H)
319    print(file=H)
320    print("#include <string>", file=H)
321    print("#include <atomic>", file=H)
322    print("#include <stdint.h>", file=H)
323    print('#include "absl/strings/string_view.h"', file=H)
324    print('#include "absl/types/optional.h"', file=H)
325    print(file=H)
326    print("namespace grpc_core {", file=H)
327    print(file=H)
328    print("class GPR_DLL ConfigVars {", file=H)
329    print(" public:", file=H)
330    print("  struct Overrides {", file=H)
331    for attr in attrs_in_packing_order:
332        print(
333            "    absl::optional<%s> %s;"
334            % (MEMBER_TYPE[attr["type"]], attr["name"]),
335            file=H,
336        )
337    print("  };", file=H)
338    print("  ConfigVars(const ConfigVars&) = delete;", file=H)
339    print("  ConfigVars& operator=(const ConfigVars&) = delete;", file=H)
340    print(
341        "  // Get the core configuration; if it does not exist, create it.",
342        file=H,
343    )
344    print("  static const ConfigVars& Get() {", file=H)
345    print("    auto* p = config_vars_.load(std::memory_order_acquire);", file=H)
346    print("    if (p != nullptr) return *p;", file=H)
347    print("    return Load();", file=H)
348    print("  }", file=H)
349    print("  static void SetOverrides(const Overrides& overrides);", file=H)
350    print(
351        "  // Drop the config vars. Users must ensure no other threads are",
352        file=H,
353    )
354    print("  // accessing the configuration.", file=H)
355    print("  static void Reset();", file=H)
356    print("  std::string ToString() const;", file=H)
357    for attr in attrs:
358        for line in attr["description"].splitlines():
359            print("  // %s" % line, file=H)
360        if attr.get("force-load-on-access", False):
361            print(
362                "  %s %s() const;"
363                % (MEMBER_TYPE[attr["type"]], snake_to_pascal(attr["name"])),
364                file=H,
365            )
366        else:
367            print(
368                "  %s %s() const { return %s_; }"
369                % (
370                    RETURN_TYPE[attr["type"]],
371                    snake_to_pascal(attr["name"]),
372                    attr["name"],
373                ),
374                file=H,
375            )
376    print(" private:", file=H)
377    print("  explicit ConfigVars(const Overrides& overrides);", file=H)
378    print("  static const ConfigVars& Load();", file=H)
379    print("  static std::atomic<ConfigVars*> config_vars_;", file=H)
380    for attr in attrs_in_packing_order:
381        if attr.get("force-load-on-access", False):
382            continue
383        print("  %s %s_;" % (MEMBER_TYPE[attr["type"]], attr["name"]), file=H)
384    for attr in attrs_in_packing_order:
385        if attr.get("force-load-on-access", False) == False:
386            continue
387        print(
388            "  absl::optional<%s> override_%s_;"
389            % (MEMBER_TYPE[attr["type"]], attr["name"]),
390            file=H,
391        )
392    print("};", file=H)
393    print(file=H)
394    print("}  // namespace grpc_core", file=H)
395    print(file=H)
396    print("#endif  // GRPC_SRC_CORE_LIB_CONFIG_CONFIG_VARS_H", file=H)
397
398with open("src/core/lib/config/config_vars.cc", "w") as C:
399    put_copyright(C)
400
401    put_banner(
402        [C],
403        [
404            "",
405            "Automatically generated by tools/codegen/core/gen_config_vars.py",
406            "",
407        ],
408    )
409
410    print("#include <grpc/support/port_platform.h>", file=C)
411    print('#include "src/core/lib/config/config_vars.h"', file=C)
412    print('#include "src/core/lib/config/load_config.h"', file=C)
413    print('#include "absl/strings/escaping.h"', file=C)
414    print('#include "absl/flags/flag.h"', file=C)
415    print(file=C)
416
417    for attr in attrs:
418        if "prelude" in attr:
419            print(attr["prelude"], file=C)
420
421    for attr in attrs:
422        print(
423            "ABSL_FLAG(%s, %s, {}, %s);"
424            % (
425                FLAG_TYPE[attr["type"]],
426                "grpc_" + attr["name"],
427                c_str(attr["description"]),
428            ),
429            file=C,
430        )
431    print(file=C)
432    print("namespace grpc_core {", file=C)
433    print(file=C)
434    print("ConfigVars::ConfigVars(const Overrides& overrides) :", file=C)
435    initializers = [
436        '%s_(LoadConfig(FLAGS_grpc_%s, "GRPC_%s", overrides.%s, %s))'
437        % (
438            attr["name"],
439            attr["name"],
440            attr["name"].upper(),
441            attr["name"],
442            DEFAULT_VALUE[attr["type"]](attr["default"], attr["name"]),
443        )
444        for attr in attrs_in_packing_order
445        if attr.get("force-load-on-access", False) == False
446    ]
447    initializers += [
448        "override_%s_(overrides.%s)" % (attr["name"], attr["name"])
449        for attr in attrs_in_packing_order
450        if attr.get("force-load-on-access", False)
451    ]
452    print(",".join(initializers), file=C)
453    print("{}", file=C)
454    print(file=C)
455    for attr in attrs:
456        if attr.get("force-load-on-access", False):
457            print(
458                "%s ConfigVars::%s() const { return LoadConfig(FLAGS_grpc_%s,"
459                ' "GRPC_%s", override_%s_, %s); }'
460                % (
461                    MEMBER_TYPE[attr["type"]],
462                    snake_to_pascal(attr["name"]),
463                    attr["name"],
464                    attr["name"].upper(),
465                    attr["name"],
466                    DEFAULT_VALUE[attr["type"]](attr["default"], attr["name"]),
467                ),
468                file=C,
469            )
470            print(file=C)
471    print("std::string ConfigVars::ToString() const {", file=C)
472    print("  return absl::StrCat(", file=C)
473    for i, attr in enumerate(attrs):
474        if i:
475            print(",", file=C)
476            print(c_str(", " + attr["name"] + ": "), file=C)
477        else:
478            print(c_str(attr["name"] + ": "), file=C)
479        print(
480            ",",
481            TO_STRING[attr["type"]].replace(
482                "$", snake_to_pascal(attr["name"]) + "()"
483            ),
484            file=C,
485        )
486    print(");}", file=C)
487    print(file=C)
488    print("}", file=C)
489