• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Protocol Buffers - Google's data interchange format
3# Copyright 2008 Google Inc.  All rights reserved.
4# https://developers.google.com/protocol-buffers/
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met:
9#
10#   * Redistributions of source code must retain the above copyright
11# notice, this list of conditions and the following disclaimer.
12#   * Redistributions in binary form must reproduce the above
13# copyright notice, this list of conditions and the following disclaimer
14# in the documentation and/or other materials provided with the
15# distribution.
16#   * Neither the name of Google Inc. nor the names of its
17# contributors may be used to endorse or promote products derived from
18# this software without specific prior written permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
32"""Script to generate a list of all modules to use in autosummary.
33
34This script creates a ReStructured Text file for each public module in the
35protobuf Python package. The script also updates the table of contents in
36``docs/index.rst`` to point to these module references.
37
38To build the docs with Sphinx:
39
401. Install the needed packages (``sphinx``, ``sphinxcontrib-napoleon`` for
41   Google-style docstring support). I've created a conda environment file to
42   make this easier:
43
44.. code:: bash
45
46   conda env create -f python/docs/environment.yml
47
482. (Optional) Generate reference docs files and regenerate index:
49
50.. code:: bash
51
52   cd python/docs
53   python generate_docs.py
54
553. Run Sphinx.
56
57.. code:: bash
58
59   make html
60"""
61
62import pathlib
63import re
64
65
66DOCS_DIR = pathlib.Path(__file__).parent.resolve()
67PYTHON_DIR = DOCS_DIR.parent
68SOURCE_DIR = PYTHON_DIR / "google" / "protobuf"
69SOURCE_POSIX = SOURCE_DIR.as_posix()
70
71# Modules which are always included:
72INCLUDED_MODULES = (
73  "google.protobuf.internal.containers",
74)
75
76# Packages to ignore, including all modules (unless in INCLUDED_MODULES):
77IGNORED_PACKAGES = (
78  "compiler",
79  "docs",
80  "internal",
81  "pyext",
82  "util",
83)
84
85# Ignored module stems in all packages (unless in INCLUDED_MODULES):
86IGNORED_MODULES = (
87  "any_test_pb2",
88  "api_pb2",
89  "unittest",
90  "source_context_pb2",
91  "test_messages_proto3_pb2",
92  "test_messages_proto2",
93)
94
95TOC_REGEX = re.compile(
96  r"\.\. START REFTOC.*\.\. END REFTOC\.\n",
97  flags=re.DOTALL,
98)
99TOC_TEMPLATE = """.. START REFTOC, generated by generate_docs.py.
100.. toctree::
101
102   {toctree}
103
104.. END REFTOC.
105"""
106
107AUTOMODULE_TEMPLATE = """.. DO NOT EDIT, generated by generate_docs.py.
108
109.. ifconfig:: build_env == 'readthedocs'
110
111   .. warning::
112
113      You are reading the documentation for the `latest committed changes
114      <https://github.com/protocolbuffers/protobuf/tree/master/python>`_ of
115      the `Protocol Buffers package for Python
116      <https://developers.google.com/protocol-buffers/docs/pythontutorial>`_.
117      Some features may not yet be released. Read the documentation for the
118      latest released package at `googleapis.dev
119      <https://googleapis.dev/python/protobuf/latest/>`_.
120
121{module}
122{underline}
123
124.. automodule:: {module}
125   :members:
126   :inherited-members:
127   :undoc-members:
128"""
129
130
131def find_modules():
132  modules = []
133  for module_path in SOURCE_DIR.glob("**/*.py"):
134    # Determine the (dotted) relative package and module names.
135    package_path = module_path.parent.relative_to(PYTHON_DIR)
136    if package_path == SOURCE_DIR:
137      package_name = ""
138      module_name = module_path.stem
139    else:
140      package_name = package_path.as_posix().replace("/", ".")
141      module_name = package_name + "." + module_path.stem
142
143    # Filter: first, accept anything in the whitelist; then, reject anything
144    # at package level, then module name level.
145    if any(include == module_name for include in INCLUDED_MODULES):
146      pass
147    elif any(ignored in package_name for ignored in IGNORED_PACKAGES):
148      continue
149    elif any(ignored in module_path.stem for ignored in IGNORED_MODULES):
150      continue
151
152    if module_path.name == "__init__.py":
153      modules.append(package_name)
154    else:
155      modules.append(module_name)
156
157  return modules
158
159
160def write_automodule(module):
161  contents = AUTOMODULE_TEMPLATE.format(module=module, underline="=" * len(module),)
162  automodule_path = DOCS_DIR.joinpath(*module.split(".")).with_suffix(".rst")
163  try:
164    automodule_path.parent.mkdir(parents=True)
165  except FileExistsError:
166    pass
167  with open(automodule_path, "w") as automodule_file:
168    automodule_file.write(contents)
169
170
171def replace_toc(modules):
172  toctree = [module.replace(".", "/") for module in modules]
173  with open(DOCS_DIR / "index.rst", "r") as index_file:
174    index_contents = index_file.read()
175  toc = TOC_TEMPLATE.format(
176    toctree="\n   ".join(toctree)
177  )
178  index_contents = re.sub(TOC_REGEX, toc, index_contents)
179  with open(DOCS_DIR / "index.rst", "w") as index_file:
180    index_file.write(index_contents)
181
182
183def main():
184  modules = list(sorted(find_modules()))
185  for module in modules:
186    print("Generating reference for {}".format(module))
187    write_automodule(module)
188  print("Generating index.rst")
189  replace_toc(modules)
190
191if __name__ == "__main__":
192    main()
193