• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2023 The Bazel Authors. All rights reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import argparse
16import io
17import itertools
18import pathlib
19import sys
20import textwrap
21from typing import Callable, TextIO, TypeVar
22
23from stardoc.proto import stardoc_output_pb2
24
25_AttributeType = stardoc_output_pb2.AttributeType
26
27_T = TypeVar("_T")
28
29
30def _anchor_id(text: str) -> str:
31    # MyST/Sphinx's markdown processing doesn't like dots in anchor ids.
32    return "#" + text.replace(".", "_").lower()
33
34
35# Create block attribute line.
36# See https://myst-parser.readthedocs.io/en/latest/syntax/optional.html#block-attributes
37def _block_attrs(*attrs: str) -> str:
38    return "{" + " ".join(attrs) + "}\n"
39
40
41def _link(display: str, link: str = "", *, ref: str = "", classes: str = "") -> str:
42    if ref:
43        ref = f"[{ref}]"
44    if link:
45        link = f"({link})"
46    if classes:
47        classes = "{" + classes + "}"
48    return f"[{display}]{ref}{link}{classes}"
49
50
51def _span(display: str, classes: str = ".span") -> str:
52    return f"[{display}]{{" + classes + "}"
53
54
55def _link_here_icon(anchor: str) -> str:
56    # The headerlink class activates some special logic to show/hide
57    # text upon mouse-over; it's how headings show a clickable link.
58    return _link("¶", anchor, classes=".headerlink")
59
60
61def _inline_anchor(anchor: str) -> str:
62    return _span("", anchor)
63
64
65def _indent_block_text(text: str) -> str:
66    return text.strip().replace("\n", "\n  ")
67
68
69def _join_csv_and(values: list[str]) -> str:
70    if len(values) == 1:
71        return values[0]
72
73    values = list(values)
74    values[-1] = "and " + values[-1]
75    return ", ".join(values)
76
77
78def _position_iter(values: list[_T]) -> tuple[bool, bool, _T]:
79    for i, value in enumerate(values):
80        yield i == 0, i == len(values) - 1, value
81
82
83def _sort_attributes_inplace(attributes):
84    # Sort attributes so the iteration order results in a Python-syntax
85    # valid signature. Keep name first because that's convention.
86    attributes.sort(key=lambda a: (a.name != "name", bool(a.default_value), a.name))
87
88
89class _MySTRenderer:
90    def __init__(
91        self,
92        module: stardoc_output_pb2.ModuleInfo,
93        out_stream: TextIO,
94        public_load_path: str,
95    ):
96        self._module = module
97        self._out_stream = out_stream
98        self._public_load_path = public_load_path
99        self._typedef_stack = []
100
101    def _get_colons(self):
102        # There's a weird behavior where increasing colon indents doesn't
103        # parse as nested objects correctly, so we have to reduce the
104        # number of colons based on the indent level
105        indent = 10 - len(self._typedef_stack)
106        assert indent >= 0
107        return ":::" + ":" * indent
108
109    def render(self):
110        self._render_module(self._module)
111
112    def _render_module(self, module: stardoc_output_pb2.ModuleInfo):
113        if self._public_load_path:
114            bzl_path = self._public_load_path
115        else:
116            bzl_path = "//" + self._module.file.split("//")[1]
117
118        self._write(":::{default-domain} bzl\n:::\n")
119        self._write(":::{bzl:currentfile} ", bzl_path, "\n:::\n\n")
120        self._write(
121            f"# {bzl_path}\n",
122            "\n",
123            module.module_docstring.strip(),
124            "\n\n",
125        )
126
127        objects = itertools.chain(
128            ((r.rule_name, r, self._render_rule) for r in module.rule_info),
129            ((p.provider_name, p, self._render_provider) for p in module.provider_info),
130            ((f.function_name, f, self._process_func_info) for f in module.func_info),
131            ((a.aspect_name, a, self._render_aspect) for a in module.aspect_info),
132            (
133                (m.extension_name, m, self._render_module_extension)
134                for m in module.module_extension_info
135            ),
136            (
137                (r.rule_name, r, self._render_repository_rule)
138                for r in module.repository_rule_info
139            ),
140        )
141        # Sort by name, ignoring case. The `.TYPEDEF` string is removed so
142        # that the .TYPEDEF entries come before what is in the typedef.
143        objects = sorted(objects, key=lambda v: v[0].removesuffix(".TYPEDEF").lower())
144
145        for name, obj, func in objects:
146            self._process_object(name, obj, func)
147            self._write("\n")
148
149        # Close any typedefs
150        while self._typedef_stack:
151            self._typedef_stack.pop()
152            self._render_typedef_end()
153
154    def _process_object(self, name, obj, renderer):
155        # The trailing doc is added to prevent matching a common prefix
156        typedef_group = name.removesuffix(".TYPEDEF") + "."
157        while self._typedef_stack and not typedef_group.startswith(
158            self._typedef_stack[-1]
159        ):
160            self._typedef_stack.pop()
161            self._render_typedef_end()
162        renderer(obj)
163        if name.endswith(".TYPEDEF"):
164            self._typedef_stack.append(typedef_group)
165
166    def _render_aspect(self, aspect: stardoc_output_pb2.AspectInfo):
167        _sort_attributes_inplace(aspect.attribute)
168        self._write("::::::{bzl:aspect} ", aspect.aspect_name, "\n\n")
169        edges = ", ".join(sorted(f"`{attr}`" for attr in aspect.aspect_attribute))
170        self._write(":aspect-attributes: ", edges, "\n\n")
171        self._write(aspect.doc_string.strip(), "\n\n")
172
173        if aspect.attribute:
174            self._render_attributes(aspect.attribute)
175            self._write("\n")
176        self._write("::::::\n")
177
178    def _render_module_extension(self, mod_ext: stardoc_output_pb2.ModuleExtensionInfo):
179        self._write("::::::{bzl:module-extension} ", mod_ext.extension_name, "\n\n")
180        self._write(mod_ext.doc_string.strip(), "\n\n")
181
182        for tag in mod_ext.tag_class:
183            tag_name = f"{mod_ext.extension_name}.{tag.tag_name}"
184            tag_name = f"{tag.tag_name}"
185            self._write(":::::{bzl:tag-class} ")
186
187            _sort_attributes_inplace(tag.attribute)
188            self._render_signature(
189                tag_name,
190                tag.attribute,
191                get_name=lambda a: a.name,
192                get_default=lambda a: a.default_value,
193            )
194
195            if doc_string := tag.doc_string.strip():
196                self._write(doc_string, "\n\n")
197            # Ensure a newline between the directive and the doc fields,
198            # otherwise they get parsed as directive options instead.
199            if not doc_string and tag.attribute:
200                self.write("\n")
201            self._render_attributes(tag.attribute)
202            self._write(":::::\n")
203        self._write("::::::\n")
204
205    def _render_repository_rule(self, repo_rule: stardoc_output_pb2.RepositoryRuleInfo):
206        self._write("::::::{bzl:repo-rule} ")
207        _sort_attributes_inplace(repo_rule.attribute)
208        self._render_signature(
209            repo_rule.rule_name,
210            repo_rule.attribute,
211            get_name=lambda a: a.name,
212            get_default=lambda a: a.default_value,
213        )
214        self._write(repo_rule.doc_string.strip(), "\n\n")
215        if repo_rule.attribute:
216            self._render_attributes(repo_rule.attribute)
217        if repo_rule.environ:
218            self._write(":envvars: ", ", ".join(sorted(repo_rule.environ)))
219        self._write("\n")
220
221    def _render_rule(self, rule: stardoc_output_pb2.RuleInfo):
222        rule_name = rule.rule_name
223        _sort_attributes_inplace(rule.attribute)
224        self._write("::::{bzl:rule} ")
225        self._render_signature(
226            rule_name,
227            rule.attribute,
228            get_name=lambda r: r.name,
229            get_default=lambda r: r.default_value,
230        )
231        self._write(rule.doc_string.strip(), "\n\n")
232
233        if rule.advertised_providers.provider_name:
234            self._write(":provides: ")
235            self._write(" | ".join(rule.advertised_providers.provider_name))
236            self._write("\n")
237        self._write("\n")
238
239        if rule.attribute:
240            self._render_attributes(rule.attribute)
241            self._write("\n")
242        self._write("::::\n")
243
244    def _rule_attr_type_string(self, attr: stardoc_output_pb2.AttributeInfo) -> str:
245        if attr.type == _AttributeType.NAME:
246            return "Name"
247        elif attr.type == _AttributeType.INT:
248            return "int"
249        elif attr.type == _AttributeType.LABEL:
250            return "label"
251        elif attr.type == _AttributeType.STRING:
252            return "str"
253        elif attr.type == _AttributeType.STRING_LIST:
254            return "list[str]"
255        elif attr.type == _AttributeType.INT_LIST:
256            return "list[int]"
257        elif attr.type == _AttributeType.LABEL_LIST:
258            return "list[label]"
259        elif attr.type == _AttributeType.BOOLEAN:
260            return "bool"
261        elif attr.type == _AttributeType.LABEL_STRING_DICT:
262            return "dict[label, str]"
263        elif attr.type == _AttributeType.STRING_DICT:
264            return "dict[str, str]"
265        elif attr.type == _AttributeType.STRING_LIST_DICT:
266            return "dict[str, list[str]]"
267        elif attr.type == _AttributeType.OUTPUT:
268            return "label"
269        elif attr.type == _AttributeType.OUTPUT_LIST:
270            return "list[label]"
271        else:
272            # If we get here, it means the value was unknown for some reason.
273            # Rather than error, give some somewhat understandable value.
274            return _AttributeType.Name(attr.type)
275
276    def _process_func_info(self, func):
277        if func.function_name.endswith(".TYPEDEF"):
278            self._render_typedef_start(func)
279        else:
280            self._render_func(func)
281
282    def _render_typedef_start(self, func):
283        self._write(
284            self._get_colons(),
285            "{bzl:typedef} ",
286            func.function_name.removesuffix(".TYPEDEF"),
287            "\n",
288        )
289        if func.doc_string:
290            self._write(func.doc_string.strip(), "\n")
291
292    def _render_typedef_end(self):
293        self._write(self._get_colons(), "\n\n")
294
295    def _render_func(self, func: stardoc_output_pb2.StarlarkFunctionInfo):
296        self._write(self._get_colons(), "{bzl:function} ")
297
298        parameters = self._render_func_signature(func)
299
300        doc_string = func.doc_string.strip()
301        if doc_string:
302            self._write(doc_string, "\n\n")
303
304        if parameters:
305            # Ensure a newline between the directive and the doc fields,
306            # otherwise they get parsed as directive options instead.
307            if not doc_string:
308                self._write("\n")
309            for param in parameters:
310                self._write(f":arg {param.name}:\n")
311                if param.default_value:
312                    default_value = self._format_default_value(param.default_value)
313                    self._write("  {default-value}`", default_value, "`\n")
314                if param.doc_string:
315                    self._write("  ", _indent_block_text(param.doc_string), "\n")
316                else:
317                    self._write("  _undocumented_\n")
318                self._write("\n")
319
320        if return_doc := getattr(func, "return").doc_string:
321            self._write(":returns:\n")
322            self._write("  ", _indent_block_text(return_doc), "\n")
323        if func.deprecated.doc_string:
324            self._write(":::::{deprecated}: unknown\n")
325            self._write("  ", _indent_block_text(func.deprecated.doc_string), "\n")
326            self._write(":::::\n")
327        self._write(self._get_colons(), "\n")
328
329    def _render_func_signature(self, func):
330        func_name = func.function_name
331        if self._typedef_stack:
332            func_name = func.function_name.removeprefix(self._typedef_stack[-1])
333        self._write(f"{func_name}(")
334        # TODO: Have an "is method" directive in the docstring to decide if
335        # the self parameter should be removed.
336        parameters = [param for param in func.parameter if param.name != "self"]
337
338        # Unfortunately, the stardoc info is incomplete and inaccurate:
339        # * The position of the `*args` param is wrong; it'll always
340        #   be last (or second to last, if kwargs is present).
341        # * Stardoc doesn't explicitly tell us if an arg is `*args` or
342        #   `**kwargs`. Hence f(*args) or f(**kwargs) is ambigiguous.
343        # See these issues:
344        # https://github.com/bazelbuild/stardoc/issues/226
345        # https://github.com/bazelbuild/stardoc/issues/225
346        #
347        # Below, we try to take what info we have and infer what the original
348        # signature was. In short:
349        # * A default=empty, mandatory=false arg is either *args or **kwargs
350        # * If two of those are seen, the first is *args and the second is
351        #   **kwargs. Recall, however, the position of *args is mis-represented.
352        # * If a single default=empty, mandatory=false arg is found, then
353        #   it's ambiguous as to whether its *args or **kwargs. To figure
354        #   that out, we:
355        #   * If it's not the last arg, then it must be *args. In practice,
356        #     this never occurs due to #226 above.
357        #   * If we saw a mandatory arg after an optional arg, then *args
358        #     was supposed to be between them (otherwise it wouldn't be
359        #     valid syntax).
360        #   * Otherwise, it's ambiguous. We just guess by looking at the
361        #     parameter name.
362        var_args = None
363        var_kwargs = None
364        saw_mandatory_after_optional = False
365        first_mandatory_after_optional_index = None
366        optionals_started = False
367        for i, p in enumerate(parameters):
368            optionals_started = optionals_started or not p.mandatory
369            if p.mandatory and optionals_started:
370                saw_mandatory_after_optional = True
371                if first_mandatory_after_optional_index is None:
372                    first_mandatory_after_optional_index = i
373
374            if not p.default_value and not p.mandatory:
375                if var_args is None:
376                    var_args = (i, p)
377                else:
378                    var_kwargs = p
379
380        if var_args and not var_kwargs:
381            if var_args[0] != len(parameters) - 1:
382                pass
383            elif saw_mandatory_after_optional:
384                var_kwargs = var_args[1]
385                var_args = None
386            elif var_args[1].name in ("kwargs", "attrs"):
387                var_kwargs = var_args[1]
388                var_args = None
389
390        # Partial workaround for
391        # https://github.com/bazelbuild/stardoc/issues/226: `*args` renders last
392        if var_args and var_kwargs and first_mandatory_after_optional_index is not None:
393            parameters.pop(var_args[0])
394            parameters.insert(first_mandatory_after_optional_index, var_args[1])
395
396        # The only way a mandatory-after-optional can occur is
397        # if there was `*args` before it. But if we didn't see it,
398        # it must have been the unbound `*` symbol, which stardoc doesn't
399        # tell us exists.
400        if saw_mandatory_after_optional and not var_args:
401            self._write("*, ")
402        for _, is_last, p in _position_iter(parameters):
403            if var_args and p.name == var_args[1].name:
404                self._write("*")
405            elif var_kwargs and p.name == var_kwargs.name:
406                self._write("**")
407            self._write(p.name)
408            if p.default_value:
409                self._write("=", self._format_default_value(p.default_value))
410            if not is_last:
411                self._write(", ")
412        self._write(")\n")
413        return parameters
414
415    def _render_provider(self, provider: stardoc_output_pb2.ProviderInfo):
416        self._write("::::::{bzl:provider} ", provider.provider_name, "\n")
417        if provider.origin_key:
418            self._render_origin_key_option(provider.origin_key)
419        self._write("\n")
420
421        self._write(provider.doc_string.strip(), "\n\n")
422
423        self._write(":::::{bzl:function} ")
424        provider.field_info.sort(key=lambda f: f.name)
425        self._render_signature(
426            "<init>",
427            provider.field_info,
428            get_name=lambda f: f.name,
429        )
430        # TODO: Add support for provider.init once our Bazel version supports
431        # that field
432        self._write(":::::\n")
433
434        for field in provider.field_info:
435            self._write(":::::{bzl:provider-field} ", field.name, "\n")
436            self._write(field.doc_string.strip())
437            self._write("\n")
438            self._write(":::::\n")
439        self._write("::::::\n")
440
441    def _render_attributes(self, attributes: list[stardoc_output_pb2.AttributeInfo]):
442        for attr in attributes:
443            attr_type = self._rule_attr_type_string(attr)
444            self._write(f":attr {attr.name}:\n")
445            if attr.default_value:
446                self._write("  {bzl:default-value}`%s`\n" % attr.default_value)
447            self._write("  {type}`%s`\n" % attr_type)
448            self._write("  ", _indent_block_text(attr.doc_string), "\n")
449            self._write("  :::{bzl:attr-info} Info\n")
450            if attr.mandatory:
451                self._write("  :mandatory:\n")
452            self._write("  :::\n")
453            self._write("\n")
454
455            if attr.provider_name_group:
456                self._write("  {required-providers}`")
457                for _, outer_is_last, provider_group in _position_iter(
458                    attr.provider_name_group
459                ):
460                    pairs = list(
461                        zip(
462                            provider_group.origin_key,
463                            provider_group.provider_name,
464                            strict=True,
465                        )
466                    )
467                    if len(pairs) > 1:
468                        self._write("[")
469                    for _, inner_is_last, (origin_key, name) in _position_iter(pairs):
470                        if origin_key.file == "<native>":
471                            origin = origin_key.name
472                        else:
473                            origin = f"{origin_key.file}%{origin_key.name}"
474                        # We have to use "title <ref>" syntax because the same
475                        # name might map to different origins. Stardoc gives us
476                        # the provider's actual name, not the name of the symbol
477                        # used in the source.
478                        self._write(f"'{name} <{origin}>'")
479                        if not inner_is_last:
480                            self._write(", ")
481
482                    if len(pairs) > 1:
483                        self._write("]")
484
485                    if not outer_is_last:
486                        self._write(" | ")
487                self._write("`\n")
488
489            self._write("\n")
490
491    def _render_signature(
492        self,
493        name: str,
494        parameters: list[_T],
495        *,
496        get_name: Callable[_T, str],
497        get_default: Callable[_T, str] = lambda v: None,
498    ):
499        self._write(name, "(")
500        for _, is_last, param in _position_iter(parameters):
501            param_name = get_name(param)
502            self._write(f"{param_name}")
503            default_value = get_default(param)
504            if default_value:
505                default_value = self._format_default_value(default_value)
506                self._write(f"={default_value}")
507            if not is_last:
508                self._write(", ")
509        self._write(")\n\n")
510
511    def _render_origin_key_option(self, origin_key, indent=""):
512        self._write(
513            indent,
514            ":origin-key: ",
515            self._format_option_value(f"{origin_key.file}%{origin_key.name}"),
516            "\n",
517        )
518
519    def _format_default_value(self, default_value):
520        # Handle <function foo from //baz:bar.bzl>
521        # For now, just use quotes for lack of a better option
522        if default_value.startswith("<"):
523            return f"'{default_value}'"
524        elif default_value.startswith("Label("):
525            # Handle Label(*, "@some//label:target")
526            start_quote = default_value.find('"')
527            end_quote = default_value.rfind('"')
528            return default_value[start_quote : end_quote + 1]
529        else:
530            return default_value
531
532    def _format_option_value(self, value):
533        # Leading @ symbols are special markup; escape them.
534        if value.startswith("@"):
535            return "\\" + value
536        else:
537            return value
538
539    def _write(self, *lines: str):
540        self._out_stream.writelines(lines)
541
542
543def _convert(
544    *,
545    proto: pathlib.Path,
546    output: pathlib.Path,
547    public_load_path: str,
548):
549    module = stardoc_output_pb2.ModuleInfo.FromString(proto.read_bytes())
550    with output.open("wt", encoding="utf8") as out_stream:
551        _MySTRenderer(module, out_stream, public_load_path).render()
552
553
554def _create_parser():
555    parser = argparse.ArgumentParser(fromfile_prefix_chars="@")
556    parser.add_argument("--proto", dest="proto", type=pathlib.Path)
557    parser.add_argument("--output", dest="output", type=pathlib.Path)
558    parser.add_argument("--public-load-path", dest="public_load_path")
559    return parser
560
561
562def main(args):
563    options = _create_parser().parse_args(args)
564    _convert(
565        proto=options.proto,
566        output=options.output,
567        public_load_path=options.public_load_path,
568    )
569    return 0
570
571
572if __name__ == "__main__":
573    sys.exit(main(sys.argv[1:]))
574