• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #!/usr/bin/python3
2 #
3 # Copyright 2016-2023 The Khronos Group Inc.
4 #
5 # SPDX-License-Identifier: Apache-2.0
6 
7 # genRef.py - create API ref pages from spec source files
8 #
9 # Usage: genRef.py files
10 
11 import argparse
12 import io
13 import os
14 import re
15 import sys
16 from collections import OrderedDict
17 from reflib import (findRefs, fixupRefs, loadFile, logDiag, logWarn, logErr,
18                     printPageInfo, setLogFile)
19 from reg import Registry
20 from generator import GeneratorOptions
21 from parse_dependency import dependencyNames
22 from apiconventions import APIConventions
23 
24 
25 # refpage 'type' attributes which are API entities and contain structured
26 # content such as API includes, valid usage blocks, etc.
27 refpage_api_types = (
28     'basetypes',
29     'consts',
30     'defines',
31     'enums',
32     'flags',
33     'funcpointers',
34     'handles',
35     'protos',
36     'structs',
37 )
38 
39 # Other refpage types - SPIR-V builtins, API feature blocks, etc. - which do
40 # not have structured content.
41 refpage_other_types = (
42     'builtins',
43     'feature',
44     'freeform',
45     'spirv'
46 )
47 
48 
49 def makeExtensionInclude(name):
50     """Return an include command for a generated extension interface.
51        - name - extension name"""
52 
53     return 'include::{}/meta/refpage.{}{}[]'.format(
54             conventions.generated_include_path,
55             name,
56             conventions.file_suffix)
57 
58 
59 def makeAPIInclude(type, name):
60     """Return an include command for a generated API interface
61        - type - type of the API, e.g. 'flags', 'handles', etc
62        - name - name of the API"""
63 
64     return 'include::{}/api/{}/{}{}\n'.format(
65             conventions.generated_include_path,
66             type, name, conventions.file_suffix)
67 
68 
69 def isextension(name):
70     """Return True if name is an API extension name (ends with an upper-case
71     author ID).
72 
73     This assumes that author IDs are at least two characters."""
74     return name[-2:].isalpha() and name[-2:].isupper()
75 
76 
77 def printCopyrightSourceComments(fp):
78     """Print Khronos CC-BY copyright notice on open file fp.
79 
80     Writes an asciidoc comment block, which copyrights the source
81     file."""
82     print('// Copyright 2014-2023 The Khronos Group Inc.', file=fp)
83     print('//', file=fp)
84     # This works around constraints of the 'reuse' tool
85     print('// SPDX' + '-License-Identifier: CC-BY-4.0', file=fp)
86     print('', file=fp)
87 
88 
89 def printFooter(fp, leveloffset=0):
90     """Print footer material at the end of each refpage on open file fp.
91 
92     If generating separate refpages, adds the copyright.
93     If generating the single combined refpage, just add a separator.
94 
95     - leveloffset - number of levels to bias section titles up or down."""
96 
97     # Generate the section header.
98     # Default depth is 2.
99     depth = max(0, leveloffset + 2)
100     prefix = '=' * depth
101 
102     print('ifdef::doctype-manpage[]',
103           f'{prefix} Copyright',
104           '',
105           'include::{config}/copyright-ccby' + conventions.file_suffix + '[]',
106           'endif::doctype-manpage[]',
107           '',
108           'ifndef::doctype-manpage[]',
109           '<<<',
110           'endif::doctype-manpage[]',
111           '',
112           sep='\n', file=fp)
113 
114 
115 def macroPrefix(name):
116     """Add a spec asciidoc macro prefix to an API name, depending on its type
117     (protos, structs, enums, etc.).
118 
119     If the name is not recognized, use the generic link macro 'reflink:'."""
120     if name in api.basetypes:
121         return 'basetype:' + name
122     if name in api.defines:
123         return 'dlink:' + name
124     if name in api.enums:
125         return 'elink:' + name
126     if name in api.flags:
127         return 'tlink:' + name
128     if name in api.funcpointers:
129         return 'tlink:' + name
130     if name in api.handles:
131         return 'slink:' + name
132     if name in api.protos:
133         return 'flink:' + name
134     if name in api.structs:
135         return 'slink:' + name
136     if name == 'TBD':
137         return 'No cross-references are available'
138     return 'reflink:' + name
139 
140 
141 def seeAlsoList(apiName, explicitRefs=None, apiAliases=[]):
142     """Return an asciidoc string with a list of 'See Also' references for the
143     API entity 'apiName', based on the relationship mapping in the api module.
144 
145     'explicitRefs' is a list of additional cross-references.
146 
147     If apiAliases is not None, it is a list of aliases of apiName whose
148     cross-references will also be included.
149 
150     If no relationships are available, return None."""
151 
152     refs = set(())
153 
154     # apiName and its aliases are treated equally
155     allApis = apiAliases.copy()
156     allApis.append(apiName)
157 
158     # Add all the implicit references to refs
159     for name in allApis:
160         if name in api.mapDict:
161             refs.update(api.mapDict[name])
162 
163     # Add all the explicit references
164     if explicitRefs is not None:
165         if isinstance(explicitRefs, str):
166             explicitRefs = explicitRefs.split()
167         refs.update(name for name in explicitRefs)
168 
169     # Add extensions / core versions based on dependencies
170     for name in allApis:
171         if name in api.requiredBy:
172             for (base,dependency) in api.requiredBy[name]:
173                 refs.add(base)
174                 if dependency is not None:
175                     # 'dependency' may be a boolean expression of extension
176                     # names.
177                     # Extract them for use in cross-references.
178                     for extname in dependencyNames(dependency):
179                         refs.add(extname)
180 
181     if len(refs) == 0:
182         return None
183     else:
184         return ', '.join(macroPrefix(name) for name in sorted(refs)) + '\n'
185 
186 
187 def remapIncludes(lines, baseDir, specDir):
188     """Remap include directives in a list of lines so they can be extracted to a
189     different directory.
190 
191     Returns remapped lines.
192 
193     - lines - text to remap
194     - baseDir - target directory
195     - specDir - source directory"""
196     # This should be compiled only once
197     includePat = re.compile(r'^include::(?P<path>.*)\[\]')
198 
199     newLines = []
200     for line in lines:
201         matches = includePat.search(line)
202         if matches is not None:
203             path = matches.group('path')
204 
205             if path[0] != '{':
206                 # Relative path to include file from here
207                 incPath = specDir + '/' + path
208                 # Remap to be relative to baseDir
209                 newPath = os.path.relpath(incPath, baseDir)
210                 newLine = 'include::' + newPath + '[]\n'
211                 logDiag('remapIncludes: remapping', line, '->', newLine)
212                 newLines.append(newLine)
213             else:
214                 # An asciidoctor variable starts the path.
215                 # This must be an absolute path, not needing to be rewritten.
216                 newLines.append(line)
217         else:
218             newLines.append(line)
219     return newLines
220 
221 
222 def refPageShell(pageName, pageDesc, fp, head_content = None, sections=None, tail_content=None, man_section=3):
223     """Generate body of a reference page.
224 
225     - pageName - string name of the page
226     - pageDesc - string short description of the page
227     - fp - file to write to
228     - head_content - text to include before the sections
229     - sections - iterable returning (title,body) for each section.
230     - tail_content - text to include after the sections
231     - man_section - Unix man page section"""
232 
233     printCopyrightSourceComments(fp)
234 
235     print(':data-uri:',
236           ':icons: font',
237           ':attribute-missing: warn',
238           conventions.extra_refpage_headers,
239           '',
240           sep='\n', file=fp)
241 
242     s = '{}({})'.format(pageName, man_section)
243     print('= ' + s,
244           '',
245           conventions.extra_refpage_body,
246           '',
247           sep='\n', file=fp)
248     if pageDesc.strip() == '':
249         pageDesc = 'NO SHORT DESCRIPTION PROVIDED'
250         logWarn('refPageHead: no short description provided for', pageName)
251 
252     print('== Name',
253           '{} - {}'.format(pageName, pageDesc),
254           '',
255           sep='\n', file=fp)
256 
257     if head_content is not None:
258         print(head_content,
259               '',
260               sep='\n', file=fp)
261 
262     if sections is not None:
263         for title, content in sections.items():
264             print('== {}'.format(title),
265                   '',
266                   content,
267                   '',
268                   sep='\n', file=fp)
269 
270     if tail_content is not None:
271         print(tail_content,
272               '',
273               sep='\n', file=fp)
274 
275 
276 def refPageHead(pageName, pageDesc, specText, fieldName, fieldText, descText, fp):
277     """Generate header of a reference page.
278 
279     - pageName - string name of the page
280     - pageDesc - string short description of the page
281     - specType - string containing 'spec' field from refpage open block, or None.
282       Used to determine containing spec name and URL.
283     - specText - string that goes in the "C Specification" section
284     - fieldName - string heading an additional section following specText, if not None
285     - fieldText - string that goes in the additional section
286     - descText - string that goes in the "Description" section
287     - fp - file to write to"""
288     sections = OrderedDict()
289 
290     if specText is not None:
291         sections['C Specification'] = specText
292 
293     if fieldName is not None:
294         sections[fieldName] = fieldText
295 
296     if descText is None or descText.strip() == '':
297         logWarn('refPageHead: no description provided for', pageName)
298 
299     if descText is not None:
300         sections['Description'] = descText
301 
302     refPageShell(pageName, pageDesc, fp, head_content=None, sections=sections)
303 
304 
305 def refPageTail(pageName,
306                 specType=None,
307                 specAnchor=None,
308                 seeAlso=None,
309                 fp=None,
310                 auto=False,
311                 leveloffset=0):
312     """Generate end boilerplate of a reference page.
313 
314     - pageName - name of the page
315     - specType - None or the 'spec' attribute from the refpage block,
316       identifying the specification name and URL this refpage links to.
317     - specAnchor - None or the 'anchor' attribute from the refpage block,
318       identifying the anchor in the specification this refpage links to. If
319       None, the pageName is assumed to be a valid anchor.
320     - seeAlso - text of the "See Also" section
321     - fp - file to write the page to
322     - auto - True if this is an entirely generated refpage, False if it is
323       handwritten content from the spec.
324     - leveloffset - number of levels to bias section titles up or down."""
325 
326     specName = conventions.api_name(specType)
327     specURL = conventions.specURL(specType)
328     if specAnchor is None:
329         specAnchor = pageName
330 
331     if seeAlso is None:
332         seeAlso = 'No cross-references are available\n'
333 
334     notes = [
335         'For more information, see the {}#{}[{} Specification^]'.format(
336             specURL, specAnchor, specName),
337         '',
338     ]
339 
340     if auto:
341         notes.extend((
342             'This page is a generated document.',
343             'Fixes and changes should be made to the generator scripts, '
344             'not directly.',
345         ))
346     else:
347         notes.extend((
348             'This page is extracted from the ' + specName + ' Specification. ',
349             'Fixes and changes should be made to the Specification, '
350             'not directly.',
351         ))
352 
353     # Generate the section header.
354     # Default depth is 2.
355     depth = max(0, leveloffset + 2)
356     prefix = '=' * depth
357 
358     print(f'{prefix} See Also',
359           '',
360           seeAlso,
361           '',
362           sep='\n', file=fp)
363 
364     print(f'{prefix} Document Notes',
365           '',
366           '\n'.join(notes),
367           '',
368           sep='\n', file=fp)
369 
370     printFooter(fp, leveloffset)
371 
372 
373 def xrefRewriteInitialize():
374     """Initialize substitution patterns for asciidoctor xrefs."""
375 
376     global refLinkPattern, refLinkSubstitute
377     global refLinkTextPattern, refLinkTextSubstitute
378     global specLinkPattern, specLinkSubstitute
379 
380     # These are xrefs to API entities, rewritten to link to refpages
381     # The refLink variants are for xrefs with only an anchor and no text.
382     # The refLinkText variants are for xrefs with both anchor and text
383     refLinkPattern = re.compile(r'<<([Vv][Kk][A-Za-z0-9_]+)>>')
384     refLinkSubstitute = r'link:\1.html[\1^]'
385 
386     refLinkTextPattern = re.compile(r'<<([Vv][Kk][A-Za-z0-9_]+)[,]?[ \t\n]*([^>,]*)>>')
387     refLinkTextSubstitute = r'link:\1.html[\2^]'
388 
389     # These are xrefs to other anchors, rewritten to link to the spec
390     specLinkPattern = re.compile(r'<<([-A-Za-z0-9_.(){}:]+)[,]?[ \t\n]*([^>,]*)>>')
391 
392     # Unfortunately, specLinkSubstitute depends on the link target,
393     # so cannot be constructed in advance.
394     specLinkSubstitute = None
395 
396 
397 def xrefRewrite(text, specURL):
398     """Rewrite asciidoctor xrefs in text to resolve properly in refpages.
399     Xrefs which are to refpages are rewritten to link to those
400     refpages. The remainder are rewritten to generate external links into
401     the supplied specification document URL.
402 
403     - text - string to rewrite, or None
404     - specURL - URL to target
405 
406     Returns rewritten text, or None, respectively"""
407 
408     global refLinkPattern, refLinkSubstitute
409     global refLinkTextPattern, refLinkTextSubstitute
410     global specLinkPattern, specLinkSubstitute
411 
412     specLinkSubstitute = r'link:{}#\1[\2^]'.format(specURL)
413 
414     if text is not None:
415         text, _ = refLinkPattern.subn(refLinkSubstitute, text)
416         text, _ = refLinkTextPattern.subn(refLinkTextSubstitute, text)
417         text, _ = specLinkPattern.subn(specLinkSubstitute, text)
418 
419     return text
420 
421 def emitPage(baseDir, specDir, pi, file):
422     """Extract a single reference page into baseDir.
423 
424     - baseDir - base directory to emit page into
425     - specDir - directory extracted page source came from
426     - pi - pageInfo for this page relative to file
427     - file - list of strings making up the file, indexed by pi"""
428     pageName = f'{baseDir}/{pi.name}{conventions.file_suffix}'
429 
430     # Add a dictionary entry for this page
431     global genDict
432     genDict[pi.name] = None
433     logDiag('emitPage:', pageName)
434 
435     # Short description
436     if pi.desc is None:
437         pi.desc = '(no short description available)'
438 
439     # Member/parameter section label and text, if there is one
440     field = None
441     fieldText = None
442 
443     # Only do structural checks on API pages
444     if pi.type in refpage_api_types:
445         if pi.include is None:
446             logWarn('emitPage:', pageName, 'INCLUDE is None, no page generated')
447             return
448 
449         # Specification text
450         lines = remapIncludes(file[pi.begin:pi.include + 1], baseDir, specDir)
451         specText = ''.join(lines)
452 
453         if pi.param is not None:
454             if pi.type == 'structs':
455                 field = 'Members'
456             elif pi.type in ['protos', 'funcpointers']:
457                 field = 'Parameters'
458             else:
459                 logWarn('emitPage: unknown field type:', pi.type,
460                         'for', pi.name)
461             lines = remapIncludes(file[pi.param:pi.body], baseDir, specDir)
462             fieldText = ''.join(lines)
463 
464         # Description text
465         if pi.body != pi.include:
466             lines = remapIncludes(file[pi.body:pi.end + 1], baseDir, specDir)
467             descText = ''.join(lines)
468         else:
469             descText = None
470             logWarn('emitPage: INCLUDE == BODY, so description will be empty for', pi.name)
471             if pi.begin != pi.include:
472                 logWarn('emitPage: Note: BEGIN != INCLUDE, so the description might be incorrectly located before the API include!')
473     elif pi.type in refpage_other_types:
474         specText = None
475         descText = ''.join(file[pi.begin:pi.end + 1])
476     else:
477         # This should be caught in the spec markup checking tests
478         logErr(f"emitPage: refpage type='{pi.type}' is unrecognized")
479 
480     # Rewrite asciidoctor xrefs to resolve properly in refpages
481     specURL = conventions.specURL(pi.spec)
482 
483     specText = xrefRewrite(specText, specURL)
484     fieldText = xrefRewrite(fieldText, specURL)
485     descText = xrefRewrite(descText, specURL)
486 
487     fp = open(pageName, 'w', encoding='utf-8')
488     refPageHead(pi.name,
489                 pi.desc,
490                 specText,
491                 field, fieldText,
492                 descText,
493                 fp)
494     refPageTail(pageName=pi.name,
495                 specType=pi.spec,
496                 specAnchor=pi.anchor,
497                 seeAlso=seeAlsoList(pi.name, pi.refs, pi.alias.split()),
498                 fp=fp,
499                 auto=False)
500     fp.close()
501 
502 
503 def autoGenEnumsPage(baseDir, pi, file):
504     """Autogenerate a single reference page in baseDir.
505 
506     Script only knows how to do this for /enums/ pages, at present.
507 
508     - baseDir - base directory to emit page into
509     - pi - pageInfo for this page relative to file
510     - file - list of strings making up the file, indexed by pi"""
511     pageName = f'{baseDir}/{pi.name}{conventions.file_suffix}'
512     fp = open(pageName, 'w', encoding='utf-8')
513 
514     # Add a dictionary entry for this page
515     global genDict
516     genDict[pi.name] = None
517     logDiag('autoGenEnumsPage:', pageName)
518 
519     # Short description
520     if pi.desc is None:
521         pi.desc = '(no short description available)'
522 
523     # Description text. Allow for the case where an enum definition
524     # is not embedded.
525     if not pi.embed:
526         embedRef = ''
527     else:
528         embedRef = ''.join((
529                            '  * The reference page for ',
530                            macroPrefix(pi.embed),
531                            ', where this interface is defined.\n'))
532 
533     txt = ''.join((
534         'For more information, see:\n\n',
535         embedRef,
536         '  * The See Also section for other reference pages using this type.\n',
537         '  * The ' + apiName + ' Specification.\n'))
538 
539     refPageHead(pi.name,
540                 pi.desc,
541                 ''.join(file[pi.begin:pi.include + 1]),
542                 None, None,
543                 txt,
544                 fp)
545     refPageTail(pageName=pi.name,
546                 specType=pi.spec,
547                 specAnchor=pi.anchor,
548                 seeAlso=seeAlsoList(pi.name, pi.refs, pi.alias.split()),
549                 fp=fp,
550                 auto=True)
551     fp.close()
552 
553 
554 # Pattern to break apart an API *Flags{authorID} name, used in
555 # autoGenFlagsPage.
556 flagNamePat = re.compile(r'(?P<name>\w+)Flags(?P<author>[A-Z]*)')
557 
558 
559 def autoGenFlagsPage(baseDir, flagName):
560     """Autogenerate a single reference page in baseDir for an API *Flags type.
561 
562     - baseDir - base directory to emit page into
563     - flagName - API *Flags name"""
564     pageName = f'{baseDir}/{flagName}{conventions.file_suffix}'
565     fp = open(pageName, 'w', encoding='utf-8')
566 
567     # Add a dictionary entry for this page
568     global genDict
569     genDict[flagName] = None
570     logDiag('autoGenFlagsPage:', pageName)
571 
572     # Short description
573     matches = flagNamePat.search(flagName)
574     if matches is not None:
575         name = matches.group('name')
576         author = matches.group('author')
577         logDiag('autoGenFlagsPage: split name into', name, 'Flags', author)
578         flagBits = name + 'FlagBits' + author
579         desc = 'Bitmask of ' + flagBits
580     else:
581         logWarn('autoGenFlagsPage:', pageName, 'does not end in "Flags{author ID}". Cannot infer FlagBits type.')
582         flagBits = None
583         desc = 'Unknown ' + apiName + ' flags type'
584 
585     # Description text
586     if flagBits is not None:
587         txt = ''.join((
588             'etext:' + flagName,
589             ' is a mask of zero or more elink:' + flagBits + '.\n',
590             'It is used as a member and/or parameter of the structures and commands\n',
591             'in the See Also section below.\n'))
592     else:
593         txt = ''.join((
594             'etext:' + flagName,
595             ' is an unknown ' + apiName + ' type, assumed to be a bitmask.\n'))
596 
597     refPageHead(flagName,
598                 desc,
599                 makeAPIInclude('flags', flagName),
600                 None, None,
601                 txt,
602                 fp)
603     refPageTail(pageName=flagName,
604                 specType=pi.spec,
605                 specAnchor=pi.anchor,
606                 seeAlso=seeAlsoList(flagName, None),
607                 fp=fp,
608                 auto=True)
609     fp.close()
610 
611 
612 def autoGenHandlePage(baseDir, handleName):
613     """Autogenerate a single handle page in baseDir for an API handle type.
614 
615     - baseDir - base directory to emit page into
616     - handleName - API handle name"""
617     # @@ Need to determine creation function & add handles/ include for the
618     # @@ interface in generator.py.
619     pageName = f'{baseDir}/{handleName}{conventions.file_suffix}'
620     fp = open(pageName, 'w', encoding='utf-8')
621 
622     # Add a dictionary entry for this page
623     global genDict
624     genDict[handleName] = None
625     logDiag('autoGenHandlePage:', pageName)
626 
627     # Short description
628     desc = apiName + ' object handle'
629 
630     descText = ''.join((
631         'sname:' + handleName,
632         ' is an object handle type, referring to an object used\n',
633         'by the ' + apiName + ' implementation. These handles are created or allocated\n',
634         'by the @@ TBD @@ function, and used by other ' + apiName + ' structures\n',
635         'and commands in the See Also section below.\n'))
636 
637     refPageHead(handleName,
638                 desc,
639                 makeAPIInclude('handles', handleName),
640                 None, None,
641                 descText,
642                 fp)
643     refPageTail(pageName=handleName,
644                 specType=pi.spec,
645                 specAnchor=pi.anchor,
646                 seeAlso=seeAlsoList(handleName, None),
647                 fp=fp,
648                 auto=True)
649     fp.close()
650 
651 
652 def genRef(specFile, baseDir):
653     """Extract reference pages from a spec asciidoc source file.
654 
655     - specFile - filename to extract from
656     - baseDir - output directory to generate page in"""
657     # We do not care the newline format used here.
658     file, _ = loadFile(specFile)
659     if file is None:
660         return
661 
662     # Save the path to this file for later use in rewriting relative includes
663     specDir = os.path.dirname(os.path.abspath(specFile))
664 
665     pageMap = findRefs(file, specFile)
666     logDiag(specFile + ': found', len(pageMap.keys()), 'potential pages')
667 
668     sys.stderr.flush()
669 
670     # Fix up references in pageMap
671     fixupRefs(pageMap, specFile, file)
672 
673     # Create each page, if possible
674     pages = {}
675 
676     for name in sorted(pageMap):
677         pi = pageMap[name]
678 
679         # Only generate the page if it is in the requested build
680         # 'freeform' pages are always generated
681         # 'feature' pages (core versions & extensions) are generated if they are in
682         # the requested feature list
683         # All other pages (APIs) are generated if they are in the API map for
684         # the build.
685         if pi.type in refpage_api_types:
686             if name not in api.typeCategory:
687                 # Also check aliases of name - api.nonexistent is the same
688                 # mapping used to rewrite *link: macros in this build.
689                 if name not in api.nonexistent:
690                     logWarn(f'genRef: NOT generating feature page {name} - API not in this build')
691                     continue
692                 else:
693                     logWarn(f'genRef: generating feature page {name} because its alias {api.nonexistent[name]} exists')
694         elif pi.type in refpage_other_types:
695             # The only non-API type which can be checked is a feature refpage
696             if pi.type == 'feature':
697                 if name not in api.features:
698                     logWarn(f'genRef: NOT generating feature page {name} - feature not in this build')
699                     continue
700 
701         printPageInfo(pi, file)
702 
703         if pi.Warning:
704             logDiag('genRef:', pi.name + ':', pi.Warning)
705 
706         if pi.extractPage:
707             emitPage(baseDir, specDir, pi, file)
708         elif pi.type == 'enums':
709             autoGenEnumsPage(baseDir, pi, file)
710         elif pi.type == 'flags':
711             autoGenFlagsPage(baseDir, pi.name)
712         else:
713             # Do not extract this page
714             logWarn('genRef: Cannot extract or autogenerate:', pi.name)
715 
716         pages[pi.name] = pi
717         for alias in pi.alias.split():
718             pages[alias] = pi
719 
720     return pages
721 
722 
723 def genSinglePageRef(baseDir):
724     """Generate the single-page version of the ref pages.
725 
726     This assumes there is a page for everything in the api module dictionaries.
727     Extensions (KHR, EXT, etc.) are currently skipped"""
728     # Accumulate head of page
729     head = io.StringIO()
730 
731     printCopyrightSourceComments(head)
732 
733     print('= ' + apiName + ' API Reference Pages',
734           ':data-uri:',
735           ':icons: font',
736           ':doctype: book',
737           ':numbered!:',
738           ':max-width: 200',
739           ':data-uri:',
740           ':toc2:',
741           ':toclevels: 2',
742           ':attribute-missing: warn',
743           '',
744           sep='\n', file=head)
745 
746     print('== Copyright', file=head)
747     print('', file=head)
748     print('include::{config}/copyright-ccby' + conventions.file_suffix + '[]', file=head)
749     print('', file=head)
750 
751     # Inject the table of contents. Asciidoc really ought to be generating
752     # this for us.
753 
754     sections = [
755         [api.protos,       'protos',       apiName + ' Commands'],
756         [api.handles,      'handles',      'Object Handles'],
757         [api.structs,      'structs',      'Structures'],
758         [api.enums,        'enums',        'Enumerations'],
759         [api.flags,        'flags',        'Flags'],
760         [api.funcpointers, 'funcpointers', 'Function Pointer Types'],
761         [api.basetypes,    'basetypes',    apiName + ' Scalar types'],
762         [api.defines,      'defines',      'C Macro Definitions'],
763         [extensions,       'extensions',   apiName + ' Extensions']
764     ]
765 
766     # Accumulate body of page
767     body = io.StringIO()
768 
769     for (apiDict, label, title) in sections:
770         # Add section title/anchor header to body
771         anchor = '[[' + label + ',' + title + ']]'
772         print(anchor,
773               '== ' + title,
774               '',
775               ':leveloffset: 2',
776               '',
777               sep='\n', file=body)
778 
779         if label == 'extensions':
780             # preserve order of extensions since we already sorted the way we want.
781             keys = apiDict.keys()
782         else:
783             keys = sorted(apiDict.keys())
784 
785         for refPage in keys:
786             # Do not generate links for aliases, which are included with the
787             # aliased page
788             if refPage not in api.alias:
789                 # Add page to body
790                 if 'FlagBits' in refPage and conventions.unified_flag_refpages:
791                     # OpenXR does not create separate ref pages for FlagBits:
792                     # the FlagBits includes go in the Flags refpage.
793                     # Previously the Vulkan script would only emit non-empty
794                     # Vk*Flags pages, via the logic
795                     #   if refPage not in api.flags or api.flags[refPage] is not None
796                     #       emit page
797                     # Now, all are emitted.
798                     continue
799                 else:
800                     print(f'include::{refPage}{conventions.file_suffix}[]', file=body)
801             else:
802                 # Alternatively, we could (probably should) link to the
803                 # aliased refpage
804                 logWarn('(Benign) Not including', refPage,
805                         'in single-page reference',
806                         'because it is an alias of', api.alias[refPage])
807 
808         print('\n' + ':leveloffset: 0' + '\n', file=body)
809 
810     # Write head and body to the output file
811     pageName = f'{baseDir}/apispec{conventions.file_suffix}'
812     fp = open(pageName, 'w', encoding='utf-8')
813 
814     print(head.getvalue(), file=fp, end='')
815     print(body.getvalue(), file=fp, end='')
816 
817     head.close()
818     body.close()
819     fp.close()
820 
821 
822 def genExtension(baseDir, extpath, name, info):
823     """Generate refpage, and add dictionary entry for an extension
824 
825     - baseDir - output directory to generate page in
826     - extpath - None, or path to per-extension specification sources if
827                 those are to be included in extension refpages
828     - name - extension name
829     - info - <extension> Element from XML"""
830 
831     # Add a dictionary entry for this page
832     global genDict
833     genDict[name] = None
834     declares = []
835     elem = info.elem
836 
837     # Type of extension (instance, device, etc.)
838     ext_type = elem.get('type')
839 
840     # Autogenerate interfaces from <extension> entry
841     for required in elem.find('require'):
842         req_name = required.get('name')
843         if not req_name:
844             # This is not what we are looking for
845             continue
846         if req_name.endswith('_SPEC_VERSION') or req_name.endswith('_EXTENSION_NAME'):
847             # Do not link to spec version or extension name - those ref pages are not created.
848             continue
849 
850         if required.get('extends'):
851             # These are either extensions of enumerated types, or const enum
852             # values: neither of which get a ref page - although we could
853             # include the enumerated types in the See Also list.
854             continue
855 
856         if req_name not in genDict:
857             if req_name in api.alias:
858                 logWarn(f'WARN: {req_name} (in extension {name}) is an alias, so does not have a ref page')
859             else:
860                 logWarn(f'ERROR: {req_name} (in extension {name}) does not have a ref page.')
861 
862         declares.append(req_name)
863 
864     appbody = None
865     tail_content = None
866     if extpath is not None:
867         try:
868             appPath = extpath + '/' + conventions.extension_file_path(name)
869             appfp = open(appPath, 'r', encoding='utf-8')
870             appbody = appfp.read()
871             appfp.close()
872 
873             # Transform internal links to crosslinks
874             specURL = conventions.specURL()
875             appbody = xrefRewrite(appbody, specURL)
876         except FileNotFoundError:
877             print('Cannot find extension appendix for', name)
878             logWarn('Cannot find extension appendix for', name)
879 
880             # Fall through to autogenerated page
881             extpath = None
882             appbody = None
883 
884             appbody = f'Cannot find extension appendix {appPath} for {name}\n'
885     else:
886         tail_content = makeExtensionInclude(name)
887 
888     # Write the extension refpage
889     pageName = f'{baseDir}/{name}{conventions.file_suffix}'
890     logDiag('genExtension:', pageName)
891     fp = open(pageName, 'w', encoding='utf-8')
892 
893     # There are no generated titled sections
894     sections = None
895 
896     refPageShell(name,
897                  "{} extension".format(ext_type),
898                  fp,
899                  appbody,
900                  sections=sections,
901                  tail_content=tail_content)
902 
903     # Restore leveloffset for boilerplate in refPageTail
904     if conventions.include_extension_appendix_in_refpage:
905         # The generated metadata include (refpage.extensionname.adoc) moved
906         # the leveloffset attribute by -1 to account for the relative
907         # structuring of the spec extension appendix section structure vs.
908         # the refpages.
909         # This restores leveloffset for the boilerplate in refPageTail.
910         leveloffset = 1
911     else:
912         leveloffset = 0
913 
914     refPageTail(pageName=name,
915                 specType=None,
916                 specAnchor=name,
917                 seeAlso=seeAlsoList(name, declares),
918                 fp=fp,
919                 auto=True,
920                 leveloffset=leveloffset)
921     fp.close()
922 
923 
924 if __name__ == '__main__':
925     global genDict, extensions, conventions, apiName
926     genDict = {}
927     extensions = OrderedDict()
928     conventions = APIConventions()
929     apiName = conventions.api_name('api')
930 
931     parser = argparse.ArgumentParser()
932 
933     parser.add_argument('-diag', action='store', dest='diagFile',
934                         help='Set the diagnostic file')
935     parser.add_argument('-warn', action='store', dest='warnFile',
936                         help='Set the warning file')
937     parser.add_argument('-log', action='store', dest='logFile',
938                         help='Set the log file for both diagnostics and warnings')
939     parser.add_argument('-genpath', action='store',
940                         default='gen',
941                         help='Path to directory containing generated files')
942     parser.add_argument('-basedir', action='store', dest='baseDir',
943                         default=None,
944                         help='Set the base directory in which pages are generated')
945     parser.add_argument('-noauto', action='store_true',
946                         help='Don\'t generate inferred ref pages automatically')
947     parser.add_argument('files', metavar='filename', nargs='*',
948                         help='a filename to extract ref pages from')
949     parser.add_argument('--version', action='version', version='%(prog)s 1.0')
950     parser.add_argument('-extension', action='append',
951                         default=[],
952                         help='Specify an extension or extensions to add to targets')
953     parser.add_argument('-rewrite', action='store',
954                         default=None,
955                         help='Name of output file to write Apache mod_rewrite directives to')
956     parser.add_argument('-toc', action='store',
957                         default=None,
958                         help='Name of output file to write an alphabetical TOC to')
959     parser.add_argument('-registry', action='store',
960                         default=conventions.registry_path,
961                         help='Use specified registry file instead of default')
962     parser.add_argument('-extpath', action='store',
963                         default=None,
964                         help='Use extension descriptions from this directory instead of autogenerating extension refpages')
965 
966     results = parser.parse_args()
967 
968     # Load the generated apimap module
969     sys.path.insert(0, results.genpath)
970     import apimap as api
971 
972     setLogFile(True,  True, results.logFile)
973     setLogFile(True, False, results.diagFile)
974     setLogFile(False, True, results.warnFile)
975 
976     # Initialize static rewrite patterns for spec xrefs
977     xrefRewriteInitialize()
978 
979     if results.baseDir is None:
980         baseDir = results.genpath + '/ref'
981     else:
982         baseDir = results.baseDir
983 
984     # Dictionary of pages & aliases
985     pages = {}
986 
987     for file in results.files:
988         d = genRef(file, baseDir)
989         pages.update(d)
990 
991     # Now figure out which pages were not generated from the spec.
992     # This relies on the dictionaries of API constructs in the api module.
993 
994     if not results.noauto:
995         # Must have an apiname selected to avoid complaints from
996         # registry.loadFile, even though it is irrelevant to our uses.
997         genOpts = GeneratorOptions(apiname = conventions.xml_api_name)
998         registry = Registry(genOpts = genOpts)
999         registry.loadFile(results.registry)
1000 
1001         if conventions.write_refpage_include:
1002             # Only extensions with a supported="..." attribute in this set
1003             # will be considered for extraction/generation.
1004             ext_names = set(k for k, v in registry.extdict.items()
1005                             if conventions.xml_api_name in v.supported.split(','))
1006 
1007             desired_extensions = ext_names.intersection(set(results.extension))
1008             for prefix in conventions.extension_index_prefixes:
1009                 # Splits up into chunks, sorted within each chunk.
1010                 filtered_extensions = sorted(
1011                     [name for name in desired_extensions
1012                      if name.startswith(prefix) and name not in extensions])
1013                 for name in filtered_extensions:
1014                     # logWarn('NOT autogenerating extension refpage for', name)
1015                     extensions[name] = None
1016                     genExtension(baseDir, results.extpath, name, registry.extdict[name])
1017 
1018         # autoGenFlagsPage is no longer needed because they are added to
1019         # the spec sources now.
1020         # for page in api.flags:
1021         #     if page not in genDict:
1022         #         autoGenFlagsPage(baseDir, page)
1023 
1024         # autoGenHandlePage is no longer needed because they are added to
1025         # the spec sources now.
1026         # for page in api.structs:
1027         #    if typeCategory[page] == 'handle':
1028         #        autoGenHandlePage(baseDir, page)
1029 
1030         sections = [
1031             (api.flags,        'Flag Types'),
1032             (api.enums,        'Enumerated Types'),
1033             (api.structs,      'Structures'),
1034             (api.protos,       'Prototypes'),
1035             (api.funcpointers, 'Function Pointers'),
1036             (api.basetypes,    apiName + ' Scalar Types'),
1037             (extensions,       apiName + ' Extensions'),
1038         ]
1039 
1040         # Summarize pages that were not generated, for good or bad reasons
1041 
1042         for (apiDict, title) in sections:
1043             # OpenXR was keeping a 'flagged' state which only printed out a
1044             # warning for the first non-generated page, but was otherwise
1045             # unused. This does not seem helpful.
1046             for page in apiDict:
1047                 if page not in genDict:
1048                     # Page was not generated - why not?
1049                     if page in api.alias:
1050                         logDiag('(Benign, is an alias) Ref page for', title, page, 'is aliased into', api.alias[page])
1051                     elif page in api.flags and api.flags[page] is None:
1052                         logDiag('(Benign, no FlagBits defined) No ref page generated for ', title,
1053                                 page)
1054                     else:
1055                         # Could introduce additional logic to detect
1056                         # external types and not emit them.
1057                         logWarn('No ref page generated for  ', title, page)
1058 
1059         genSinglePageRef(baseDir)
1060 
1061     if results.rewrite:
1062         # Generate Apache rewrite directives for refpage aliases
1063         fp = open(results.rewrite, 'w', encoding='utf-8')
1064 
1065         for page in sorted(pages):
1066             p = pages[page]
1067             rewrite = p.name
1068 
1069             if page != rewrite:
1070                 print('RewriteRule ^', page, '.html$ ', rewrite, '.html',
1071                       sep='', file=fp)
1072         fp.close()
1073 
1074     if results.toc:
1075         # Generate dynamic portion of refpage TOC
1076         fp = open(results.toc, 'w', encoding='utf-8')
1077 
1078         # Run through dictionary of pages generating an TOC
1079         print(12 * ' ', '<li class="Level1">Alphabetic Contents', sep='', file=fp)
1080         print(16 * ' ', '<ul class="Level2">', sep='', file=fp)
1081         lastLetter = None
1082 
1083         for page in sorted(pages, key=str.upper):
1084             p = pages[page]
1085             letter = page[0:1].upper()
1086 
1087             if letter != lastLetter:
1088                 if lastLetter:
1089                     # End previous block
1090                     print(24 * ' ', '</ul>', sep='', file=fp)
1091                     print(20 * ' ', '</li>', sep='', file=fp)
1092                 # Start new block
1093                 print(20 * ' ', '<li>', letter, sep='', file=fp)
1094                 print(24 * ' ', '<ul class="Level3">', sep='', file=fp)
1095                 lastLetter = letter
1096 
1097             # Add this page to the list
1098             print(28 * ' ', '<li><a href="', p.name, '.html" ',
1099                   'target="pagedisplay">', page, '</a></li>',
1100                   sep='', file=fp)
1101 
1102         if lastLetter:
1103             # Close the final letter block
1104             print(24 * ' ', '</ul>', sep='', file=fp)
1105             print(20 * ' ', '</li>', sep='', file=fp)
1106 
1107         # Close the list
1108         print(16 * ' ', '</ul>', sep='', file=fp)
1109         print(12 * ' ', '</li>', sep='', file=fp)
1110 
1111         # print('name {} -> page {}'.format(page, pages[page].name))
1112 
1113         fp.close()
1114