1#!/usr/bin/python3 2# 3# Copyright 2016-2021 The Khronos Group Inc. 4# 5# SPDX-License-Identifier: Apache-2.0 6 7# genRef.py - create API ref pages from spec source files 8# 9# Usage: genRef.py files 10 11import argparse 12import io 13import os 14import re 15import sys 16from collections import OrderedDict 17from reflib import (findRefs, fixupRefs, loadFile, logDiag, logWarn, 18 printPageInfo, setLogFile) 19from reg import Registry 20from vkconventions import VulkanConventions as APIConventions 21 22 23# refpage 'type' attributes which are API entities and contain structured 24# content such as API includes, valid usage blocks, etc. 25refpage_api_types = ( 26 'basetypes', 27 'consts', 28 'defines', 29 'enums', 30 'flags', 31 'funcpointers', 32 'handles', 33 'protos', 34 'structs', 35) 36 37# Other refpage types - SPIR-V builtins, API feature blocks, etc. - which do 38# not have structured content. 39refpage_other_types = ( 40 'builtins', 41 'feature', 42 'freeform', 43 'spirv' 44) 45 46def makeExtensionInclude(name): 47 """Return an include command, given an extension name.""" 48 return 'include::{}/refpage.{}{}[]'.format( 49 conventions.specification_path, 50 name, 51 conventions.file_suffix) 52 53 54def makeAPIInclude(type, name): 55 """Return an include command for a generated API interface 56 - type - type of the API, e.g. 'flags', 'handles', etc 57 - name - name of the API""" 58 59 return 'include::{}/api/{}/{}{}\n'.format( 60 conventions.refpage_generated_include_path, 61 type, name, conventions.file_suffix) 62 63 64def isextension(name): 65 """Return True if name is an API extension name (ends with an upper-case 66 author ID). 67 68 This assumes that author IDs are at least two characters.""" 69 return name[-2:].isalpha() and name[-2:].isupper() 70 71 72def printCopyrightSourceComments(fp): 73 """Print Khronos CC-BY copyright notice on open file fp. 74 75 Writes an asciidoc comment block, which copyrights the source 76 file.""" 77 print('// Copyright 2014-2021 The Khronos Group, Inc.', file=fp) 78 print('//', file=fp) 79 # This works around constraints of the 'reuse' tool 80 print('// SPDX' + '-License-Identifier: CC-BY-4.0', file=fp) 81 print('', file=fp) 82 83 84def printFooter(fp, leveloffset=0): 85 """Print footer material at the end of each refpage on open file fp. 86 87 If generating separate refpages, adds the copyright. 88 If generating the single combined refpage, just add a separator. 89 90 - leveloffset - number of levels to bias section titles up or down.""" 91 92 # Generate the section header. 93 # Default depth is 2. 94 depth = max(0, leveloffset + 2) 95 prefix = '=' * depth 96 97 print('ifdef::doctype-manpage[]', 98 f'{prefix} Copyright', 99 '', 100 'include::{config}/copyright-ccby.txt[]', 101 'endif::doctype-manpage[]', 102 '', 103 'ifndef::doctype-manpage[]', 104 '<<<', 105 'endif::doctype-manpage[]', 106 '', 107 sep='\n', file=fp) 108 109 110def macroPrefix(name): 111 """Add a spec asciidoc macro prefix to an API name, depending on its type 112 (protos, structs, enums, etc.). 113 114 If the name is not recognized, use the generic link macro 'reflink:'.""" 115 if name in api.basetypes: 116 return 'basetype:' + name 117 if name in api.defines: 118 return 'dlink:' + name 119 if name in api.enums: 120 return 'elink:' + name 121 if name in api.flags: 122 return 'tlink:' + name 123 if name in api.funcpointers: 124 return 'tlink:' + name 125 if name in api.handles: 126 return 'slink:' + name 127 if name in api.protos: 128 return 'flink:' + name 129 if name in api.structs: 130 return 'slink:' + name 131 if name == 'TBD': 132 return 'No cross-references are available' 133 return 'reflink:' + name 134 135 136def seeAlsoList(apiName, explicitRefs=None, apiAliases=[]): 137 """Return an asciidoc string with a list of 'See Also' references for the 138 API entity 'apiName', based on the relationship mapping in the api module. 139 140 'explicitRefs' is a list of additional cross-references. 141 142 If apiAliases is not None, it is a list of aliases of apiName whose 143 cross-references will also be included. 144 145 If no relationships are available, return None.""" 146 147 refs = set(()) 148 149 # apiName and its aliases are treated equally 150 allApis = apiAliases.copy() 151 allApis.append(apiName) 152 153 # Add all the implicit references to refs 154 for name in allApis: 155 if name in api.mapDict: 156 refs.update(api.mapDict[name]) 157 158 # Add all the explicit references 159 if explicitRefs is not None: 160 if isinstance(explicitRefs, str): 161 explicitRefs = explicitRefs.split() 162 refs.update(name for name in explicitRefs) 163 164 # Add extensions / core versions based on dependencies 165 for name in allApis: 166 if name in api.requiredBy: 167 for (base,dependency) in api.requiredBy[name]: 168 refs.add(base) 169 if dependency is not None: 170 refs.add(dependency) 171 172 if len(refs) == 0: 173 return None 174 else: 175 return ', '.join(macroPrefix(name) for name in sorted(refs)) + '\n' 176 177 178def remapIncludes(lines, baseDir, specDir): 179 """Remap include directives in a list of lines so they can be extracted to a 180 different directory. 181 182 Returns remapped lines. 183 184 - lines - text to remap 185 - baseDir - target directory 186 - specDir - source directory""" 187 # This should be compiled only once 188 includePat = re.compile(r'^include::(?P<path>.*)\[\]') 189 190 newLines = [] 191 for line in lines: 192 matches = includePat.search(line) 193 if matches is not None: 194 path = matches.group('path') 195 196 if path[0] != '{': 197 # Relative path to include file from here 198 incPath = specDir + '/' + path 199 # Remap to be relative to baseDir 200 newPath = os.path.relpath(incPath, baseDir) 201 newLine = 'include::' + newPath + '[]\n' 202 logDiag('remapIncludes: remapping', line, '->', newLine) 203 newLines.append(newLine) 204 else: 205 # An asciidoctor variable starts the path. 206 # This must be an absolute path, not needing to be rewritten. 207 newLines.append(line) 208 else: 209 newLines.append(line) 210 return newLines 211 212 213def refPageShell(pageName, pageDesc, fp, head_content = None, sections=None, tail_content=None, man_section=3): 214 """Generate body of a reference page. 215 216 - pageName - string name of the page 217 - pageDesc - string short description of the page 218 - fp - file to write to 219 - head_content - text to include before the sections 220 - sections - iterable returning (title,body) for each section. 221 - tail_content - text to include after the sections 222 - man_section - Unix man page section""" 223 224 printCopyrightSourceComments(fp) 225 226 print(':data-uri:', 227 ':icons: font', 228 conventions.extra_refpage_headers, 229 '', 230 sep='\n', file=fp) 231 232 s = '{}({})'.format(pageName, man_section) 233 print('= ' + s, 234 '', 235 sep='\n', file=fp) 236 if pageDesc.strip() == '': 237 pageDesc = 'NO SHORT DESCRIPTION PROVIDED' 238 logWarn('refPageHead: no short description provided for', pageName) 239 240 print('== Name', 241 '{} - {}'.format(pageName, pageDesc), 242 '', 243 sep='\n', file=fp) 244 245 if head_content is not None: 246 print(head_content, 247 '', 248 sep='\n', file=fp) 249 250 if sections is not None: 251 for title, content in sections.items(): 252 print('== {}'.format(title), 253 '', 254 content, 255 '', 256 sep='\n', file=fp) 257 258 if tail_content is not None: 259 print(tail_content, 260 '', 261 sep='\n', file=fp) 262 263 264def refPageHead(pageName, pageDesc, specText, fieldName, fieldText, descText, fp): 265 """Generate header of a reference page. 266 267 - pageName - string name of the page 268 - pageDesc - string short description of the page 269 - specType - string containing 'spec' field from refpage open block, or None. 270 Used to determine containing spec name and URL. 271 - specText - string that goes in the "C Specification" section 272 - fieldName - string heading an additional section following specText, if not None 273 - fieldText - string that goes in the additional section 274 - descText - string that goes in the "Description" section 275 - fp - file to write to""" 276 sections = OrderedDict() 277 278 if specText is not None: 279 sections['C Specification'] = specText 280 281 if fieldName is not None: 282 sections[fieldName] = fieldText 283 284 if descText is None or descText.strip() == '': 285 logWarn('refPageHead: no description provided for', pageName) 286 287 if descText is not None: 288 sections['Description'] = descText 289 290 refPageShell(pageName, pageDesc, fp, head_content=None, sections=sections) 291 292 293def refPageTail(pageName, 294 specType=None, 295 specAnchor=None, 296 seeAlso=None, 297 fp=None, 298 auto=False, 299 leveloffset=0): 300 """Generate end boilerplate of a reference page. 301 302 - pageName - name of the page 303 - specType - None or the 'spec' attribute from the refpage block, 304 identifying the specification name and URL this refpage links to. 305 - specAnchor - None or the 'anchor' attribute from the refpage block, 306 identifying the anchor in the specification this refpage links to. If 307 None, the pageName is assumed to be a valid anchor. 308 - seeAlso - text of the "See Also" section 309 - fp - file to write the page to 310 - auto - True if this is an entirely generated refpage, False if it's 311 handwritten content from the spec. 312 - leveloffset - number of levels to bias section titles up or down.""" 313 314 specName = conventions.api_name(specType) 315 specURL = conventions.specURL(specType) 316 if specAnchor is None: 317 specAnchor = pageName 318 319 if seeAlso is None: 320 seeAlso = 'No cross-references are available\n' 321 322 notes = [ 323 'For more information, see the {}#{}[{} Specification^]'.format( 324 specURL, specAnchor, specName), 325 '', 326 ] 327 328 if auto: 329 notes.extend(( 330 'This page is a generated document.', 331 'Fixes and changes should be made to the generator scripts, ' 332 'not directly.', 333 )) 334 else: 335 notes.extend(( 336 'This page is extracted from the ' + specName + ' Specification. ', 337 'Fixes and changes should be made to the Specification, ' 338 'not directly.', 339 )) 340 341 # Generate the section header. 342 # Default depth is 2. 343 depth = max(0, leveloffset + 2) 344 prefix = '=' * depth 345 346 print(f'{prefix} See Also', 347 '', 348 seeAlso, 349 '', 350 sep='\n', file=fp) 351 352 print(f'{prefix} Document Notes', 353 '', 354 '\n'.join(notes), 355 '', 356 sep='\n', file=fp) 357 358 printFooter(fp, leveloffset) 359 360 361def xrefRewriteInitialize(): 362 """Initialize substitution patterns for asciidoctor xrefs.""" 363 364 global refLinkPattern, refLinkSubstitute 365 global refLinkTextPattern, refLinkTextSubstitute 366 global specLinkPattern, specLinkSubstitute 367 368 # These are xrefs to Vulkan API entities, rewritten to link to refpages 369 # The refLink variants are for xrefs with only an anchor and no text. 370 # The refLinkText variants are for xrefs with both anchor and text 371 refLinkPattern = re.compile(r'<<([Vv][Kk][^>,]+)>>') 372 refLinkSubstitute = r'link:\1.html[\1^]' 373 374 refLinkTextPattern = re.compile(r'<<([Vv][Kk][^>,]+)[,]?[ \t\n]*([^>,]*)>>') 375 refLinkTextSubstitute = r'link:\1.html[\2^]' 376 377 # These are xrefs to other anchors, rewritten to link to the spec 378 specLinkPattern = re.compile(r'<<([^>,]+)[,]?[ \t\n]*([^>,]*)>>') 379 380 # Unfortunately, specLinkSubstitute depends on the link target, 381 # so can't be constructed in advance. 382 specLinkSubstitute = None 383 384 385def xrefRewrite(text, specURL): 386 """Rewrite asciidoctor xrefs in text to resolve properly in refpages. 387 Xrefs which are to Vulkan refpages are rewritten to link to those 388 refpages. The remainder are rewritten to generate external links into 389 the supplied specification document URL. 390 391 - text - string to rewrite, or None 392 - specURL - URL to target 393 394 Returns rewritten text, or None, respectively""" 395 396 global refLinkPattern, refLinkSubstitute 397 global refLinkTextPattern, refLinkTextSubstitute 398 global specLinkPattern, specLinkSubstitute 399 400 specLinkSubstitute = r'link:{}#\1[\2^]'.format(specURL) 401 402 if text is not None: 403 text, _ = refLinkPattern.subn(refLinkSubstitute, text) 404 text, _ = refLinkTextPattern.subn(refLinkTextSubstitute, text) 405 text, _ = specLinkPattern.subn(specLinkSubstitute, text) 406 407 return text 408 409def emitPage(baseDir, specDir, pi, file): 410 """Extract a single reference page into baseDir. 411 412 - baseDir - base directory to emit page into 413 - specDir - directory extracted page source came from 414 - pi - pageInfo for this page relative to file 415 - file - list of strings making up the file, indexed by pi""" 416 pageName = baseDir + '/' + pi.name + '.txt' 417 418 # Add a dictionary entry for this page 419 global genDict 420 genDict[pi.name] = None 421 logDiag('emitPage:', pageName) 422 423 # Short description 424 if pi.desc is None: 425 pi.desc = '(no short description available)' 426 427 # Member/parameter section label and text, if there is one 428 field = None 429 fieldText = None 430 431 # Only do structural checks on API pages 432 if pi.type in refpage_api_types: 433 if pi.include is None: 434 logWarn('emitPage:', pageName, 'INCLUDE is None, no page generated') 435 return 436 437 # Specification text 438 lines = remapIncludes(file[pi.begin:pi.include + 1], baseDir, specDir) 439 specText = ''.join(lines) 440 441 if pi.param is not None: 442 if pi.type == 'structs': 443 field = 'Members' 444 elif pi.type in ['protos', 'funcpointers']: 445 field = 'Parameters' 446 else: 447 logWarn('emitPage: unknown field type:', pi.type, 448 'for', pi.name) 449 lines = remapIncludes(file[pi.param:pi.body], baseDir, specDir) 450 fieldText = ''.join(lines) 451 452 # Description text 453 if pi.body != pi.include: 454 lines = remapIncludes(file[pi.body:pi.end + 1], baseDir, specDir) 455 descText = ''.join(lines) 456 else: 457 descText = None 458 logWarn('emitPage: INCLUDE == BODY, so description will be empty for', pi.name) 459 if pi.begin != pi.include: 460 logWarn('emitPage: Note: BEGIN != INCLUDE, so the description might be incorrectly located before the API include!') 461 elif pi.type in refpage_other_types: 462 specText = None 463 descText = ''.join(file[pi.begin:pi.end + 1]) 464 else: 465 # This should be caught in the spec markup checking tests 466 logErr(f"emitPage: refpage type='{pi.type}' is unrecognized") 467 468 # Rewrite asciidoctor xrefs to resolve properly in refpages 469 specURL = conventions.specURL(pi.spec) 470 471 specText = xrefRewrite(specText, specURL) 472 fieldText = xrefRewrite(fieldText, specURL) 473 descText = xrefRewrite(descText, specURL) 474 475 fp = open(pageName, 'w', encoding='utf-8') 476 refPageHead(pi.name, 477 pi.desc, 478 specText, 479 field, fieldText, 480 descText, 481 fp) 482 refPageTail(pageName=pi.name, 483 specType=pi.spec, 484 specAnchor=pi.anchor, 485 seeAlso=seeAlsoList(pi.name, pi.refs, pi.alias.split()), 486 fp=fp, 487 auto=False) 488 fp.close() 489 490 491def autoGenEnumsPage(baseDir, pi, file): 492 """Autogenerate a single reference page in baseDir. 493 494 Script only knows how to do this for /enums/ pages, at present. 495 496 - baseDir - base directory to emit page into 497 - pi - pageInfo for this page relative to file 498 - file - list of strings making up the file, indexed by pi""" 499 pageName = baseDir + '/' + pi.name + '.txt' 500 fp = open(pageName, 'w', encoding='utf-8') 501 502 # Add a dictionary entry for this page 503 global genDict 504 genDict[pi.name] = None 505 logDiag('autoGenEnumsPage:', pageName) 506 507 # Short description 508 if pi.desc is None: 509 pi.desc = '(no short description available)' 510 511 # Description text. Allow for the case where an enum definition 512 # is not embedded. 513 if not pi.embed: 514 embedRef = '' 515 else: 516 embedRef = ''.join(( 517 ' * The reference page for ', 518 macroPrefix(pi.embed), 519 ', where this interface is defined.\n')) 520 521 txt = ''.join(( 522 'For more information, see:\n\n', 523 embedRef, 524 ' * The See Also section for other reference pages using this type.\n', 525 ' * The ' + apiName + ' Specification.\n')) 526 527 refPageHead(pi.name, 528 pi.desc, 529 ''.join(file[pi.begin:pi.include + 1]), 530 None, None, 531 txt, 532 fp) 533 refPageTail(pageName=pi.name, 534 specType=pi.spec, 535 specAnchor=pi.anchor, 536 seeAlso=seeAlsoList(pi.name, pi.refs, pi.alias.split()), 537 fp=fp, 538 auto=True) 539 fp.close() 540 541 542# Pattern to break apart an API *Flags{authorID} name, used in 543# autoGenFlagsPage. 544flagNamePat = re.compile(r'(?P<name>\w+)Flags(?P<author>[A-Z]*)') 545 546 547def autoGenFlagsPage(baseDir, flagName): 548 """Autogenerate a single reference page in baseDir for an API *Flags type. 549 550 - baseDir - base directory to emit page into 551 - flagName - API *Flags name""" 552 pageName = baseDir + '/' + flagName + '.txt' 553 fp = open(pageName, 'w', encoding='utf-8') 554 555 # Add a dictionary entry for this page 556 global genDict 557 genDict[flagName] = None 558 logDiag('autoGenFlagsPage:', pageName) 559 560 # Short description 561 matches = flagNamePat.search(flagName) 562 if matches is not None: 563 name = matches.group('name') 564 author = matches.group('author') 565 logDiag('autoGenFlagsPage: split name into', name, 'Flags', author) 566 flagBits = name + 'FlagBits' + author 567 desc = 'Bitmask of ' + flagBits 568 else: 569 logWarn('autoGenFlagsPage:', pageName, 'does not end in "Flags{author ID}". Cannot infer FlagBits type.') 570 flagBits = None 571 desc = 'Unknown ' + apiName + ' flags type' 572 573 # Description text 574 if flagBits is not None: 575 txt = ''.join(( 576 'etext:' + flagName, 577 ' is a mask of zero or more elink:' + flagBits + '.\n', 578 'It is used as a member and/or parameter of the structures and commands\n', 579 'in the See Also section below.\n')) 580 else: 581 txt = ''.join(( 582 'etext:' + flagName, 583 ' is an unknown ' + apiName + ' type, assumed to be a bitmask.\n')) 584 585 refPageHead(flagName, 586 desc, 587 makeAPIInclude('flags', flagName), 588 None, None, 589 txt, 590 fp) 591 refPageTail(pageName=flagName, 592 specType=pi.spec, 593 specAnchor=pi.anchor, 594 seeAlso=seeAlsoList(flagName, None), 595 fp=fp, 596 auto=True) 597 fp.close() 598 599 600def autoGenHandlePage(baseDir, handleName): 601 """Autogenerate a single handle page in baseDir for an API handle type. 602 603 - baseDir - base directory to emit page into 604 - handleName - API handle name""" 605 # @@ Need to determine creation function & add handles/ include for the 606 # @@ interface in generator.py. 607 pageName = baseDir + '/' + handleName + '.txt' 608 fp = open(pageName, 'w', encoding='utf-8') 609 610 # Add a dictionary entry for this page 611 global genDict 612 genDict[handleName] = None 613 logDiag('autoGenHandlePage:', pageName) 614 615 # Short description 616 desc = apiName + ' object handle' 617 618 descText = ''.join(( 619 'sname:' + handleName, 620 ' is an object handle type, referring to an object used\n', 621 'by the ' + apiName + ' implementation. These handles are created or allocated\n', 622 'by the @@ TBD @@ function, and used by other ' + apiName + ' structures\n', 623 'and commands in the See Also section below.\n')) 624 625 refPageHead(handleName, 626 desc, 627 makeAPIInclude('handles', handleName), 628 None, None, 629 descText, 630 fp) 631 refPageTail(pageName=handleName, 632 specType=pi.spec, 633 specAnchor=pi.anchor, 634 seeAlso=seeAlsoList(handleName, None), 635 fp=fp, 636 auto=True) 637 fp.close() 638 639 640def genRef(specFile, baseDir): 641 """Extract reference pages from a spec asciidoc source file. 642 643 - specFile - filename to extract from 644 - baseDir - output directory to generate page in""" 645 file = loadFile(specFile) 646 if file is None: 647 return 648 649 # Save the path to this file for later use in rewriting relative includes 650 specDir = os.path.dirname(os.path.abspath(specFile)) 651 652 pageMap = findRefs(file, specFile) 653 logDiag(specFile + ': found', len(pageMap.keys()), 'potential pages') 654 655 sys.stderr.flush() 656 657 # Fix up references in pageMap 658 fixupRefs(pageMap, specFile, file) 659 660 # Create each page, if possible 661 pages = {} 662 663 for name in sorted(pageMap): 664 pi = pageMap[name] 665 666 # Only generate the page if it's in the requested build 667 # 'freeform' pages are always generated 668 # 'feature' pages (core versions & extensions) are generated if they're in 669 # the requested feature list 670 # All other pages (APIs) are generated if they're in the API map for 671 # the build. 672 if pi.type in refpage_api_types: 673 if name not in api.typeCategory: 674 # Also check aliases of name - api.nonexistent is the same 675 # mapping used to rewrite *link: macros in this build. 676 if name not in api.nonexistent: 677 logWarn(f'genRef: NOT generating feature page {name} - API not in this build') 678 continue 679 else: 680 logWarn(f'genRef: generating feature page {name} because its alias {api.nonexistent[name]} exists') 681 elif pi.type in refpage_other_types: 682 # The only non-API type which can be checked is a feature refpage 683 if pi.type == 'feature': 684 if name not in api.features: 685 logWarn(f'genRef: NOT generating feature page {name} - feature not in this build') 686 continue 687 688 printPageInfo(pi, file) 689 690 if pi.Warning: 691 logDiag('genRef:', pi.name + ':', pi.Warning) 692 693 if pi.extractPage: 694 emitPage(baseDir, specDir, pi, file) 695 elif pi.type == 'enums': 696 autoGenEnumsPage(baseDir, pi, file) 697 elif pi.type == 'flags': 698 autoGenFlagsPage(baseDir, pi.name) 699 else: 700 # Don't extract this page 701 logWarn('genRef: Cannot extract or autogenerate:', pi.name) 702 703 pages[pi.name] = pi 704 for alias in pi.alias.split(): 705 pages[alias] = pi 706 707 return pages 708 709 710def genSinglePageRef(baseDir): 711 """Generate baseDir/apispec.txt, the single-page version of the ref pages. 712 713 This assumes there's a page for everything in the api module dictionaries. 714 Extensions (KHR, EXT, etc.) are currently skipped""" 715 # Accumulate head of page 716 head = io.StringIO() 717 718 printCopyrightSourceComments(head) 719 720 print('= ' + apiName + ' API Reference Pages', 721 ':data-uri:', 722 ':icons: font', 723 ':doctype: book', 724 ':numbered!:', 725 ':max-width: 200', 726 ':data-uri:', 727 ':toc2:', 728 ':toclevels: 2', 729 '', 730 sep='\n', file=head) 731 732 print('== Copyright', file=head) 733 print('', file=head) 734 print('include::{config}/copyright-ccby.txt[]', file=head) 735 print('', file=head) 736 # Inject the table of contents. Asciidoc really ought to be generating 737 # this for us. 738 739 sections = [ 740 [api.protos, 'protos', apiName + ' Commands'], 741 [api.handles, 'handles', 'Object Handles'], 742 [api.structs, 'structs', 'Structures'], 743 [api.enums, 'enums', 'Enumerations'], 744 [api.flags, 'flags', 'Flags'], 745 [api.funcpointers, 'funcpointers', 'Function Pointer Types'], 746 [api.basetypes, 'basetypes', apiName + ' Scalar types'], 747 [api.defines, 'defines', 'C Macro Definitions'], 748 [extensions, 'extensions', apiName + ' Extensions'] 749 ] 750 751 # Accumulate body of page 752 body = io.StringIO() 753 754 for (apiDict, label, title) in sections: 755 # Add section title/anchor header to body 756 anchor = '[[' + label + ',' + title + ']]' 757 print(anchor, 758 '== ' + title, 759 '', 760 ':leveloffset: 2', 761 '', 762 sep='\n', file=body) 763 764 if label == 'extensions': 765 # preserve order of extensions since we already sorted the way we want. 766 keys = apiDict.keys() 767 else: 768 keys = sorted(apiDict.keys()) 769 770 for refPage in keys: 771 # Don't generate links for aliases, which are included with the 772 # aliased page 773 if refPage not in api.alias: 774 # Add page to body 775 if 'FlagBits' in refPage and conventions.unified_flag_refpages: 776 # OpenXR does not create separate ref pages for FlagBits: 777 # the FlagBits includes go in the Flags refpage. 778 # Previously the Vulkan script would only emit non-empty 779 # Vk*Flags pages, via the logic 780 # if refPage not in api.flags or api.flags[refPage] is not None 781 # emit page 782 # Now, all are emitted. 783 continue 784 else: 785 print('include::' + refPage + '.txt[]', file=body) 786 else: 787 # Alternatively, we could (probably should) link to the 788 # aliased refpage 789 logWarn('(Benign) Not including', refPage, 790 'in single-page reference', 791 'because it is an alias of', api.alias[refPage]) 792 793 print('\n' + ':leveloffset: 0' + '\n', file=body) 794 795 # Write head and body to the output file 796 pageName = baseDir + '/apispec.txt' 797 fp = open(pageName, 'w', encoding='utf-8') 798 799 print(head.getvalue(), file=fp, end='') 800 print(body.getvalue(), file=fp, end='') 801 802 head.close() 803 body.close() 804 fp.close() 805 806 807def genExtension(baseDir, extpath, name, info): 808 """Generate refpage, and add dictionary entry for an extension 809 810 - baseDir - output directory to generate page in 811 - extpath - None, or path to per-extension specification sources if 812 those are to be included in extension refpages 813 - name - extension name 814 - info - <extension> Element from XML""" 815 816 # Add a dictionary entry for this page 817 global genDict 818 genDict[name] = None 819 declares = [] 820 elem = info.elem 821 822 # Type of extension (instance, device, etc.) 823 ext_type = elem.get('type') 824 825 # Autogenerate interfaces from <extension> entry 826 for required in elem.find('require'): 827 req_name = required.get('name') 828 if not req_name: 829 # This isn't what we're looking for 830 continue 831 if req_name.endswith('_SPEC_VERSION') or req_name.endswith('_EXTENSION_NAME'): 832 # Don't link to spec version or extension name - those ref pages aren't created. 833 continue 834 835 if required.get('extends'): 836 # These are either extensions of enumerated types, or const enum 837 # values: neither of which get a ref page - although we could 838 # include the enumerated types in the See Also list. 839 continue 840 841 if req_name not in genDict: 842 logWarn('ERROR: {} (in extension {}) does not have a ref page.'.format(req_name, name)) 843 844 declares.append(req_name) 845 846 # import pdb 847 # pdb.set_trace() 848 849 appbody = None 850 if extpath is not None: 851 appfp = open('{}/{}.txt'.format(extpath, name), 'r', encoding='utf-8') 852 if appfp is not None: 853 appbody = appfp.read() 854 855 # Transform internal links to crosslinks 856 specURL = conventions.specURL() 857 appbody = xrefRewrite(appbody, specURL) 858 else: 859 logWarn('Cannot find extension appendix for', name) 860 861 # Fall through to autogenerated page 862 extpath = None 863 appbody = None 864 appfp.close() 865 866 # Include the extension appendix without an extra title 867 # head_content = 'include::{{appendices}}/{}.txt[]'.format(name) 868 869 # Write the extension refpage 870 pageName = baseDir + '/' + name + '.txt' 871 logDiag('genExtension:', pageName) 872 fp = open(pageName, 'w', encoding='utf-8') 873 874 # There are no generated titled sections 875 sections = None 876 877 # 'See link:{html_spec_relative}#%s[ %s] in the main specification for complete information.' % ( 878 # name, name) 879 refPageShell(name, 880 "{} extension".format(ext_type), 881 fp, 882 appbody, 883 sections=sections) 884 885 # The generated metadata include moved the leveloffset attribute by -1 886 # to account for the relative structuring of the spec extension appendix 887 # section structure vs. the refpages. 888 # This restores leveloffset for the boilerplate in refPageTail. 889 refPageTail(pageName=name, 890 specType=None, 891 specAnchor=name, 892 seeAlso=seeAlsoList(name, declares), 893 fp=fp, 894 auto=True, 895 leveloffset=1) 896 fp.close() 897 898 899if __name__ == '__main__': 900 global genDict, extensions, conventions, apiName 901 genDict = {} 902 extensions = OrderedDict() 903 conventions = APIConventions() 904 apiName = conventions.api_name('api') 905 906 parser = argparse.ArgumentParser() 907 908 parser.add_argument('-diag', action='store', dest='diagFile', 909 help='Set the diagnostic file') 910 parser.add_argument('-warn', action='store', dest='warnFile', 911 help='Set the warning file') 912 parser.add_argument('-log', action='store', dest='logFile', 913 help='Set the log file for both diagnostics and warnings') 914 parser.add_argument('-genpath', action='store', 915 default='gen', 916 help='Path to directory containing generated files') 917 parser.add_argument('-basedir', action='store', dest='baseDir', 918 default=None, 919 help='Set the base directory in which pages are generated') 920 parser.add_argument('-noauto', action='store_true', 921 help='Don\'t generate inferred ref pages automatically') 922 parser.add_argument('files', metavar='filename', nargs='*', 923 help='a filename to extract ref pages from') 924 parser.add_argument('--version', action='version', version='%(prog)s 1.0') 925 parser.add_argument('-extension', action='append', 926 default=[], 927 help='Specify an extension or extensions to add to targets') 928 parser.add_argument('-rewrite', action='store', 929 default=None, 930 help='Name of output file to write Apache mod_rewrite directives to') 931 parser.add_argument('-toc', action='store', 932 default=None, 933 help='Name of output file to write an alphabetical TOC to') 934 parser.add_argument('-registry', action='store', 935 default=conventions.registry_path, 936 help='Use specified registry file instead of default') 937 parser.add_argument('-extpath', action='store', 938 default=None, 939 help='Use extension descriptions from this directory instead of autogenerating extension refpages') 940 941 results = parser.parse_args() 942 943 # Look for api.py in the specified directory 944 if results.genpath is not None: 945 sys.path.insert(0, results.genpath) 946 import api 947 948 setLogFile(True, True, results.logFile) 949 setLogFile(True, False, results.diagFile) 950 setLogFile(False, True, results.warnFile) 951 952 # Initialize static rewrite patterns for spec xrefs 953 xrefRewriteInitialize() 954 955 if results.baseDir is None: 956 baseDir = results.genpath + '/ref' 957 else: 958 baseDir = results.baseDir 959 960 # Dictionary of pages & aliases 961 pages = {} 962 963 for file in results.files: 964 d = genRef(file, baseDir) 965 pages.update(d) 966 967 # Now figure out which pages *weren't* generated from the spec. 968 # This relies on the dictionaries of API constructs in the api module. 969 970 if not results.noauto: 971 registry = Registry() 972 registry.loadFile(results.registry) 973 974 if conventions.write_refpage_include: 975 # Only extensions with a supported="..." attribute in this set 976 # will be considered for extraction/generation. 977 supported_strings = set((conventions.xml_api_name,)) 978 ext_names = set(k for k, v in registry.extdict.items() 979 if v.supported in supported_strings) 980 981 desired_extensions = ext_names.intersection(set(results.extension)) 982 for prefix in conventions.extension_index_prefixes: 983 # Splits up into chunks, sorted within each chunk. 984 filtered_extensions = sorted( 985 [name for name in desired_extensions 986 if name.startswith(prefix) and name not in extensions]) 987 for name in filtered_extensions: 988 # logWarn('NOT autogenerating extension refpage for', name) 989 extensions[name] = None 990 genExtension(baseDir, results.extpath, name, registry.extdict[name]) 991 992 # autoGenFlagsPage is no longer needed because they are added to 993 # the spec sources now. 994 # for page in api.flags: 995 # if page not in genDict: 996 # autoGenFlagsPage(baseDir, page) 997 998 # autoGenHandlePage is no longer needed because they are added to 999 # the spec sources now. 1000 # for page in api.structs: 1001 # if typeCategory[page] == 'handle': 1002 # autoGenHandlePage(baseDir, page) 1003 1004 sections = [ 1005 (api.flags, 'Flag Types'), 1006 (api.enums, 'Enumerated Types'), 1007 (api.structs, 'Structures'), 1008 (api.protos, 'Prototypes'), 1009 (api.funcpointers, 'Function Pointers'), 1010 (api.basetypes, apiName + ' Scalar Types'), 1011 (extensions, apiName + ' Extensions'), 1012 ] 1013 1014 # Summarize pages that weren't generated, for good or bad reasons 1015 1016 for (apiDict, title) in sections: 1017 # OpenXR was keeping a 'flagged' state which only printed out a 1018 # warning for the first non-generated page, but was otherwise 1019 # unused. This doesn't seem helpful. 1020 for page in apiDict: 1021 if page not in genDict: 1022 # Page was not generated - why not? 1023 if page in api.alias: 1024 logWarn('(Benign, is an alias) Ref page for', title, page, 'is aliased into', api.alias[page]) 1025 elif page in api.flags and api.flags[page] is None: 1026 logWarn('(Benign, no FlagBits defined) No ref page generated for ', title, 1027 page) 1028 else: 1029 # Could introduce additional logic to detect 1030 # external types and not emit them. 1031 logWarn('No ref page generated for ', title, page) 1032 1033 genSinglePageRef(baseDir) 1034 1035 if results.rewrite: 1036 # Generate Apache rewrite directives for refpage aliases 1037 fp = open(results.rewrite, 'w', encoding='utf-8') 1038 1039 for page in sorted(pages): 1040 p = pages[page] 1041 rewrite = p.name 1042 1043 if page != rewrite: 1044 print('RewriteRule ^', page, '.html$ ', rewrite, '.html', 1045 sep='', file=fp) 1046 fp.close() 1047 1048 if results.toc: 1049 # Generate dynamic portion of refpage TOC 1050 fp = open(results.toc, 'w', encoding='utf-8') 1051 1052 # Run through dictionary of pages generating an TOC 1053 print(12 * ' ', '<li class="Level1">Alphabetic Contents', sep='', file=fp) 1054 print(16 * ' ', '<ul class="Level2">', sep='', file=fp) 1055 lastLetter = None 1056 1057 for page in sorted(pages, key=str.upper): 1058 p = pages[page] 1059 letter = page[0:1].upper() 1060 1061 if letter != lastLetter: 1062 if lastLetter: 1063 # End previous block 1064 print(24 * ' ', '</ul>', sep='', file=fp) 1065 print(20 * ' ', '</li>', sep='', file=fp) 1066 # Start new block 1067 print(20 * ' ', '<li>', letter, sep='', file=fp) 1068 print(24 * ' ', '<ul class="Level3">', sep='', file=fp) 1069 lastLetter = letter 1070 1071 # Add this page to the list 1072 print(28 * ' ', '<li><a href="', p.name, '.html" ', 1073 'target="pagedisplay">', page, '</a></li>', 1074 sep='', file=fp) 1075 1076 if lastLetter: 1077 # Close the final letter block 1078 print(24 * ' ', '</ul>', sep='', file=fp) 1079 print(20 * ' ', '</li>', sep='', file=fp) 1080 1081 # Close the list 1082 print(16 * ' ', '</ul>', sep='', file=fp) 1083 print(12 * ' ', '</li>', sep='', file=fp) 1084 1085 # print('name {} -> page {}'.format(page, pages[page].name)) 1086 1087 fp.close() 1088