1#!/usr/bin/python3 2# 3# Copyright (c) 2016-2020 The Khronos Group Inc. 4# 5# SPDX-License-Identifier: Apache-2.0 6 7# Utility functions for automatic ref page generation and other script stuff 8 9import io 10import re 11import sys 12import subprocess 13 14# global errFile, warnFile, diagFile 15 16errFile = sys.stderr 17warnFile = sys.stdout 18diagFile = None 19logSourcefile = None 20logProcname = None 21logLine = None 22 23def unescapeQuotes(s): 24 """Remove \' escape sequences in a string (refpage description)""" 25 return s.replace('\\\'', '\'') 26 27def write(*args, **kwargs ): 28 file = kwargs.pop('file',sys.stdout) 29 end = kwargs.pop('end','\n') 30 file.write(' '.join(str(arg) for arg in args)) 31 file.write(end) 32 33def setLogSourcefile(filename): 34 """Metadata which may be printed (if not None) for diagnostic messages""" 35 global logSourcefile 36 logSourcefile = filename 37 38def setLogProcname(procname): 39 global logProcname 40 logProcname = procname 41 42def setLogLine(line): 43 global logLine 44 logLine = line 45 46def logHeader(severity): 47 """Generate prefix for a diagnostic line using metadata and severity""" 48 global logSourcefile, logProcname, logLine 49 50 msg = severity + ': ' 51 if logProcname: 52 msg = msg + ' in ' + logProcname 53 if logSourcefile: 54 msg = msg + ' for ' + logSourcefile 55 if logLine: 56 msg = msg + ' line ' + str(logLine) 57 return msg + ' ' 58 59def setLogFile(setDiag, setWarn, filename): 60 """Set the file handle to log either or both warnings and diagnostics to. 61 62 - setDiag and setWarn are True if the corresponding handle is to be set. 63 - filename is None for no logging, '-' for stdout, or a pathname.""" 64 global diagFile, warnFile 65 66 if filename is None: 67 return 68 69 if filename == '-': 70 fp = sys.stdout 71 else: 72 fp = open(filename, 'w', encoding='utf-8') 73 74 if setDiag: 75 diagFile = fp 76 if setWarn: 77 warnFile = fp 78 79def logDiag(*args, **kwargs): 80 file = kwargs.pop('file', diagFile) 81 end = kwargs.pop('end','\n') 82 if file is not None: 83 file.write(logHeader('DIAG') + ' '.join(str(arg) for arg in args)) 84 file.write(end) 85 86def logWarn(*args, **kwargs): 87 file = kwargs.pop('file', warnFile) 88 end = kwargs.pop('end','\n') 89 if file is not None: 90 file.write(logHeader('WARN') + ' '.join(str(arg) for arg in args)) 91 file.write(end) 92 93def logErr(*args, **kwargs): 94 file = kwargs.pop('file', errFile) 95 end = kwargs.pop('end','\n') 96 97 strfile = io.StringIO() 98 strfile.write(logHeader('ERROR') + ' '.join(str(arg) for arg in args)) 99 strfile.write(end) 100 101 if file is not None: 102 file.write(strfile.getvalue()) 103 sys.exit(1) 104 105def isempty(s): 106 """Return True if s is nothing but white space, False otherwise""" 107 return len(''.join(s.split())) == 0 108 109class pageInfo: 110 """Information about a ref page relative to the file it's extracted from.""" 111 def __init__(self): 112 self.extractPage = True 113 """True if page should be extracted""" 114 115 self.Warning = None 116 """string warning if page is suboptimal or can't be generated""" 117 118 self.embed = False 119 """False or the name of the ref page this include is embedded within""" 120 121 self.type = None 122 """'structs', 'protos', 'funcpointers', 'flags', 'enums'""" 123 124 self.name = None 125 """struct/proto/enumerant/etc. name""" 126 127 self.desc = None 128 """short description of ref page""" 129 130 self.begin = None 131 """index of first line of the page (heuristic or // refBegin)""" 132 133 self.include = None 134 """index of include:: line defining the page""" 135 136 self.param = None 137 """index of first line of parameter/member definitions""" 138 139 self.body = None 140 """index of first line of body text""" 141 142 self.validity = None 143 """index of validity include""" 144 145 self.end = None 146 """index of last line of the page (heuristic validity include, or // refEnd)""" 147 148 self.alias = '' 149 """aliases of this name, if supplied, or ''""" 150 151 self.refs = '' 152 """cross-references on // refEnd line, if supplied""" 153 154 self.spec = None 155 """'spec' attribute in refpage open block, if supplied, or None for the default ('api') type""" 156 157 self.anchor = None 158 """'anchor' attribute in refpage open block, if supplied, or inferred to be the same as the 'name'""" 159 160def printPageInfoField(desc, line, file): 161 """Print a single field of a pageInfo struct, possibly None. 162 163 - desc - string description of field 164 - line - field value or None 165 - file - indexed by line""" 166 if line is not None: 167 logDiag(desc + ':', line + 1, '\t-> ', file[line], end='') 168 else: 169 logDiag(desc + ':', line) 170 171def printPageInfo(pi, file): 172 """Print out fields of a pageInfo struct 173 174 - pi - pageInfo 175 - file - indexed by pageInfo""" 176 logDiag('TYPE: ', pi.type) 177 logDiag('NAME: ', pi.name) 178 logDiag('WARNING:', pi.Warning) 179 logDiag('EXTRACT:', pi.extractPage) 180 logDiag('EMBED: ', pi.embed) 181 logDiag('DESC: ', pi.desc) 182 printPageInfoField('BEGIN ', pi.begin, file) 183 printPageInfoField('INCLUDE ', pi.include, file) 184 printPageInfoField('PARAM ', pi.param, file) 185 printPageInfoField('BODY ', pi.body, file) 186 printPageInfoField('VALIDITY', pi.validity, file) 187 printPageInfoField('END ', pi.end, file) 188 logDiag('REFS: "' + pi.refs + '"') 189 190def prevPara(file, line): 191 """Go back one paragraph from the specified line and return the line number 192 of the first line of that paragraph. 193 194 Paragraphs are delimited by blank lines. It is assumed that the 195 current line is the first line of a paragraph. 196 197 - file is an array of strings 198 - line is the starting point (zero-based)""" 199 # Skip over current paragraph 200 while (line >= 0 and not isempty(file[line])): 201 line = line - 1 202 # Skip over white space 203 while (line >= 0 and isempty(file[line])): 204 line = line - 1 205 # Skip to first line of previous paragraph 206 while (line >= 1 and not isempty(file[line-1])): 207 line = line - 1 208 return line 209 210def nextPara(file, line): 211 """Go forward one paragraph from the specified line and return the line 212 number of the first line of that paragraph. 213 214 Paragraphs are delimited by blank lines. It is assumed that the 215 current line is standalone (which is bogus). 216 217 - file is an array of strings 218 - line is the starting point (zero-based)""" 219 maxLine = len(file) - 1 220 # Skip over current paragraph 221 while (line != maxLine and not isempty(file[line])): 222 line = line + 1 223 # Skip over white space 224 while (line != maxLine and isempty(file[line])): 225 line = line + 1 226 return line 227 228def lookupPage(pageMap, name): 229 """Return (creating if needed) the pageInfo entry in pageMap for name""" 230 if name not in pageMap: 231 pi = pageInfo() 232 pi.name = name 233 pageMap[name] = pi 234 else: 235 pi = pageMap[name] 236 return pi 237 238def loadFile(filename): 239 """Load a file into a list of strings. Return the list or None on failure""" 240 try: 241 fp = open(filename, 'r', encoding='utf-8') 242 except: 243 logWarn('Cannot open file', filename, ':', sys.exc_info()[0]) 244 return None 245 246 file = fp.readlines() 247 fp.close() 248 249 return file 250 251def clampToBlock(line, minline, maxline): 252 """Clamp a line number to be in the range [minline,maxline]. 253 254 If the line number is None, just return it. 255 If minline is None, don't clamp to that value.""" 256 if line is None: 257 return line 258 if minline and line < minline: 259 return minline 260 if line > maxline: 261 return maxline 262 263 return line 264 265def fixupRefs(pageMap, specFile, file): 266 """Fill in missing fields in pageInfo structures, to the extent they can be 267 inferred. 268 269 - pageMap - dictionary of pageInfo structures 270 - specFile - filename 271 - file - list of strings making up the file, indexed by pageInfo""" 272 # All potential ref pages are now in pageMap. Process them to 273 # identify actual page start/end/description boundaries, if 274 # not already determined from the text. 275 for name in sorted(pageMap.keys()): 276 pi = pageMap[name] 277 278 # # If nothing is found but an include line with no begin, validity, 279 # # or end, this is not intended as a ref page (yet). Set the begin 280 # # line to the include line, so autogeneration can at least 281 # # pull the include out, but mark it not to be extracted. 282 # # Examples include the host sync table includes in 283 # # chapters/fundamentals.txt and the table of Vk*Flag types in 284 # # appendices/boilerplate.txt. 285 # if pi.begin is None and pi.validity is None and pi.end is None: 286 # pi.begin = pi.include 287 # pi.extractPage = False 288 # pi.Warning = 'No begin, validity, or end lines identified' 289 # continue 290 291 # Using open block delimiters, ref pages must *always* have a 292 # defined begin and end. If either is undefined, that's fatal. 293 if pi.begin is None: 294 pi.extractPage = False 295 pi.Warning = 'Can\'t identify begin of ref page open block' 296 continue 297 298 if pi.end is None: 299 pi.extractPage = False 300 pi.Warning = 'Can\'t identify end of ref page open block' 301 continue 302 303 # If there's no description of the page, infer one from the type 304 if pi.desc is None: 305 if pi.type is not None: 306 # pi.desc = pi.type[0:len(pi.type)-1] + ' (no short description available)' 307 pi.Warning = 'No short description available; could infer from the type and name' 308 else: 309 pi.extractPage = False 310 pi.Warning = 'No short description available, cannot infer from the type' 311 continue 312 313 # Try to determine where the parameter and body sections of the page 314 # begin. funcpointer, proto, and struct pages infer the location of 315 # the parameter and body sections. Other pages infer the location of 316 # the body, but have no parameter sections. 317 if pi.include is not None: 318 if pi.type in ['funcpointers', 'protos', 'structs']: 319 pi.param = nextPara(file, pi.include) 320 if pi.body is None: 321 pi.body = nextPara(file, pi.param) 322 else: 323 if pi.body is None: 324 pi.body = nextPara(file, pi.include) 325 else: 326 pi.Warning = 'Page does not have an API definition include::' 327 328 # It's possible for the inferred param and body lines to run past 329 # the end of block, if, for example, there is no parameter section. 330 pi.param = clampToBlock(pi.param, pi.include, pi.end) 331 pi.body = clampToBlock(pi.body, pi.param, pi.end) 332 333 # We can get to this point with .include, .param, and .validity 334 # all being None, indicating those sections weren't found. 335 336 logDiag('fixupRefs: after processing,', pi.name, 'looks like:') 337 printPageInfo(pi, file) 338 339 # Now that all the valid pages have been found, try to make some 340 # inferences about invalid pages. 341 # 342 # If a reference without a .end is entirely inside a valid reference, 343 # then it's intentionally embedded - may want to create an indirect 344 # page that links into the embedding page. This is done by a very 345 # inefficient double loop, but the loop depth is small. 346 for name in sorted(pageMap.keys()): 347 pi = pageMap[name] 348 349 if pi.end is None: 350 for embedName in sorted(pageMap.keys()): 351 logDiag('fixupRefs: comparing', pi.name, 'to', embedName) 352 embed = pageMap[embedName] 353 # Don't check embeddings which are themselves invalid 354 if not embed.extractPage: 355 logDiag('Skipping check for embedding in:', embed.name) 356 continue 357 if embed.begin is None or embed.end is None: 358 logDiag('fixupRefs:', name + ':', 359 'can\'t compare to unanchored ref:', embed.name, 360 'in', specFile, 'at line', pi.include ) 361 printPageInfo(pi, file) 362 printPageInfo(embed, file) 363 # If an embed is found, change the error to a warning 364 elif (pi.include is not None and pi.include >= embed.begin and 365 pi.include <= embed.end): 366 logDiag('fixupRefs: Found embed for:', name, 367 'inside:', embedName, 368 'in', specFile, 'at line', pi.include ) 369 pi.embed = embed.name 370 pi.Warning = 'Embedded in definition for ' + embed.name 371 break 372 else: 373 logDiag('fixupRefs: No embed match for:', name, 374 'inside:', embedName, 'in', specFile, 375 'at line', pi.include) 376 377 378# Patterns used to recognize interesting lines in an asciidoc source file. 379# These patterns are only compiled once. 380INCSVAR_DEF = re.compile(r':INCS-VAR: (?P<value>.*)') 381endifPat = re.compile(r'^endif::(?P<condition>[\w_+,]+)\[\]') 382beginPat = re.compile(r'^\[open,(?P<attribs>refpage=.*)\]') 383# attribute key/value pairs of an open block 384attribStr = r"([a-z]+)='([^'\\]*(?:\\.[^'\\]*)*)'" 385attribPat = re.compile(attribStr) 386bodyPat = re.compile(r'^// *refBody') 387errorPat = re.compile(r'^// *refError') 388 389# This regex transplanted from check_spec_links 390# It looks for either OpenXR or Vulkan generated file conventions, and for 391# the api/validity include (generated_type), protos/struct/etc path 392# (category), and API name (entity_name). It could be put into the API 393# conventions object. 394INCLUDE = re.compile( 395 r'include::(?P<directory_traverse>((../){1,4}|\{INCS-VAR\}/|\{generated\}/)(generated/)?)(?P<generated_type>[\w]+)/(?P<category>\w+)/(?P<entity_name>[^./]+).txt[\[][\]]') 396 397 398def findRefs(file, filename): 399 """Identify reference pages in a list of strings, returning a dictionary of 400 pageInfo entries for each one found, or None on failure.""" 401 setLogSourcefile(filename) 402 setLogProcname('findRefs') 403 404 # To reliably detect the open blocks around reference pages, we must 405 # first detect the '[open,refpage=...]' markup delimiting the block; 406 # skip past the '--' block delimiter on the next line; and identify the 407 # '--' block delimiter closing the page. 408 # This can't be done solely with pattern matching, and requires state to 409 # track 'inside/outside block'. 410 # When looking for open blocks, possible states are: 411 # 'outside' - outside a block 412 # 'start' - have found the '[open...]' line 413 # 'inside' - have found the following '--' line 414 openBlockState = 'outside' 415 416 # Dictionary of interesting line numbers and strings related to an API 417 # name 418 pageMap = {} 419 420 numLines = len(file) 421 line = 0 422 423 # Track the pageInfo object corresponding to the current open block 424 pi = None 425 incsvar = None 426 427 while (line < numLines): 428 setLogLine(line) 429 430 # Look for a file-wide definition 431 matches = INCSVAR_DEF.match(file[line]) 432 if matches: 433 incsvar = matches.group('value') 434 logDiag('Matched INCS-VAR definition:', incsvar) 435 436 line = line + 1 437 continue 438 439 # Perform INCS-VAR substitution immediately. 440 if incsvar and '{INCS-VAR}' in file[line]: 441 newLine = file[line].replace('{INCS-VAR}', incsvar) 442 logDiag('PERFORMING SUBSTITUTION', file[line], '->', newLine) 443 file[line] = newLine 444 445 # Only one of the patterns can possibly match. Add it to 446 # the dictionary for that name. 447 448 # [open,refpage=...] starting a refpage block 449 matches = beginPat.search(file[line]) 450 if matches is not None: 451 logDiag('Matched open block pattern') 452 attribs = matches.group('attribs') 453 454 # If the previous open block wasn't closed, raise an error 455 if openBlockState != 'outside': 456 logErr('Nested open block starting at line', line, 'of', 457 filename) 458 459 openBlockState = 'start' 460 461 # Parse the block attributes 462 matches = attribPat.findall(attribs) 463 464 # Extract each attribute 465 name = None 466 desc = None 467 refpage_type = None 468 spec_type = None 469 anchor = None 470 alias = None 471 xrefs = None 472 473 for (key,value) in matches: 474 logDiag('got attribute', key, '=', value) 475 if key == 'refpage': 476 name = value 477 elif key == 'desc': 478 desc = unescapeQuotes(value) 479 elif key == 'type': 480 refpage_type = value 481 elif key == 'spec': 482 spec_type = value 483 elif key == 'anchor': 484 anchor = value 485 elif key == 'alias': 486 alias = value 487 elif key == 'xrefs': 488 xrefs = value 489 else: 490 logWarn('unknown open block attribute:', key) 491 492 if name is None or desc is None or refpage_type is None: 493 logWarn('missing one or more required open block attributes:' 494 'refpage, desc, or type') 495 # Leave pi is None so open block delimiters are ignored 496 else: 497 pi = lookupPage(pageMap, name) 498 pi.desc = desc 499 # Must match later type definitions in interface/validity includes 500 pi.type = refpage_type 501 pi.spec = spec_type 502 pi.anchor = anchor 503 if alias: 504 pi.alias = alias 505 if xrefs: 506 pi.refs = xrefs 507 logDiag('open block for', name, 'added DESC =', desc, 508 'TYPE =', refpage_type, 'ALIAS =', alias, 509 'XREFS =', xrefs, 'SPEC =', spec_type, 510 'ANCHOR =', anchor) 511 512 line = line + 1 513 continue 514 515 # '--' starting or ending and open block 516 if file[line].rstrip() == '--': 517 if openBlockState == 'outside': 518 # Only refpage open blocks should use -- delimiters 519 logWarn('Unexpected double-dash block delimiters') 520 elif openBlockState == 'start': 521 # -- delimiter following [open,refpage=...] 522 openBlockState = 'inside' 523 524 if pi is None: 525 logWarn('no pageInfo available for opening -- delimiter') 526 else: 527 pi.begin = line + 1 528 logDiag('opening -- delimiter: added BEGIN =', pi.begin) 529 elif openBlockState == 'inside': 530 # -- delimiter ending an open block 531 if pi is None: 532 logWarn('no pageInfo available for closing -- delimiter') 533 else: 534 pi.end = line - 1 535 logDiag('closing -- delimiter: added END =', pi.end) 536 537 openBlockState = 'outside' 538 pi = None 539 else: 540 logWarn('unknown openBlockState:', openBlockState) 541 542 line = line + 1 543 continue 544 545 matches = INCLUDE.search(file[line]) 546 if matches is not None: 547 # Something got included, not sure what yet. 548 gen_type = matches.group('generated_type') 549 refpage_type = matches.group('category') 550 name = matches.group('entity_name') 551 552 # This will never match in OpenCL 553 if gen_type == 'validity': 554 logDiag('Matched validity pattern') 555 if pi is not None: 556 if pi.type and refpage_type != pi.type: 557 logWarn('ERROR: pageMap[' + name + '] type:', 558 pi.type, 'does not match type:', refpage_type) 559 pi.type = refpage_type 560 pi.validity = line 561 logDiag('added TYPE =', pi.type, 'VALIDITY =', pi.validity) 562 else: 563 logWarn('validity include:: line NOT inside block') 564 565 line = line + 1 566 continue 567 568 if gen_type == 'api': 569 logDiag('Matched include pattern') 570 if pi is not None: 571 if pi.include is not None: 572 logDiag('found multiple includes for this block') 573 if pi.type and refpage_type != pi.type: 574 logWarn('ERROR: pageMap[' + name + '] type:', 575 pi.type, 'does not match type:', refpage_type) 576 pi.type = refpage_type 577 pi.include = line 578 logDiag('added TYPE =', pi.type, 'INCLUDE =', pi.include) 579 else: 580 logWarn('interface include:: line NOT inside block') 581 582 line = line + 1 583 continue 584 585 logDiag('ignoring unrecognized include line ', matches.group()) 586 587 # Vulkan 1.1 markup allows the last API include construct to be 588 # followed by an asciidoctor endif:: construct (and also preceded, 589 # at some distance). 590 # This looks for endif:: immediately following an include:: line 591 # and, if found, moves the include boundary to this line. 592 matches = endifPat.search(file[line]) 593 if matches is not None and pi is not None: 594 if pi.include == line - 1: 595 logDiag('Matched endif pattern following include; moving include') 596 pi.include = line 597 else: 598 logDiag('Matched endif pattern (not following include)') 599 600 line = line + 1 601 continue 602 603 matches = bodyPat.search(file[line]) 604 if matches is not None: 605 logDiag('Matched // refBody pattern') 606 if pi is not None: 607 pi.body = line 608 logDiag('added BODY =', pi.body) 609 else: 610 logWarn('// refBody line NOT inside block') 611 612 line = line + 1 613 continue 614 615 # OpenCL spec uses // refError to tag "validity" (Errors) language, 616 # instead of /validity/ includes. 617 matches = errorPat.search(file[line]) 618 if matches is not None: 619 logDiag('Matched // refError pattern') 620 if pi is not None: 621 pi.validity = line 622 logDiag('added VALIDITY (refError) =', pi.validity) 623 else: 624 logWarn('// refError line NOT inside block') 625 626 line = line + 1 627 continue 628 629 line = line + 1 630 continue 631 632 if pi is not None: 633 logErr('Unclosed open block at EOF!') 634 635 setLogSourcefile(None) 636 setLogProcname(None) 637 setLogLine(None) 638 639 return pageMap 640 641 642def getBranch(): 643 """Determine current git branch 644 645 Returns (branch name, ''), or (None, stderr output) if the branch name 646 can't be determined""" 647 648 command = [ 'git', 'symbolic-ref', '--short', 'HEAD' ] 649 results = subprocess.run(command, 650 stdout=subprocess.PIPE, 651 stderr=subprocess.PIPE) 652 653 # git command failed 654 if len(results.stderr) > 0: 655 return (None, results.stderr) 656 657 # Remove newline from output and convert to a string 658 branch = results.stdout.rstrip().decode() 659 if len(branch) > 0: 660 # Strip trailing newline 661 branch = results.stdout.decode()[0:-1] 662 663 return (branch, '') 664