1#!/usr/bin/python3 2# 3# Copyright 2016-2021 The Khronos Group Inc. 4# 5# SPDX-License-Identifier: Apache-2.0 6 7# Utility functions for automatic ref page generation and other script stuff 8 9import io 10import re 11import sys 12import subprocess 13 14# global errFile, warnFile, diagFile 15 16errFile = sys.stderr 17warnFile = sys.stdout 18diagFile = None 19logSourcefile = None 20logProcname = None 21logLine = None 22 23def unescapeQuotes(s): 24 """Remove \' escape sequences in a string (refpage description)""" 25 return s.replace('\\\'', '\'') 26 27def write(*args, **kwargs ): 28 file = kwargs.pop('file',sys.stdout) 29 end = kwargs.pop('end','\n') 30 file.write(' '.join(str(arg) for arg in args)) 31 file.write(end) 32 33def setLogSourcefile(filename): 34 """Metadata which may be printed (if not None) for diagnostic messages""" 35 global logSourcefile 36 logSourcefile = filename 37 38def setLogProcname(procname): 39 global logProcname 40 logProcname = procname 41 42def setLogLine(line): 43 global logLine 44 logLine = line 45 46def logHeader(severity): 47 """Generate prefix for a diagnostic line using metadata and severity""" 48 global logSourcefile, logProcname, logLine 49 50 msg = severity + ': ' 51 if logProcname: 52 msg = msg + ' in ' + logProcname 53 if logSourcefile: 54 msg = msg + ' for ' + logSourcefile 55 if logLine: 56 msg = msg + ' line ' + str(logLine) 57 return msg + ' ' 58 59def setLogFile(setDiag, setWarn, filename): 60 """Set the file handle to log either or both warnings and diagnostics to. 61 62 - setDiag and setWarn are True if the corresponding handle is to be set. 63 - filename is None for no logging, '-' for stdout, or a pathname.""" 64 global diagFile, warnFile 65 66 if filename is None: 67 return 68 69 if filename == '-': 70 fp = sys.stdout 71 else: 72 fp = open(filename, 'w', encoding='utf-8') 73 74 if setDiag: 75 diagFile = fp 76 if setWarn: 77 warnFile = fp 78 79def logDiag(*args, **kwargs): 80 file = kwargs.pop('file', diagFile) 81 end = kwargs.pop('end','\n') 82 if file is not None: 83 file.write(logHeader('DIAG') + ' '.join(str(arg) for arg in args)) 84 file.write(end) 85 86def logWarn(*args, **kwargs): 87 file = kwargs.pop('file', warnFile) 88 end = kwargs.pop('end','\n') 89 if file is not None: 90 file.write(logHeader('WARN') + ' '.join(str(arg) for arg in args)) 91 file.write(end) 92 93def logErr(*args, **kwargs): 94 file = kwargs.pop('file', errFile) 95 end = kwargs.pop('end','\n') 96 97 strfile = io.StringIO() 98 strfile.write(logHeader('ERROR') + ' '.join(str(arg) for arg in args)) 99 strfile.write(end) 100 101 if file is not None: 102 file.write(strfile.getvalue()) 103 sys.exit(1) 104 105def isempty(s): 106 """Return True if s is nothing but white space, False otherwise""" 107 return len(''.join(s.split())) == 0 108 109class pageInfo: 110 """Information about a ref page relative to the file it is extracted from.""" 111 def __init__(self): 112 self.extractPage = True 113 """True if page should be extracted""" 114 115 self.Warning = None 116 """string warning if page is suboptimal or cannot be generated""" 117 118 self.embed = False 119 """False or the name of the ref page this include is embedded within""" 120 121 self.type = None 122 """refpage type attribute - 'structs', 'protos', 'freeform', etc.""" 123 124 self.name = None 125 """struct/proto/enumerant/etc. name""" 126 127 self.desc = None 128 """short description of ref page""" 129 130 self.begin = None 131 """index of first line of the page (heuristic or // refBegin)""" 132 133 self.include = None 134 """index of include:: line defining the page""" 135 136 self.param = None 137 """index of first line of parameter/member definitions""" 138 139 self.body = None 140 """index of first line of body text""" 141 142 self.validity = None 143 """index of validity include""" 144 145 self.end = None 146 """index of last line of the page (heuristic validity include, or // refEnd)""" 147 148 self.alias = '' 149 """aliases of this name, if supplied, or ''""" 150 151 self.refs = '' 152 """cross-references on // refEnd line, if supplied""" 153 154 self.spec = None 155 """'spec' attribute in refpage open block, if supplied, or None for the default ('api') type""" 156 157 self.anchor = None 158 """'anchor' attribute in refpage open block, if supplied, or inferred to be the same as the 'name'""" 159 160def printPageInfoField(desc, line, file): 161 """Print a single field of a pageInfo struct, possibly None. 162 163 - desc - string description of field 164 - line - field value or None 165 - file - indexed by line""" 166 if line is not None: 167 logDiag(desc + ':', line + 1, '\t-> ', file[line], end='') 168 else: 169 logDiag(desc + ':', line) 170 171def printPageInfo(pi, file): 172 """Print out fields of a pageInfo struct 173 174 - pi - pageInfo 175 - file - indexed by pageInfo""" 176 logDiag('TYPE: ', pi.type) 177 logDiag('NAME: ', pi.name) 178 logDiag('WARNING:', pi.Warning) 179 logDiag('EXTRACT:', pi.extractPage) 180 logDiag('EMBED: ', pi.embed) 181 logDiag('DESC: ', pi.desc) 182 printPageInfoField('BEGIN ', pi.begin, file) 183 printPageInfoField('INCLUDE ', pi.include, file) 184 printPageInfoField('PARAM ', pi.param, file) 185 printPageInfoField('BODY ', pi.body, file) 186 printPageInfoField('VALIDITY', pi.validity, file) 187 printPageInfoField('END ', pi.end, file) 188 logDiag('REFS: "' + pi.refs + '"') 189 190def prevPara(file, line): 191 """Go back one paragraph from the specified line and return the line number 192 of the first line of that paragraph. 193 194 Paragraphs are delimited by blank lines. It is assumed that the 195 current line is the first line of a paragraph. 196 197 - file is an array of strings 198 - line is the starting point (zero-based)""" 199 # Skip over current paragraph 200 while (line >= 0 and not isempty(file[line])): 201 line = line - 1 202 # Skip over white space 203 while (line >= 0 and isempty(file[line])): 204 line = line - 1 205 # Skip to first line of previous paragraph 206 while (line >= 1 and not isempty(file[line-1])): 207 line = line - 1 208 return line 209 210def nextPara(file, line): 211 """Go forward one paragraph from the specified line and return the line 212 number of the first line of that paragraph. 213 214 Paragraphs are delimited by blank lines. It is assumed that the 215 current line is standalone (which is bogus). 216 217 - file is an array of strings 218 - line is the starting point (zero-based)""" 219 maxLine = len(file) - 1 220 # Skip over current paragraph 221 while (line != maxLine and not isempty(file[line])): 222 line = line + 1 223 # Skip over white space 224 while (line != maxLine and isempty(file[line])): 225 line = line + 1 226 return line 227 228def lookupPage(pageMap, name): 229 """Return (creating if needed) the pageInfo entry in pageMap for name""" 230 if name not in pageMap: 231 pi = pageInfo() 232 pi.name = name 233 pageMap[name] = pi 234 else: 235 pi = pageMap[name] 236 return pi 237 238def loadFile(filename): 239 """Load a file into a list of strings. Return the list or None on failure""" 240 try: 241 fp = open(filename, 'r', encoding='utf-8') 242 except: 243 logWarn('Cannot open file', filename, ':', sys.exc_info()[0]) 244 return None 245 246 file = fp.readlines() 247 fp.close() 248 249 return file 250 251def clampToBlock(line, minline, maxline): 252 """Clamp a line number to be in the range [minline,maxline]. 253 254 If the line number is None, just return it. 255 If minline is None, do not clamp to that value.""" 256 if line is None: 257 return line 258 if minline and line < minline: 259 return minline 260 if line > maxline: 261 return maxline 262 263 return line 264 265def fixupRefs(pageMap, specFile, file): 266 """Fill in missing fields in pageInfo structures, to the extent they can be 267 inferred. 268 269 - pageMap - dictionary of pageInfo structures 270 - specFile - filename 271 - file - list of strings making up the file, indexed by pageInfo""" 272 # All potential ref pages are now in pageMap. Process them to 273 # identify actual page start/end/description boundaries, if 274 # not already determined from the text. 275 for name in sorted(pageMap.keys()): 276 pi = pageMap[name] 277 278 # # If nothing is found but an include line with no begin, validity, 279 # # or end, this is not intended as a ref page (yet). Set the begin 280 # # line to the include line, so autogeneration can at least 281 # # pull the include out, but mark it not to be extracted. 282 # # Examples include the host sync table includes in 283 # # chapters/fundamentals.txt and the table of Vk*Flag types in 284 # # appendices/boilerplate.txt. 285 # if pi.begin is None and pi.validity is None and pi.end is None: 286 # pi.begin = pi.include 287 # pi.extractPage = False 288 # pi.Warning = 'No begin, validity, or end lines identified' 289 # continue 290 291 # Using open block delimiters, ref pages must *always* have a 292 # defined begin and end. If either is undefined, that is fatal. 293 if pi.begin is None: 294 pi.extractPage = False 295 pi.Warning = 'Can\'t identify begin of ref page open block' 296 continue 297 298 if pi.end is None: 299 pi.extractPage = False 300 pi.Warning = 'Can\'t identify end of ref page open block' 301 continue 302 303 # If there is no description of the page, infer one from the type 304 if pi.desc is None: 305 if pi.type is not None: 306 # pi.desc = pi.type[0:len(pi.type)-1] + ' (no short description available)' 307 pi.Warning = 'No short description available; could infer from the type and name' 308 else: 309 pi.extractPage = False 310 pi.Warning = 'No short description available, cannot infer from the type' 311 continue 312 313 # Try to determine where the parameter and body sections of the page 314 # begin. funcpointer, proto, and struct pages infer the location of 315 # the parameter and body sections. Other pages infer the location of 316 # the body, but have no parameter sections. 317 # 318 #@ Probably some other types infer this as well - refer to list of 319 #@ all page types in genRef.py:emitPage() 320 if pi.include is not None: 321 if pi.type in ['funcpointers', 'protos', 'structs']: 322 pi.param = nextPara(file, pi.include) 323 if pi.body is None: 324 pi.body = nextPara(file, pi.param) 325 else: 326 if pi.body is None: 327 pi.body = nextPara(file, pi.include) 328 else: 329 pi.Warning = 'Page does not have an API definition include::' 330 331 # It is possible for the inferred param and body lines to run past 332 # the end of block, if, for example, there is no parameter section. 333 pi.param = clampToBlock(pi.param, pi.include, pi.end) 334 pi.body = clampToBlock(pi.body, pi.param, pi.end) 335 336 # We can get to this point with .include, .param, and .validity 337 # all being None, indicating those sections were not found. 338 339 logDiag('fixupRefs: after processing,', pi.name, 'looks like:') 340 printPageInfo(pi, file) 341 342 # Now that all the valid pages have been found, try to make some 343 # inferences about invalid pages. 344 # 345 # If a reference without a .end is entirely inside a valid reference, 346 # then it is intentionally embedded - may want to create an indirect 347 # page that links into the embedding page. This is done by a very 348 # inefficient double loop, but the loop depth is small. 349 for name in sorted(pageMap.keys()): 350 pi = pageMap[name] 351 352 if pi.end is None: 353 for embedName in sorted(pageMap.keys()): 354 logDiag('fixupRefs: comparing', pi.name, 'to', embedName) 355 embed = pageMap[embedName] 356 # Do not check embeddings which are themselves invalid 357 if not embed.extractPage: 358 logDiag('Skipping check for embedding in:', embed.name) 359 continue 360 if embed.begin is None or embed.end is None: 361 logDiag('fixupRefs:', name + ':', 362 'can\'t compare to unanchored ref:', embed.name, 363 'in', specFile, 'at line', pi.include ) 364 printPageInfo(pi, file) 365 printPageInfo(embed, file) 366 # If an embed is found, change the error to a warning 367 elif (pi.include is not None and pi.include >= embed.begin and 368 pi.include <= embed.end): 369 logDiag('fixupRefs: Found embed for:', name, 370 'inside:', embedName, 371 'in', specFile, 'at line', pi.include ) 372 pi.embed = embed.name 373 pi.Warning = 'Embedded in definition for ' + embed.name 374 break 375 else: 376 logDiag('fixupRefs: No embed match for:', name, 377 'inside:', embedName, 'in', specFile, 378 'at line', pi.include) 379 380 381# Patterns used to recognize interesting lines in an asciidoc source file. 382# These patterns are only compiled once. 383INCSVAR_DEF = re.compile(r':INCS-VAR: (?P<value>.*)') 384endifPat = re.compile(r'^endif::(?P<condition>[\w_+,]+)\[\]') 385beginPat = re.compile(r'^\[open,(?P<attribs>refpage=.*)\]') 386# attribute key/value pairs of an open block 387attribStr = r"([a-z]+)='([^'\\]*(?:\\.[^'\\]*)*)'" 388attribPat = re.compile(attribStr) 389bodyPat = re.compile(r'^// *refBody') 390errorPat = re.compile(r'^// *refError') 391 392# This regex transplanted from check_spec_links 393# It looks for either OpenXR or Vulkan generated file conventions, and for 394# the api/validity include (generated_type), protos/struct/etc path 395# (category), and API name (entity_name). It could be put into the API 396# conventions object. 397INCLUDE = re.compile( 398 r'include::(?P<directory_traverse>((../){1,4}|\{INCS-VAR\}/|\{generated\}/)(generated/)?)(?P<generated_type>[\w]+)/(?P<category>\w+)/(?P<entity_name>[^./]+).txt[\[][\]]') 399 400 401def findRefs(file, filename): 402 """Identify reference pages in a list of strings, returning a dictionary of 403 pageInfo entries for each one found, or None on failure.""" 404 setLogSourcefile(filename) 405 setLogProcname('findRefs') 406 407 # To reliably detect the open blocks around reference pages, we must 408 # first detect the '[open,refpage=...]' markup delimiting the block; 409 # skip past the '--' block delimiter on the next line; and identify the 410 # '--' block delimiter closing the page. 411 # This cannot be done solely with pattern matching, and requires state to 412 # track 'inside/outside block'. 413 # When looking for open blocks, possible states are: 414 # 'outside' - outside a block 415 # 'start' - have found the '[open...]' line 416 # 'inside' - have found the following '--' line 417 openBlockState = 'outside' 418 419 # Dictionary of interesting line numbers and strings related to an API 420 # name 421 pageMap = {} 422 423 numLines = len(file) 424 line = 0 425 426 # Track the pageInfo object corresponding to the current open block 427 pi = None 428 incsvar = None 429 430 while (line < numLines): 431 setLogLine(line) 432 433 # Look for a file-wide definition 434 matches = INCSVAR_DEF.match(file[line]) 435 if matches: 436 incsvar = matches.group('value') 437 logDiag('Matched INCS-VAR definition:', incsvar) 438 439 line = line + 1 440 continue 441 442 # Perform INCS-VAR substitution immediately. 443 if incsvar and '{INCS-VAR}' in file[line]: 444 newLine = file[line].replace('{INCS-VAR}', incsvar) 445 logDiag('PERFORMING SUBSTITUTION', file[line], '->', newLine) 446 file[line] = newLine 447 448 # Only one of the patterns can possibly match. Add it to 449 # the dictionary for that name. 450 451 # [open,refpage=...] starting a refpage block 452 matches = beginPat.search(file[line]) 453 if matches is not None: 454 logDiag('Matched open block pattern') 455 attribs = matches.group('attribs') 456 457 # If the previous open block was not closed, raise an error 458 if openBlockState != 'outside': 459 logErr('Nested open block starting at line', line, 'of', 460 filename) 461 462 openBlockState = 'start' 463 464 # Parse the block attributes 465 matches = attribPat.findall(attribs) 466 467 # Extract each attribute 468 name = None 469 desc = None 470 refpage_type = None 471 spec_type = None 472 anchor = None 473 alias = None 474 xrefs = None 475 476 for (key,value) in matches: 477 logDiag('got attribute', key, '=', value) 478 if key == 'refpage': 479 name = value 480 elif key == 'desc': 481 desc = unescapeQuotes(value) 482 elif key == 'type': 483 refpage_type = value 484 elif key == 'spec': 485 spec_type = value 486 elif key == 'anchor': 487 anchor = value 488 elif key == 'alias': 489 alias = value 490 elif key == 'xrefs': 491 xrefs = value 492 else: 493 logWarn('unknown open block attribute:', key) 494 495 if name is None or desc is None or refpage_type is None: 496 logWarn('missing one or more required open block attributes:' 497 'refpage, desc, or type') 498 # Leave pi is None so open block delimiters are ignored 499 else: 500 pi = lookupPage(pageMap, name) 501 pi.desc = desc 502 # Must match later type definitions in interface/validity includes 503 pi.type = refpage_type 504 pi.spec = spec_type 505 pi.anchor = anchor 506 if alias: 507 pi.alias = alias 508 if xrefs: 509 pi.refs = xrefs 510 logDiag('open block for', name, 'added DESC =', desc, 511 'TYPE =', refpage_type, 'ALIAS =', alias, 512 'XREFS =', xrefs, 'SPEC =', spec_type, 513 'ANCHOR =', anchor) 514 515 line = line + 1 516 continue 517 518 # '--' starting or ending and open block 519 if file[line].rstrip() == '--': 520 if openBlockState == 'outside': 521 # Only refpage open blocks should use -- delimiters 522 logWarn('Unexpected double-dash block delimiters') 523 elif openBlockState == 'start': 524 # -- delimiter following [open,refpage=...] 525 openBlockState = 'inside' 526 527 if pi is None: 528 logWarn('no pageInfo available for opening -- delimiter') 529 else: 530 pi.begin = line + 1 531 logDiag('opening -- delimiter: added BEGIN =', pi.begin) 532 elif openBlockState == 'inside': 533 # -- delimiter ending an open block 534 if pi is None: 535 logWarn('no pageInfo available for closing -- delimiter') 536 else: 537 pi.end = line - 1 538 logDiag('closing -- delimiter: added END =', pi.end) 539 540 openBlockState = 'outside' 541 pi = None 542 else: 543 logWarn('unknown openBlockState:', openBlockState) 544 545 line = line + 1 546 continue 547 548 matches = INCLUDE.search(file[line]) 549 if matches is not None: 550 # Something got included, not sure what yet. 551 gen_type = matches.group('generated_type') 552 refpage_type = matches.group('category') 553 name = matches.group('entity_name') 554 555 # This will never match in OpenCL 556 if gen_type == 'validity': 557 logDiag('Matched validity pattern') 558 if pi is not None: 559 if pi.type and refpage_type != pi.type: 560 logWarn('ERROR: pageMap[' + name + '] type:', 561 pi.type, 'does not match type:', refpage_type) 562 pi.type = refpage_type 563 pi.validity = line 564 logDiag('added TYPE =', pi.type, 'VALIDITY =', pi.validity) 565 else: 566 logWarn('validity include:: line NOT inside block') 567 568 line = line + 1 569 continue 570 571 if gen_type == 'api': 572 logDiag('Matched include pattern') 573 if pi is not None: 574 if pi.include is not None: 575 logDiag('found multiple includes for this block') 576 if pi.type and refpage_type != pi.type: 577 logWarn('ERROR: pageMap[' + name + '] type:', 578 pi.type, 'does not match type:', refpage_type) 579 pi.type = refpage_type 580 pi.include = line 581 logDiag('added TYPE =', pi.type, 'INCLUDE =', pi.include) 582 else: 583 logWarn('interface include:: line NOT inside block') 584 585 line = line + 1 586 continue 587 588 logDiag('ignoring unrecognized include line ', matches.group()) 589 590 # Vulkan 1.1 markup allows the last API include construct to be 591 # followed by an asciidoctor endif:: construct (and also preceded, 592 # at some distance). 593 # This looks for endif:: immediately following an include:: line 594 # and, if found, moves the include boundary to this line. 595 matches = endifPat.search(file[line]) 596 if matches is not None and pi is not None: 597 if pi.include == line - 1: 598 logDiag('Matched endif pattern following include; moving include') 599 pi.include = line 600 else: 601 logDiag('Matched endif pattern (not following include)') 602 603 line = line + 1 604 continue 605 606 matches = bodyPat.search(file[line]) 607 if matches is not None: 608 logDiag('Matched // refBody pattern') 609 if pi is not None: 610 pi.body = line 611 logDiag('added BODY =', pi.body) 612 else: 613 logWarn('// refBody line NOT inside block') 614 615 line = line + 1 616 continue 617 618 # OpenCL spec uses // refError to tag "validity" (Errors) language, 619 # instead of /validity/ includes. 620 matches = errorPat.search(file[line]) 621 if matches is not None: 622 logDiag('Matched // refError pattern') 623 if pi is not None: 624 pi.validity = line 625 logDiag('added VALIDITY (refError) =', pi.validity) 626 else: 627 logWarn('// refError line NOT inside block') 628 629 line = line + 1 630 continue 631 632 line = line + 1 633 continue 634 635 if pi is not None: 636 logErr('Unclosed open block at EOF!') 637 638 setLogSourcefile(None) 639 setLogProcname(None) 640 setLogLine(None) 641 642 return pageMap 643 644 645def getBranch(): 646 """Determine current git branch 647 648 Returns (branch name, ''), or (None, stderr output) if the branch name 649 cannot be determined""" 650 651 command = [ 'git', 'symbolic-ref', '--short', 'HEAD' ] 652 results = subprocess.run(command, 653 stdout=subprocess.PIPE, 654 stderr=subprocess.PIPE) 655 656 # git command failed 657 if len(results.stderr) > 0: 658 return (None, results.stderr) 659 660 # Remove newline from output and convert to a string 661 branch = results.stdout.rstrip().decode() 662 if len(branch) > 0: 663 # Strip trailing newline 664 branch = results.stdout.decode()[0:-1] 665 666 return (branch, '') 667