1#!/usr/bin/python3 2# 3# Copyright (c) 2016-2020 The Khronos Group Inc. 4# 5# SPDX-License-Identifier: Apache-2.0 6 7"""Used for automatic reflow of spec sources to satisfy the agreed layout to 8minimize git churn. Most of the logic has to do with detecting asciidoc 9markup or block types that *shouldn't* be reflowed (tables, code) and 10ignoring them. It's very likely there are many asciidoc constructs not yet 11accounted for in the script, our usage of asciidoc markup is intentionally 12somewhat limited. 13 14Also used to insert identifying tags on explicit Valid Usage statements. 15 16Usage: `reflow.py [-noflow] [-tagvu] [-nextvu #] [-overwrite] [-out dir] [-suffix str] files` 17 18- `-noflow` acts as a passthrough, instead of reflowing text. Other 19 processing may occur. 20- `-tagvu` generates explicit VUID tag for Valid Usage statements which 21 don't already have them. 22- `-nextvu #` starts VUID tag generation at the specified # instead of 23 the value wired into the `reflow.py` script. 24- `-overwrite` updates in place (can be risky, make sure there are backups) 25- `-out` specifies directory to create output file in, default 'out' 26- `-suffix` specifies suffix to add to output files, default '' 27- `files` are asciidoc source files from the spec to reflow. 28""" 29# For error and file-loading interfaces only 30import argparse 31import os 32import re 33import sys 34from reflib import loadFile, logDiag, logWarn, logErr, setLogFile, getBranch 35from vuidCounts import vuidCounts 36 37# Vulkan-specific - will consolidate into scripts/ like OpenXR soon 38sys.path.insert(0, 'xml') 39 40from vkconventions import VulkanConventions as APIConventions 41conventions = APIConventions() 42 43# Markup that always ends a paragraph 44# empty line or whitespace 45# [block options] 46# [[anchor]] 47# // comment 48# <<<< page break 49# :attribute-setting 50# macro-directive::terms 51# + standalone list item continuation 52# label:: labelled list - label must be standalone 53endPara = re.compile(r'^( *|\[.*\]|//.*|<<<<|:.*|[a-z]+::.*|\+|.*::)$') 54 55# Special case of markup ending a paragraph, used to track the current 56# command/structure. This allows for either OpenXR or Vulkan API path 57# conventions. Nominally it should use the file suffix defined by the API 58# conventions (conventions.file_suffix), except that XR uses '.txt' for 59# generated API include files, not '.adoc' like its other includes. 60includePat = re.compile( 61 r'include::(?P<directory_traverse>((../){1,4}|\{INCS-VAR\}/|\{generated\}/)(generated/)?)(?P<generated_type>[\w]+)/(?P<category>\w+)/(?P<entity_name>[^./]+).txt[\[][\]]') 62 63# Find the first pname: or code: pattern in a Valid Usage statement 64pnamePat = re.compile(r'pname:(?P<param>\w+)') 65codePat = re.compile(r'code:(?P<param>\w+)') 66 67# Markup that's OK in a contiguous paragraph but otherwise passed through 68# .anything (except .., which indicates a literal block) 69# === Section Titles 70endParaContinue = re.compile(r'^(\.[^.].*|=+ .*)$') 71 72# Markup for block delimiters whose contents *should* be reformatted 73# -- (exactly two) (open block) 74# **** (4 or more) (sidebar block - why do we have these?!) 75# ==== (4 or more) (example block) 76# ____ (4 or more) (quote block) 77blockReflow = re.compile(r'^(--|[*=_]{4,})$') 78 79# Fake block delimiters for "common" VU statements 80blockCommonReflow = '// Common Valid Usage\n' 81 82# Markup for block delimiters whose contents should *not* be reformatted 83# |=== (3 or more) (table) 84# ++++ (4 or more) (passthrough block) 85# .... (4 or more) (literal block) 86# //// (4 or more) (comment block) 87# ---- (4 or more) (listing block) 88# ``` (3 or more) (listing block) 89# **** (4 or more) (sidebar block) 90blockPassthrough = re.compile(r'^(\|={3,}|[`]{3}|[-+./]{4,})$') 91 92# Markup for introducing lists (hanging paragraphs) 93# * bullet 94# ** bullet 95# -- bullet 96# . bullet 97# :: bullet (no longer supported by asciidoctor 2) 98# {empty}:: bullet 99# 1. list item 100beginBullet = re.compile(r'^ *([*-.]+|\{empty\}::|::|[0-9]+[.]) ') 101 102# Text that (may) not end sentences 103 104# A single letter followed by a period, typically a middle initial. 105endInitial = re.compile(r'^[A-Z]\.$') 106# An abbreviation, which doesn't (usually) end a line. 107endAbbrev = re.compile(r'(e\.g|i\.e|c\.f|vs)\.$', re.IGNORECASE) 108 109class ReflowState: 110 """State machine for reflowing. 111 112 Represents the state of the reflow operation""" 113 def __init__(self, 114 filename, 115 margin = 76, 116 file = sys.stdout, 117 breakPeriod = True, 118 reflow = True, 119 nextvu = None, 120 maxvu = None): 121 122 self.blockStack = [ None ] 123 """The last element is a line with the asciidoc block delimiter that's currently in effect, 124 such as '--', '----', '****', '======', or '+++++++++'. 125 This affects whether or not the block contents should be formatted.""" 126 127 self.reflowStack = [ True ] 128 """The last element is True or False if the current blockStack contents 129 should be reflowed.""" 130 self.vuStack = [ False ] 131 """the last element is True or False if the current blockStack contents 132 are an explicit Valid Usage block.""" 133 134 self.margin = margin 135 """margin to reflow text to.""" 136 137 self.para = [] 138 """list of lines in the paragraph being accumulated. 139 When this is non-empty, there is a current paragraph.""" 140 141 self.lastTitle = False 142 """true if the previous line was a document title line 143 (e.g. :leveloffset: 0 - no attempt to track changes to this is made).""" 144 145 self.leadIndent = 0 146 """indent level (in spaces) of the first line of a paragraph.""" 147 148 self.hangIndent = 0 149 """indent level of the remaining lines of a paragraph.""" 150 151 self.file = file 152 """file pointer to write to.""" 153 154 self.filename = filename 155 """base name of file being read from.""" 156 157 self.lineNumber = 0 158 """line number being read from the input file.""" 159 160 self.breakPeriod = breakPeriod 161 """True if justification should break to a new line after the end of a sentence.""" 162 163 self.breakInitial = True 164 """True if justification should break to a new line after 165 something that appears to be an initial in someone's name. **TBD**""" 166 167 self.reflow = reflow 168 """True if text should be reflowed, False to pass through unchanged.""" 169 170 self.vuPrefix = 'VUID' 171 """Prefix of generated Valid Usage tags""" 172 173 self.vuFormat = '{0}-{1}-{2}-{3:0>5d}' 174 """Format string for generating Valid Usage tags. 175 First argument is vuPrefix, second is command/struct name, third is parameter name, fourth is the tag number.""" 176 177 self.nextvu = nextvu 178 """Integer to start tagging un-numbered Valid Usage statements with, 179 or None if no tagging should be done.""" 180 181 self.maxvu = maxvu 182 """Maximum tag to use for Valid Usage statements, or None if no 183 tagging should be done.""" 184 185 self.defaultApiName = '{refpage}' 186 self.apiName = self.defaultApiName 187 """String name of a Vulkan structure or command for VUID tag 188 generation, or {refpage} if one hasn't been included in this file 189 yet.""" 190 191 def incrLineNumber(self): 192 self.lineNumber = self.lineNumber + 1 193 194 def printLines(self, lines): 195 """Print an array of lines with newlines already present""" 196 if len(lines) > 0: 197 logDiag(':: printLines:', len(lines), 'lines: ', lines[0], end='') 198 199 for line in lines: 200 print(line, file=self.file, end='') 201 202 def endSentence(self, word): 203 """Return True if word ends with a sentence-period, False otherwise. 204 205 Allows for contraction cases which won't end a line: 206 207 - A single letter (if breakInitial is True) 208 - Abbreviations: 'c.f.', 'e.g.', 'i.e.' (or mixed-case versions)""" 209 if (word[-1:] != '.' or 210 endAbbrev.search(word) or 211 (self.breakInitial and endInitial.match(word))): 212 return False 213 214 return True 215 216 def vuidAnchor(self, word): 217 """Return True if word is a Valid Usage ID Tag anchor.""" 218 return (word[0:7] == '[[VUID-') 219 220 def isOpenBlockDelimiter(self, line): 221 """Returns True if line is an open block delimiter.""" 222 return line[0:2] == '--' 223 224 def reflowPara(self): 225 """Reflow the current paragraph, respecting the paragraph lead and 226 hanging indentation levels. 227 228 The algorithm also respects trailing '+' signs that indicate embedded newlines, 229 and will not reflow a very long word immediately after a bullet point. 230 231 Just return the paragraph unchanged if the -noflow argument was 232 given.""" 233 if not self.reflow: 234 return self.para 235 236 logDiag('reflowPara lead indent = ', self.leadIndent, 237 'hangIndent =', self.hangIndent, 238 'para:', self.para[0], end='') 239 240 # Total words processed (we care about the *first* word vs. others) 241 wordCount = 0 242 243 # Tracks the *previous* word processed. It must not be empty. 244 prevWord = ' ' 245 246 # Track the previous line and paragraph being indented, if any 247 outLine = None 248 outPara = [] 249 250 for line in self.para: 251 line = line.rstrip() 252 words = line.split() 253 254 # logDiag('reflowPara: input line =', line) 255 numWords = len(words) - 1 256 257 for i in range(0, numWords + 1): 258 word = words[i] 259 wordLen = len(word) 260 wordCount += 1 261 262 endEscape = False 263 if i == numWords and word == '+': 264 # Trailing ' +' must stay on the same line 265 endEscape = word 266 # logDiag('reflowPara last word of line =', word, 'prevWord =', prevWord, 'endEscape =', endEscape) 267 else: 268 pass 269 # logDiag('reflowPara wordCount =', wordCount, 'word =', word, 'prevWord =', prevWord) 270 271 if wordCount == 1: 272 # The first word of the paragraph is treated specially. 273 # The loop logic becomes trickier if all this code is 274 # done prior to looping over lines and words, so all the 275 # setup logic is done here. 276 277 outPara = [] 278 outLine = ''.ljust(self.leadIndent) + word 279 outLineLen = self.leadIndent + wordLen 280 281 # If the paragraph begins with a bullet point, generate 282 # a hanging indent level if there isn't one already. 283 if beginBullet.match(self.para[0]): 284 bulletPoint = True 285 if len(self.para) > 1: 286 logDiag('reflowPara first line matches bullet point', 287 'but indent already hanging @ input line', 288 self.lineNumber) 289 else: 290 logDiag('reflowPara first line matches bullet point -' 291 'single line, assuming hangIndent @ input line', 292 self.lineNumber) 293 self.hangIndent = outLineLen + 1 294 else: 295 bulletPoint = False 296 else: 297 # Possible actions to take with this word 298 # 299 # addWord - add word to current line 300 # closeLine - append line and start a new (null) one 301 # startLine - add word to a new line 302 303 # Default behavior if all the tests below fail is to add 304 # this word to the current line, and keep accumulating 305 # that line. 306 (addWord, closeLine, startLine) = (True, False, False) 307 308 # How long would this line be if the word were added? 309 newLen = outLineLen + 1 + wordLen 310 311 # Are we on the first word following a bullet point? 312 firstBullet = (wordCount == 2 and bulletPoint) 313 314 if endEscape: 315 # If the new word ends the input line with ' +', 316 # add it to the current line. 317 318 (addWord, closeLine, startLine) = (True, True, False) 319 elif self.vuidAnchor(word): 320 # If the new word is a Valid Usage anchor, break the 321 # line afterwards. Note that this should only happen 322 # immediately after a bullet point, but we don't 323 # currently check for this. 324 (addWord, closeLine, startLine) = (True, True, False) 325 elif newLen > self.margin: 326 if firstBullet: 327 # If the word follows a bullet point, add it to 328 # the current line no matter its length. 329 330 (addWord, closeLine, startLine) = (True, True, False) 331 else: 332 # The word overflows, so add it to a new line. 333 334 (addWord, closeLine, startLine) = (False, True, True) 335 elif (self.breakPeriod and 336 (wordCount > 2 or not firstBullet) and 337 self.endSentence(prevWord)): 338 # If the previous word ends a sentence and 339 # breakPeriod is set, start a new line. 340 # The complicated logic allows for leading bullet 341 # points which are periods (implicitly numbered lists). 342 # @@@ But not yet for explicitly numbered lists. 343 344 (addWord, closeLine, startLine) = (False, True, True) 345 346 # Add a word to the current line 347 if addWord: 348 if outLine: 349 outLine += ' ' + word 350 outLineLen = newLen 351 else: 352 # Fall through to startLine case if there's no 353 # current line yet. 354 startLine = True 355 356 # Add current line to the output paragraph. Force 357 # starting a new line, although we don't yet know if it 358 # will ever have contents. 359 if closeLine: 360 if outLine: 361 outPara.append(outLine + '\n') 362 outLine = None 363 364 # Start a new line and add a word to it 365 if startLine: 366 outLine = ''.ljust(self.hangIndent) + word 367 outLineLen = self.hangIndent + wordLen 368 369 # Track the previous word, for use in breaking at end of 370 # a sentence 371 prevWord = word 372 373 # Add this line to the output paragraph. 374 if outLine: 375 outPara.append(outLine + '\n') 376 377 return outPara 378 379 def emitPara(self): 380 """Emit a paragraph, possibly reflowing it depending on the block context. 381 382 Resets the paragraph accumulator.""" 383 if self.para != []: 384 if self.vuStack[-1] and self.nextvu is not None: 385 # If: 386 # - this paragraph is in a Valid Usage block, 387 # - VUID tags are being assigned, 388 # Try to assign VUIDs 389 390 if nestedVuPat.search(self.para[0]): 391 # Check for nested bullet points. These should not be 392 # assigned VUIDs, nor present at all, because they break 393 # the VU extractor. 394 logWarn(self.filename + ': Invalid nested bullet point in VU block:', self.para[0]) 395 elif self.vuPrefix not in self.para[0]: 396 # If: 397 # - a tag is not already present, and 398 # - the paragraph is a properly marked-up list item 399 # Then add a VUID tag starting with the next free ID. 400 401 # Split the first line after the bullet point 402 matches = vuPat.search(self.para[0]) 403 if matches is not None: 404 logDiag('findRefs: Matched vuPat on line:', self.para[0], end='') 405 head = matches.group('head') 406 tail = matches.group('tail') 407 408 # Use the first pname: or code: tag in the paragraph as 409 # the parameter name in the VUID tag. This won't always 410 # be correct, but should be highly reliable. 411 for vuLine in self.para: 412 matches = pnamePat.search(vuLine) 413 if matches is not None: 414 break 415 matches = codePat.search(vuLine) 416 if matches is not None: 417 break 418 419 if matches is not None: 420 paramName = matches.group('param') 421 else: 422 paramName = 'None' 423 logWarn(self.filename, 424 'No param name found for VUID tag on line:', 425 self.para[0]) 426 427 newline = (head + ' [[' + 428 self.vuFormat.format(self.vuPrefix, 429 self.apiName, 430 paramName, 431 self.nextvu) + ']] ' + tail) 432 433 logDiag('Assigning', self.vuPrefix, self.apiName, self.nextvu, 434 ' on line:', self.para[0], '->', newline, 'END') 435 436 # Don't actually assign the VUID unless it's in the reserved range 437 if self.nextvu <= self.maxvu: 438 if self.nextvu == self.maxvu: 439 logWarn('Skipping VUID assignment, no more VUIDs available') 440 self.para[0] = newline 441 self.nextvu = self.nextvu + 1 442 # else: 443 # There are only a few cases of this, and they're all 444 # legitimate. Leave detecting this case to another tool 445 # or hand inspection. 446 # logWarn(self.filename + ': Unexpected non-bullet item in VU block (harmless if following an ifdef):', 447 # self.para[0]) 448 449 if self.reflowStack[-1]: 450 self.printLines(self.reflowPara()) 451 else: 452 self.printLines(self.para) 453 454 # Reset the paragraph, including its indentation level 455 self.para = [] 456 self.leadIndent = 0 457 self.hangIndent = 0 458 459 def endPara(self, line): 460 """'line' ends a paragraph and should itself be emitted. 461 line may be None to indicate EOF or other exception.""" 462 logDiag('endPara line', self.lineNumber, ': emitting paragraph') 463 464 # Emit current paragraph, this line, and reset tracker 465 self.emitPara() 466 467 if line: 468 self.printLines( [ line ] ) 469 470 def endParaContinue(self, line): 471 """'line' ends a paragraph (unless there's already a paragraph being 472 accumulated, e.g. len(para) > 0 - currently not implemented)""" 473 self.endPara(line) 474 475 def endBlock(self, line, reflow = False, vuBlock = False): 476 """'line' begins or ends a block. 477 478 If beginning a block, tag whether or not to reflow the contents. 479 480 vuBlock is True if the previous line indicates this is a Valid Usage block.""" 481 self.endPara(line) 482 483 if self.blockStack[-1] == line: 484 logDiag('endBlock line', self.lineNumber, 485 ': popping block end depth:', len(self.blockStack), 486 ':', line, end='') 487 488 # Reset apiName at the end of an open block. 489 # Open blocks cannot be nested (at present), so this is safe. 490 if self.isOpenBlockDelimiter(line): 491 logDiag('reset apiName to empty at line', self.lineNumber) 492 self.apiName = self.defaultApiName 493 else: 494 logDiag('NOT resetting apiName to default at line', self.lineNumber) 495 496 self.blockStack.pop() 497 self.reflowStack.pop() 498 self.vuStack.pop() 499 else: 500 # Start a block 501 self.blockStack.append(line) 502 self.reflowStack.append(reflow) 503 self.vuStack.append(vuBlock) 504 505 logDiag('endBlock reflow =', reflow, ' line', self.lineNumber, 506 ': pushing block start depth', len(self.blockStack), 507 ':', line, end='') 508 509 def endParaBlockReflow(self, line, vuBlock): 510 """'line' begins or ends a block. The paragraphs in the block *should* be 511 reformatted (e.g. a NOTE).""" 512 self.endBlock(line, reflow = True, vuBlock = vuBlock) 513 514 def endParaBlockPassthrough(self, line): 515 """'line' begins or ends a block. The paragraphs in the block should 516 *not* be reformatted (e.g. a code listing).""" 517 self.endBlock(line, reflow = False) 518 519 def addLine(self, line): 520 """'line' starts or continues a paragraph. 521 522 Paragraphs may have "hanging indent", e.g. 523 524 ``` 525 * Bullet point... 526 ... continued 527 ``` 528 529 In this case, when the higher indentation level ends, so does the 530 paragraph.""" 531 logDiag('addLine line', self.lineNumber, ':', line, end='') 532 533 # See https://stackoverflow.com/questions/13648813/what-is-the-pythonic-way-to-count-the-leading-spaces-in-a-string 534 indent = len(line) - len(line.lstrip()) 535 536 # A hanging paragraph ends due to a less-indented line. 537 if self.para != [] and indent < self.hangIndent: 538 logDiag('addLine: line reduces indentation, emit paragraph') 539 self.emitPara() 540 541 # A bullet point (or something that looks like one) always ends the 542 # current paragraph. 543 if beginBullet.match(line): 544 logDiag('addLine: line matches beginBullet, emit paragraph') 545 self.emitPara() 546 547 if self.para == []: 548 # Begin a new paragraph 549 self.para = [ line ] 550 self.leadIndent = indent 551 self.hangIndent = indent 552 else: 553 # Add a line to a paragraph. Increase the hanging indentation 554 # level - once. 555 if self.hangIndent == self.leadIndent: 556 self.hangIndent = indent 557 self.para.append(line) 558 559def apiMatch(oldname, newname): 560 """Returns whether oldname and newname match, up to an API suffix.""" 561 upper = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 562 return oldname.rstrip(upper) == newname.rstrip(upper) 563 564def reflowFile(filename, args): 565 logDiag('reflow: filename', filename) 566 567 lines = loadFile(filename) 568 if lines is None: 569 return 570 571 # Output file handle and reflow object for this file. There are no race 572 # conditions on overwriting the input, but it's not recommended unless 573 # you have backing store such as git. 574 575 if args.overwrite: 576 outFilename = filename 577 else: 578 outFilename = args.outDir + '/' + os.path.basename(filename) + args.suffix 579 580 try: 581 fp = open(outFilename, 'w', encoding='utf8') 582 except: 583 logWarn('Cannot open output file', filename, ':', sys.exc_info()[0]) 584 return 585 586 state = ReflowState(filename, 587 file = fp, 588 reflow = not args.noflow, 589 nextvu = args.nextvu, 590 maxvu = args.maxvu) 591 592 for line in lines: 593 state.incrLineNumber() 594 595 # Is this a title line (leading '= ' followed by text)? 596 thisTitle = False 597 598 # The logic here is broken. If we're in a non-reflowable block and 599 # this line *doesn't* end the block, it should always be 600 # accumulated. 601 602 # Test for a blockCommonReflow delimiter comment first, to avoid 603 # treating it solely as a end-Paragraph marker comment. 604 if line == blockCommonReflow: 605 # Starting or ending a pseudo-block for "common" VU statements. 606 state.endParaBlockReflow(line, vuBlock = True) 607 608 elif blockReflow.match(line): 609 # Starting or ending a block whose contents may be reflowed. 610 # Blocks cannot be nested. 611 612 # Is this is an explicit Valid Usage block? 613 vuBlock = (state.lineNumber > 1 and 614 lines[state.lineNumber-2] == '.Valid Usage\n') 615 616 state.endParaBlockReflow(line, vuBlock) 617 618 elif endPara.match(line): 619 # Ending a paragraph. Emit the current paragraph, if any, and 620 # prepare to begin a new paragraph. 621 622 state.endPara(line) 623 624 # If this is an include:: line starting the definition of a 625 # structure or command, track that for use in VUID generation. 626 627 matches = includePat.search(line) 628 if matches is not None: 629 generated_type = matches.group('generated_type') 630 include_type = matches.group('category') 631 if generated_type == 'api' and include_type in ('protos', 'structs'): 632 apiName = matches.group('entity_name') 633 if state.apiName != state.defaultApiName: 634 # This happens when there are multiple API include 635 # lines in a single block. The style guideline is to 636 # always place the API which others are promoted to 637 # first. In virtually all cases, the promoted API 638 # will differ solely in the vendor suffix (or 639 # absence of it), which is benign. 640 if not apiMatch(state.apiName, apiName): 641 logWarn('Promoted API name mismatch at line', 642 state.lineNumber, 643 ':', 644 'apiName:', apiName, 645 'does not match state.apiName:', 646 state.apiName) 647 else: 648 state.apiName = apiName 649 650 elif endParaContinue.match(line): 651 # For now, always just end the paragraph. 652 # Could check see if len(para) > 0 to accumulate. 653 654 state.endParaContinue(line) 655 656 # If it's a title line, track that 657 if line[0:2] == '= ': 658 thisTitle = True 659 660 elif blockPassthrough.match(line): 661 # Starting or ending a block whose contents must not be reflowed. 662 # These are tables, etc. Blocks cannot be nested. 663 664 state.endParaBlockPassthrough(line) 665 elif state.lastTitle: 666 # The previous line was a document title line. This line 667 # is the author / credits line and must not be reflowed. 668 669 state.endPara(line) 670 else: 671 # Just accumulate a line to the current paragraph. Watch out for 672 # hanging indents / bullet-points and track that indent level. 673 674 state.addLine(line) 675 676 state.lastTitle = thisTitle 677 678 # Cleanup at end of file 679 state.endPara(None) 680 681 # Check for sensible block nesting 682 if len(state.blockStack) > 1: 683 logWarn('file', filename, 684 'mismatched asciidoc block delimiters at EOF:', 685 state.blockStack[-1]) 686 687 fp.close() 688 689 # Update the 'nextvu' value 690 if args.nextvu != state.nextvu: 691 logWarn('Updated nextvu to', state.nextvu, 'after file', filename) 692 args.nextvu = state.nextvu 693 694def reflowAllAdocFiles(folder_to_reflow, args): 695 for root, subdirs, files in os.walk(folder_to_reflow): 696 for file in files: 697 if file.endswith(conventions.file_suffix): 698 file_path = os.path.join(root, file) 699 reflowFile(file_path, args) 700 for subdir in subdirs: 701 sub_folder = os.path.join(root, subdir) 702 print('Sub-folder = %s' % sub_folder) 703 if subdir.lower() not in conventions.spec_no_reflow_dirs: 704 print(' Parsing = %s' % sub_folder) 705 reflowAllAdocFiles(sub_folder, args) 706 else: 707 print(' Skipping = %s' % sub_folder) 708 709# Patterns used to recognize interesting lines in an asciidoc source file. 710# These patterns are only compiled once. 711 712# Explicit Valid Usage list item with one or more leading asterisks 713# The re.DOTALL is needed to prevent vuPat.search() from stripping 714# the trailing newline. 715vuPat = re.compile(r'^(?P<head> [*]+)( *)(?P<tail>.*)', re.DOTALL) 716 717# Pattern matching leading nested bullet points 718global nestedVuPat 719nestedVuPat = re.compile(r'^ \*\*') 720 721if __name__ == '__main__': 722 parser = argparse.ArgumentParser() 723 724 parser.add_argument('-diag', action='store', dest='diagFile', 725 help='Set the diagnostic file') 726 parser.add_argument('-warn', action='store', dest='warnFile', 727 help='Set the warning file') 728 parser.add_argument('-log', action='store', dest='logFile', 729 help='Set the log file for both diagnostics and warnings') 730 parser.add_argument('-overwrite', action='store_true', 731 help='Overwrite input filenames instead of writing different output filenames') 732 parser.add_argument('-out', action='store', dest='outDir', 733 default='out', 734 help='Set the output directory in which updated files are generated (default: out)') 735 parser.add_argument('-tagvu', action='store_true', 736 help='Tag un-tagged Valid Usage statements starting at the value wired into reflow.py') 737 parser.add_argument('-nextvu', action='store', dest='nextvu', type=int, 738 default=None, 739 help='Specify start VUID to use instead of the value wired into vuidCounts.py') 740 parser.add_argument('-maxvu', action='store', dest='maxvu', type=int, 741 default=None, 742 help='Specify maximum VUID instead of the value wired into vuidCounts.py') 743 parser.add_argument('-branch', action='store', dest='branch', 744 help='Specify branch to assign VUIDs for.') 745 parser.add_argument('-noflow', action='store_true', dest='noflow', 746 help='Do not reflow text. Other actions may apply.') 747 parser.add_argument('-suffix', action='store', dest='suffix', 748 default='', 749 help='Set the suffix added to updated file names (default: none)') 750 parser.add_argument('files', metavar='filename', nargs='*', 751 help='a filename to reflow text in') 752 parser.add_argument('--version', action='version', version='%(prog)s 1.0') 753 754 args = parser.parse_args() 755 756 setLogFile(True, True, args.logFile) 757 setLogFile(True, False, args.diagFile) 758 setLogFile(False, True, args.warnFile) 759 760 if args.overwrite: 761 logWarn("reflow.py: will overwrite all input files") 762 763 errors = '' 764 if args.branch is None: 765 (args.branch, errors) = getBranch() 766 if args.branch is None: 767 logErr('Cannot determine current git branch:', errors) 768 769 if args.tagvu and args.nextvu is None: 770 if args.branch not in vuidCounts: 771 logErr('Branch', args.branch, 'not in vuidCounts, cannot continue') 772 maxVUID = vuidCounts[args.branch][1] 773 startVUID = vuidCounts[args.branch][2] 774 args.nextvu = startVUID 775 args.maxvu = maxVUID 776 777 if args.nextvu is not None: 778 logWarn('Tagging untagged Valid Usage statements starting at', args.nextvu) 779 780 # If no files are specified, reflow the entire specification chapters folder 781 if not args.files: 782 folder_to_reflow = conventions.spec_reflow_path 783 logWarn('Reflowing all asciidoc files under', folder_to_reflow) 784 reflowAllAdocFiles(folder_to_reflow, args) 785 else: 786 for file in args.files: 787 reflowFile(file, args) 788 789 if args.nextvu is not None and args.nextvu != startVUID: 790 # Update next free VUID to assign 791 vuidCounts[args.branch][2] = args.nextvu 792 try: 793 reflow_count_file_path = os.path.dirname(os.path.realpath(__file__)) 794 reflow_count_file_path += '/vuidCounts.py' 795 reflow_count_file = open(reflow_count_file_path, 'w', encoding='utf8') 796 print('# Do not edit this file!', file=reflow_count_file) 797 print('# VUID ranges reserved for branches', file=reflow_count_file) 798 print('# Key is branch name, value is [ start, end, nextfree ]', file=reflow_count_file) 799 print('vuidCounts = {', file=reflow_count_file) 800 for key in sorted(vuidCounts): 801 print(" '{}': [ {}, {}, {} ],".format( 802 key, 803 vuidCounts[key][0], 804 vuidCounts[key][1], 805 vuidCounts[key][2]), 806 file=reflow_count_file) 807 print('}', file=reflow_count_file) 808 reflow_count_file.close() 809 except: 810 logWarn('Cannot open output count file vuidCounts.py', ':', sys.exc_info()[0]) 811