• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python3
2#
3# Copyright 2016-2021 The Khronos Group Inc.
4#
5# SPDX-License-Identifier: Apache-2.0
6
7# Utility functions for automatic ref page generation and other script stuff
8
9import io
10import re
11import sys
12import subprocess
13
14# global errFile, warnFile, diagFile
15
16errFile = sys.stderr
17warnFile = sys.stdout
18diagFile = None
19logSourcefile = None
20logProcname = None
21logLine = None
22
23def unescapeQuotes(s):
24    """Remove \' escape sequences in a string (refpage description)"""
25    return s.replace('\\\'', '\'')
26
27def write(*args, **kwargs ):
28    file = kwargs.pop('file',sys.stdout)
29    end = kwargs.pop('end','\n')
30    file.write(' '.join(str(arg) for arg in args))
31    file.write(end)
32
33def setLogSourcefile(filename):
34    """Metadata which may be printed (if not None) for diagnostic messages"""
35    global logSourcefile
36    logSourcefile = filename
37
38def setLogProcname(procname):
39    global logProcname
40    logProcname = procname
41
42def setLogLine(line):
43    global logLine
44    logLine = line
45
46def logHeader(severity):
47    """Generate prefix for a diagnostic line using metadata and severity"""
48    global logSourcefile, logProcname, logLine
49
50    msg = severity + ': '
51    if logProcname:
52        msg = msg + ' in ' + logProcname
53    if logSourcefile:
54        msg = msg + ' for ' + logSourcefile
55    if logLine:
56        msg = msg + ' line ' + str(logLine)
57    return msg + ' '
58
59def setLogFile(setDiag, setWarn, filename):
60    """Set the file handle to log either or both warnings and diagnostics to.
61
62    - setDiag and setWarn are True if the corresponding handle is to be set.
63    - filename is None for no logging, '-' for stdout, or a pathname."""
64    global diagFile, warnFile
65
66    if filename is None:
67        return
68
69    if filename == '-':
70        fp = sys.stdout
71    else:
72        fp = open(filename, 'w', encoding='utf-8')
73
74    if setDiag:
75        diagFile = fp
76    if setWarn:
77        warnFile = fp
78
79def logDiag(*args, **kwargs):
80    file = kwargs.pop('file', diagFile)
81    end = kwargs.pop('end','\n')
82    if file is not None:
83        file.write(logHeader('DIAG') + ' '.join(str(arg) for arg in args))
84        file.write(end)
85
86def logWarn(*args, **kwargs):
87    file = kwargs.pop('file', warnFile)
88    end = kwargs.pop('end','\n')
89    if file is not None:
90        file.write(logHeader('WARN') + ' '.join(str(arg) for arg in args))
91        file.write(end)
92
93def logErr(*args, **kwargs):
94    file = kwargs.pop('file', errFile)
95    end = kwargs.pop('end','\n')
96
97    strfile = io.StringIO()
98    strfile.write(logHeader('ERROR') + ' '.join(str(arg) for arg in args))
99    strfile.write(end)
100
101    if file is not None:
102        file.write(strfile.getvalue())
103    sys.exit(1)
104
105def isempty(s):
106    """Return True if s is nothing but white space, False otherwise"""
107    return len(''.join(s.split())) == 0
108
109class pageInfo:
110    """Information about a ref page relative to the file it is extracted from."""
111    def __init__(self):
112        self.extractPage = True
113        """True if page should be extracted"""
114
115        self.Warning  = None
116        """string warning if page is suboptimal or cannot be generated"""
117
118        self.embed    = False
119        """False or the name of the ref page this include is embedded within"""
120
121        self.type     = None
122        """refpage type attribute - 'structs', 'protos', 'freeform', etc."""
123
124        self.name     = None
125        """struct/proto/enumerant/etc. name"""
126
127        self.desc     = None
128        """short description of ref page"""
129
130        self.begin    = None
131        """index of first line of the page (heuristic or // refBegin)"""
132
133        self.include  = None
134        """index of include:: line defining the page"""
135
136        self.param    = None
137        """index of first line of parameter/member definitions"""
138
139        self.body     = None
140        """index of first line of body text"""
141
142        self.validity = None
143        """index of validity include"""
144
145        self.end      = None
146        """index of last line of the page (heuristic validity include, or // refEnd)"""
147
148        self.alias    = ''
149        """aliases of this name, if supplied, or ''"""
150
151        self.refs     = ''
152        """cross-references on // refEnd line, if supplied"""
153
154        self.spec     = None
155        """'spec' attribute in refpage open block, if supplied, or None for the default ('api') type"""
156
157        self.anchor   = None
158        """'anchor' attribute in refpage open block, if supplied, or inferred to be the same as the 'name'"""
159
160def printPageInfoField(desc, line, file):
161    """Print a single field of a pageInfo struct, possibly None.
162
163    - desc - string description of field
164    - line - field value or None
165    - file - indexed by line"""
166    if line is not None:
167        logDiag(desc + ':', line + 1, '\t-> ', file[line], end='')
168    else:
169        logDiag(desc + ':', line)
170
171def printPageInfo(pi, file):
172    """Print out fields of a pageInfo struct
173
174    - pi - pageInfo
175    - file - indexed by pageInfo"""
176    logDiag('TYPE:   ', pi.type)
177    logDiag('NAME:   ', pi.name)
178    logDiag('WARNING:', pi.Warning)
179    logDiag('EXTRACT:', pi.extractPage)
180    logDiag('EMBED:  ', pi.embed)
181    logDiag('DESC:   ', pi.desc)
182    printPageInfoField('BEGIN   ', pi.begin,    file)
183    printPageInfoField('INCLUDE ', pi.include,  file)
184    printPageInfoField('PARAM   ', pi.param,    file)
185    printPageInfoField('BODY    ', pi.body,     file)
186    printPageInfoField('VALIDITY', pi.validity, file)
187    printPageInfoField('END     ', pi.end,      file)
188    logDiag('REFS: "' + pi.refs + '"')
189
190def prevPara(file, line):
191    """Go back one paragraph from the specified line and return the line number
192    of the first line of that paragraph.
193
194    Paragraphs are delimited by blank lines. It is assumed that the
195    current line is the first line of a paragraph.
196
197    - file is an array of strings
198    - line is the starting point (zero-based)"""
199    # Skip over current paragraph
200    while (line >= 0 and not isempty(file[line])):
201        line = line - 1
202    # Skip over white space
203    while (line >= 0 and isempty(file[line])):
204        line = line - 1
205    # Skip to first line of previous paragraph
206    while (line >= 1 and not isempty(file[line-1])):
207        line = line - 1
208    return line
209
210def nextPara(file, line):
211    """Go forward one paragraph from the specified line and return the line
212    number of the first line of that paragraph.
213
214    Paragraphs are delimited by blank lines. It is assumed that the
215    current line is standalone (which is bogus).
216
217    - file is an array of strings
218    - line is the starting point (zero-based)"""
219    maxLine = len(file) - 1
220    # Skip over current paragraph
221    while (line != maxLine and not isempty(file[line])):
222        line = line + 1
223    # Skip over white space
224    while (line != maxLine and isempty(file[line])):
225        line = line + 1
226    return line
227
228def lookupPage(pageMap, name):
229    """Return (creating if needed) the pageInfo entry in pageMap for name"""
230    if name not in pageMap:
231        pi = pageInfo()
232        pi.name = name
233        pageMap[name] = pi
234    else:
235        pi = pageMap[name]
236    return pi
237
238def loadFile(filename):
239    """Load a file into a list of strings. Return the list or None on failure"""
240    try:
241        fp = open(filename, 'r', encoding='utf-8')
242    except:
243        logWarn('Cannot open file', filename, ':', sys.exc_info()[0])
244        return None
245
246    file = fp.readlines()
247    fp.close()
248
249    return file
250
251def clampToBlock(line, minline, maxline):
252    """Clamp a line number to be in the range [minline,maxline].
253
254    If the line number is None, just return it.
255    If minline is None, do not clamp to that value."""
256    if line is None:
257        return line
258    if minline and line < minline:
259        return minline
260    if line > maxline:
261        return maxline
262
263    return line
264
265def fixupRefs(pageMap, specFile, file):
266    """Fill in missing fields in pageInfo structures, to the extent they can be
267    inferred.
268
269    - pageMap - dictionary of pageInfo structures
270    - specFile - filename
271    - file - list of strings making up the file, indexed by pageInfo"""
272    # All potential ref pages are now in pageMap. Process them to
273    # identify actual page start/end/description boundaries, if
274    # not already determined from the text.
275    for name in sorted(pageMap.keys()):
276        pi = pageMap[name]
277
278        # # If nothing is found but an include line with no begin, validity,
279        # # or end, this is not intended as a ref page (yet). Set the begin
280        # # line to the include line, so autogeneration can at least
281        # # pull the include out, but mark it not to be extracted.
282        # # Examples include the host sync table includes in
283        # # chapters/fundamentals.txt and the table of Vk*Flag types in
284        # # appendices/boilerplate.txt.
285        # if pi.begin is None and pi.validity is None and pi.end is None:
286        #     pi.begin = pi.include
287        #     pi.extractPage = False
288        #     pi.Warning = 'No begin, validity, or end lines identified'
289        #     continue
290
291        # Using open block delimiters, ref pages must *always* have a
292        # defined begin and end. If either is undefined, that is fatal.
293        if pi.begin is None:
294            pi.extractPage = False
295            pi.Warning = 'Can\'t identify begin of ref page open block'
296            continue
297
298        if pi.end is None:
299            pi.extractPage = False
300            pi.Warning = 'Can\'t identify end of ref page open block'
301            continue
302
303        # If there is no description of the page, infer one from the type
304        if pi.desc is None:
305            if pi.type is not None:
306                # pi.desc = pi.type[0:len(pi.type)-1] + ' (no short description available)'
307                pi.Warning = 'No short description available; could infer from the type and name'
308            else:
309                pi.extractPage = False
310                pi.Warning = 'No short description available, cannot infer from the type'
311                continue
312
313        # Try to determine where the parameter and body sections of the page
314        # begin. funcpointer, proto, and struct pages infer the location of
315        # the parameter and body sections. Other pages infer the location of
316        # the body, but have no parameter sections.
317        #
318        #@ Probably some other types infer this as well - refer to list of
319        #@ all page types in genRef.py:emitPage()
320        if pi.include is not None:
321            if pi.type in ['funcpointers', 'protos', 'structs']:
322                pi.param = nextPara(file, pi.include)
323                if pi.body is None:
324                    pi.body = nextPara(file, pi.param)
325            else:
326                if pi.body is None:
327                    pi.body = nextPara(file, pi.include)
328        else:
329            pi.Warning = 'Page does not have an API definition include::'
330
331        # It is possible for the inferred param and body lines to run past
332        # the end of block, if, for example, there is no parameter section.
333        pi.param = clampToBlock(pi.param, pi.include, pi.end)
334        pi.body = clampToBlock(pi.body, pi.param, pi.end)
335
336        # We can get to this point with .include, .param, and .validity
337        # all being None, indicating those sections were not found.
338
339        logDiag('fixupRefs: after processing,', pi.name, 'looks like:')
340        printPageInfo(pi, file)
341
342    # Now that all the valid pages have been found, try to make some
343    # inferences about invalid pages.
344    #
345    # If a reference without a .end is entirely inside a valid reference,
346    # then it is intentionally embedded - may want to create an indirect
347    # page that links into the embedding page. This is done by a very
348    # inefficient double loop, but the loop depth is small.
349    for name in sorted(pageMap.keys()):
350        pi = pageMap[name]
351
352        if pi.end is None:
353            for embedName in sorted(pageMap.keys()):
354                logDiag('fixupRefs: comparing', pi.name, 'to', embedName)
355                embed = pageMap[embedName]
356                # Do not check embeddings which are themselves invalid
357                if not embed.extractPage:
358                    logDiag('Skipping check for embedding in:', embed.name)
359                    continue
360                if embed.begin is None or embed.end is None:
361                    logDiag('fixupRefs:', name + ':',
362                            'can\'t compare to unanchored ref:', embed.name,
363                            'in', specFile, 'at line', pi.include )
364                    printPageInfo(pi, file)
365                    printPageInfo(embed, file)
366                # If an embed is found, change the error to a warning
367                elif (pi.include is not None and pi.include >= embed.begin and
368                      pi.include <= embed.end):
369                    logDiag('fixupRefs: Found embed for:', name,
370                            'inside:', embedName,
371                            'in', specFile, 'at line', pi.include )
372                    pi.embed = embed.name
373                    pi.Warning = 'Embedded in definition for ' + embed.name
374                    break
375                else:
376                    logDiag('fixupRefs: No embed match for:', name,
377                            'inside:', embedName, 'in', specFile,
378                            'at line', pi.include)
379
380
381# Patterns used to recognize interesting lines in an asciidoc source file.
382# These patterns are only compiled once.
383INCSVAR_DEF = re.compile(r':INCS-VAR: (?P<value>.*)')
384endifPat   = re.compile(r'^endif::(?P<condition>[\w_+,]+)\[\]')
385beginPat   = re.compile(r'^\[open,(?P<attribs>refpage=.*)\]')
386# attribute key/value pairs of an open block
387attribStr  = r"([a-z]+)='([^'\\]*(?:\\.[^'\\]*)*)'"
388attribPat  = re.compile(attribStr)
389bodyPat    = re.compile(r'^// *refBody')
390errorPat   = re.compile(r'^// *refError')
391
392# This regex transplanted from check_spec_links
393# It looks for either OpenXR or Vulkan generated file conventions, and for
394# the api/validity include (generated_type), protos/struct/etc path
395# (category), and API name (entity_name). It could be put into the API
396# conventions object.
397INCLUDE = re.compile(
398        r'include::(?P<directory_traverse>((../){1,4}|\{INCS-VAR\}/|\{generated\}/)(generated/)?)(?P<generated_type>[\w]+)/(?P<category>\w+)/(?P<entity_name>[^./]+).txt[\[][\]]')
399
400
401def findRefs(file, filename):
402    """Identify reference pages in a list of strings, returning a dictionary of
403    pageInfo entries for each one found, or None on failure."""
404    setLogSourcefile(filename)
405    setLogProcname('findRefs')
406
407    # To reliably detect the open blocks around reference pages, we must
408    # first detect the '[open,refpage=...]' markup delimiting the block;
409    # skip past the '--' block delimiter on the next line; and identify the
410    # '--' block delimiter closing the page.
411    # This cannot be done solely with pattern matching, and requires state to
412    # track 'inside/outside block'.
413    # When looking for open blocks, possible states are:
414    #   'outside' - outside a block
415    #   'start' - have found the '[open...]' line
416    #   'inside' - have found the following '--' line
417    openBlockState = 'outside'
418
419    # Dictionary of interesting line numbers and strings related to an API
420    # name
421    pageMap = {}
422
423    numLines = len(file)
424    line = 0
425
426    # Track the pageInfo object corresponding to the current open block
427    pi = None
428    incsvar = None
429
430    while (line < numLines):
431        setLogLine(line)
432
433        # Look for a file-wide definition
434        matches = INCSVAR_DEF.match(file[line])
435        if matches:
436            incsvar = matches.group('value')
437            logDiag('Matched INCS-VAR definition:', incsvar)
438
439            line = line + 1
440            continue
441
442        # Perform INCS-VAR substitution immediately.
443        if incsvar and '{INCS-VAR}' in file[line]:
444            newLine = file[line].replace('{INCS-VAR}', incsvar)
445            logDiag('PERFORMING SUBSTITUTION', file[line], '->', newLine)
446            file[line] = newLine
447
448        # Only one of the patterns can possibly match. Add it to
449        # the dictionary for that name.
450
451        # [open,refpage=...] starting a refpage block
452        matches = beginPat.search(file[line])
453        if matches is not None:
454            logDiag('Matched open block pattern')
455            attribs = matches.group('attribs')
456
457            # If the previous open block was not closed, raise an error
458            if openBlockState != 'outside':
459                logErr('Nested open block starting at line', line, 'of',
460                       filename)
461
462            openBlockState = 'start'
463
464            # Parse the block attributes
465            matches = attribPat.findall(attribs)
466
467            # Extract each attribute
468            name = None
469            desc = None
470            refpage_type = None
471            spec_type = None
472            anchor = None
473            alias = None
474            xrefs = None
475
476            for (key,value) in matches:
477                logDiag('got attribute', key, '=', value)
478                if key == 'refpage':
479                    name = value
480                elif key == 'desc':
481                    desc = unescapeQuotes(value)
482                elif key == 'type':
483                    refpage_type = value
484                elif key == 'spec':
485                    spec_type = value
486                elif key == 'anchor':
487                    anchor = value
488                elif key == 'alias':
489                    alias = value
490                elif key == 'xrefs':
491                    xrefs = value
492                else:
493                    logWarn('unknown open block attribute:', key)
494
495            if name is None or desc is None or refpage_type is None:
496                logWarn('missing one or more required open block attributes:'
497                        'refpage, desc, or type')
498                # Leave pi is None so open block delimiters are ignored
499            else:
500                pi = lookupPage(pageMap, name)
501                pi.desc = desc
502                # Must match later type definitions in interface/validity includes
503                pi.type = refpage_type
504                pi.spec = spec_type
505                pi.anchor = anchor
506                if alias:
507                    pi.alias = alias
508                if xrefs:
509                    pi.refs = xrefs
510                logDiag('open block for', name, 'added DESC =', desc,
511                        'TYPE =', refpage_type, 'ALIAS =', alias,
512                        'XREFS =', xrefs, 'SPEC =', spec_type,
513                        'ANCHOR =', anchor)
514
515            line = line + 1
516            continue
517
518        # '--' starting or ending and open block
519        if file[line].rstrip() == '--':
520            if openBlockState == 'outside':
521                # Only refpage open blocks should use -- delimiters
522                logWarn('Unexpected double-dash block delimiters')
523            elif openBlockState == 'start':
524                # -- delimiter following [open,refpage=...]
525                openBlockState = 'inside'
526
527                if pi is None:
528                    logWarn('no pageInfo available for opening -- delimiter')
529                else:
530                    pi.begin = line + 1
531                    logDiag('opening -- delimiter: added BEGIN =', pi.begin)
532            elif openBlockState == 'inside':
533                # -- delimiter ending an open block
534                if pi is None:
535                    logWarn('no pageInfo available for closing -- delimiter')
536                else:
537                    pi.end = line - 1
538                    logDiag('closing -- delimiter: added END =', pi.end)
539
540                openBlockState = 'outside'
541                pi = None
542            else:
543                logWarn('unknown openBlockState:', openBlockState)
544
545            line = line + 1
546            continue
547
548        matches = INCLUDE.search(file[line])
549        if matches is not None:
550            # Something got included, not sure what yet.
551            gen_type = matches.group('generated_type')
552            refpage_type = matches.group('category')
553            name = matches.group('entity_name')
554
555            # This will never match in OpenCL
556            if gen_type == 'validity':
557                logDiag('Matched validity pattern')
558                if pi is not None:
559                    if pi.type and refpage_type != pi.type:
560                        logWarn('ERROR: pageMap[' + name + '] type:',
561                                pi.type, 'does not match type:', refpage_type)
562                    pi.type = refpage_type
563                    pi.validity = line
564                    logDiag('added TYPE =', pi.type, 'VALIDITY =', pi.validity)
565                else:
566                    logWarn('validity include:: line NOT inside block')
567
568                line = line + 1
569                continue
570
571            if gen_type == 'api':
572                logDiag('Matched include pattern')
573                if pi is not None:
574                    if pi.include is not None:
575                        logDiag('found multiple includes for this block')
576                    if pi.type and refpage_type != pi.type:
577                        logWarn('ERROR: pageMap[' + name + '] type:',
578                                pi.type, 'does not match type:', refpage_type)
579                    pi.type = refpage_type
580                    pi.include = line
581                    logDiag('added TYPE =', pi.type, 'INCLUDE =', pi.include)
582                else:
583                    logWarn('interface include:: line NOT inside block')
584
585                line = line + 1
586                continue
587
588            logDiag('ignoring unrecognized include line ', matches.group())
589
590        # Vulkan 1.1 markup allows the last API include construct to be
591        # followed by an asciidoctor endif:: construct (and also preceded,
592        # at some distance).
593        # This looks for endif:: immediately following an include:: line
594        # and, if found, moves the include boundary to this line.
595        matches = endifPat.search(file[line])
596        if matches is not None and pi is not None:
597            if pi.include == line - 1:
598                logDiag('Matched endif pattern following include; moving include')
599                pi.include = line
600            else:
601                logDiag('Matched endif pattern (not following include)')
602
603            line = line + 1
604            continue
605
606        matches = bodyPat.search(file[line])
607        if matches is not None:
608            logDiag('Matched // refBody pattern')
609            if pi is not None:
610                pi.body = line
611                logDiag('added BODY =', pi.body)
612            else:
613                logWarn('// refBody line NOT inside block')
614
615            line = line + 1
616            continue
617
618        # OpenCL spec uses // refError to tag "validity" (Errors) language,
619        # instead of /validity/ includes.
620        matches = errorPat.search(file[line])
621        if matches is not None:
622            logDiag('Matched // refError pattern')
623            if pi is not None:
624                pi.validity = line
625                logDiag('added VALIDITY (refError) =', pi.validity)
626            else:
627                logWarn('// refError line NOT inside block')
628
629            line = line + 1
630            continue
631
632        line = line + 1
633        continue
634
635    if pi is not None:
636        logErr('Unclosed open block at EOF!')
637
638    setLogSourcefile(None)
639    setLogProcname(None)
640    setLogLine(None)
641
642    return pageMap
643
644
645def getBranch():
646    """Determine current git branch
647
648    Returns (branch name, ''), or (None, stderr output) if the branch name
649    cannot be determined"""
650
651    command = [ 'git', 'symbolic-ref', '--short', 'HEAD' ]
652    results = subprocess.run(command,
653                             stdout=subprocess.PIPE,
654                             stderr=subprocess.PIPE)
655
656    # git command failed
657    if len(results.stderr) > 0:
658        return (None, results.stderr)
659
660    # Remove newline from output and convert to a string
661    branch = results.stdout.rstrip().decode()
662    if len(branch) > 0:
663        # Strip trailing newline
664        branch = results.stdout.decode()[0:-1]
665
666    return (branch, '')
667