1#!/usr/bin/python -i 2 3import sys 4try: 5 import urllib.request as urllib2 6except ImportError: 7 import urllib2 8from bs4 import BeautifulSoup 9import json 10import vuid_mapping 11 12############################# 13# spec.py script 14# 15# Overview - this script is intended to generate validation error codes and message strings from the json spec file 16# that contains all of the valid usage statements. In addition to generating the header file, it provides a number of 17# corrollary services to aid in generating/updating the header. 18# 19# Ideal flow - Pull the valid usage text and IDs from the spec json, pull the IDs from the validation error database, 20# then update the database with any new IDs from the json file and generate new database and header file. 21# 22# TODO: 23# 1. When VUs go away (in error DB, but not in json) need to report them and remove from DB as deleted 24# 25############################# 26 27 28out_filename = "../layers/vk_validation_error_messages.h" # can override w/ '-out <filename>' option 29db_filename = "../layers/vk_validation_error_database.txt" # can override w/ '-gendb <filename>' option 30json_filename = None # con pass in w/ '-json <filename> option 31gen_db = False # set to True when '-gendb <filename>' option provided 32json_compare = False # compare existing DB to json file input 33json_url = "https://www.khronos.org/registry/vulkan/specs/1.0-extensions/validation/validusage.json" 34read_json = False 35# This is the root spec link that is used in error messages to point users to spec sections 36#old_spec_url = "https://www.khronos.org/registry/vulkan/specs/1.0/xhtml/vkspec.html" 37spec_url = "https://www.khronos.org/registry/vulkan/specs/1.0-extensions/html/vkspec.html" 38core_url = "https://www.khronos.org/registry/vulkan/specs/1.0/html/vkspec.html" 39ext_url = "https://www.khronos.org/registry/vulkan/specs/1.0-extensions/html/vkspec.html" 40# After the custom validation error message, this is the prefix for the standard message that includes the 41# spec valid usage language as well as the link to nearest section of spec to that language 42error_msg_prefix = "The spec valid usage text states " 43validation_error_enum_name = "VALIDATION_ERROR_" 44 45def printHelp(): 46 print ("Usage: python spec.py [-out <headerfile.h>] [-gendb <databasefile.txt>] [-update] [-json <json_file>] [-help]") 47 print ("\n Default script behavior is to parse the specfile and generate a header of unique error enums and corresponding error messages based on the specfile.\n") 48 print (" Default specfile is from online at %s" % (spec_url)) 49 print (" Default headerfile is %s" % (out_filename)) 50 print (" Default databasefile is %s" % (db_filename)) 51 print ("\nIf '-gendb' option is specified then a database file is generated to default file or <databasefile.txt> if supplied. The database file stores") 52 print (" the list of enums and their error messages.") 53 print ("\nIf '-update' option is specified this triggers the master flow to automate updating header and database files using default db file as baseline") 54 print (" and online spec file as the latest. The default header and database files will be updated in-place for review and commit to the git repo.") 55 print ("\nIf '-json' option is used trigger the script to load in data from a json file.") 56 print ("\nIf '-json-file' option is it will point to a local json file, else '%s' is used from the web." % (json_url)) 57 58def get8digithex(dec_num): 59 """Convert a decimal # into an 8-digit hex""" 60 if dec_num > 4294967295: 61 print ("ERROR: Decimal # %d can't be represented in 8 hex digits" % (dec_num)) 62 sys.exit() 63 hex_num = hex(dec_num) 64 return hex_num[2:].zfill(8) 65 66class Specification: 67 def __init__(self): 68 self.tree = None 69 self.error_db_dict = {} # dict of previous error values read in from database file 70 self.delimiter = '~^~' # delimiter for db file 71 # Global dicts used for tracking spec updates from old to new VUs 72 self.orig_no_link_msg_dict = {} # Pair of API,Original msg w/o spec link to ID list mapping 73 self.orig_core_msg_dict = {} # Pair of API,Original core msg (no link or section) to ID list mapping 74 self.last_mapped_id = -10 # start as negative so we don't hit an accidental sequence 75 self.orig_test_imp_enums = set() # Track old enums w/ tests and/or implementation to flag any that aren't carried fwd 76 # Dict of data from json DB 77 # Key is API,<short_msg> which leads to dict w/ following values 78 # 'ext' -> <core|<ext_name>> 79 # 'string_vuid' -> <string_vuid> 80 # 'number_vuid' -> <numerical_vuid> 81 self.json_db = {} 82 self.json_missing = 0 83 self.struct_to_func_map = {} # Map structs to the API func that they fall under in the spec 84 self.duplicate_json_key_count = 0 85 self.copyright = """/* THIS FILE IS GENERATED. DO NOT EDIT. */ 86 87/* 88 * Vulkan 89 * 90 * Copyright (c) 2016 Google Inc. 91 * Copyright (c) 2016 LunarG, Inc. 92 * 93 * Licensed under the Apache License, Version 2.0 (the "License"); 94 * you may not use this file except in compliance with the License. 95 * You may obtain a copy of the License at 96 * 97 * http://www.apache.org/licenses/LICENSE-2.0 98 * 99 * Unless required by applicable law or agreed to in writing, software 100 * distributed under the License is distributed on an "AS IS" BASIS, 101 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 102 * See the License for the specific language governing permissions and 103 * limitations under the License. 104 * 105 * Author: Tobin Ehlis <tobine@google.com> 106 */""" 107 108 def readJSON(self): 109 """Read in JSON file""" 110 if json_filename is not None: 111 with open(json_filename) as jsf: 112 self.json_data = json.load(jsf, encoding='utf-8') 113 else: 114 response = urllib2.urlopen(json_url).read().decode('utf-8') 115 self.json_data = json.loads(response) 116 117 def parseJSON(self): 118 """Parse JSON VUIDs into data struct""" 119 # Format of JSON file is: 120 # "API": { "core|EXT": [ {"vuid": "<id>", "text": "<VU txt>"}]}, 121 # "VK_KHX_external_memory" & "VK_KHX_device_group" - extension case (vs. "core") 122 for top_level in sorted(self.json_data): 123 if "validation" == top_level: 124 for api in sorted(self.json_data[top_level]): 125 for ext in sorted(self.json_data[top_level][api]): 126 for vu_txt_dict in self.json_data[top_level][api][ext]: 127 print ("Looking at dict for api:ext entry %s:%s" % (api, ext)) 128 vuid = vu_txt_dict['vuid'] 129 vutxt = vu_txt_dict['text'] 130 #print ("%s:%s:%s:%s" % (api, ext, vuid, vutxt)) 131 #print ("VUTXT orig:%s" % (vutxt)) 132 just_txt = BeautifulSoup(vutxt, 'html.parser') 133 #print ("VUTXT only:%s" % (just_txt.get_text())) 134 num_vuid = vuid_mapping.convertVUID(vuid) 135 self.json_db[vuid] = {} 136 self.json_db[vuid]['ext'] = ext 137 self.json_db[vuid]['number_vuid'] = num_vuid 138 self.json_db[vuid]['struct_func'] = api 139 just_txt = just_txt.get_text().strip() 140 unicode_map = { 141 u"\u2019" : "'", 142 u"\u201c" : "\"", 143 u"\u201d" : "\"", 144 u"\u2192" : "->", 145 } 146 for um in unicode_map: 147 just_txt = just_txt.replace(um, unicode_map[um]) 148 self.json_db[vuid]['vu_txt'] = just_txt.replace("\\", "") 149 print ("Spec vu txt:%s" % (self.json_db[vuid]['vu_txt'])) 150 #sys.exit() 151 152 def compareJSON(self): 153 """Compare parsed json file with existing data read in from DB file""" 154 json_db_set = set() 155 for vuid in self.json_db: # pull entries out and see which fields we're missing from error_db 156 json_db_set.add(vuid) 157 for enum in self.error_db_dict: 158 vuid_string = self.error_db_dict[enum]['vuid_string'] 159 if vuid_string not in self.json_db: 160 #print ("Full string for %s is:%s" % (enum, full_error_string)) 161 print ("WARN: Couldn't find vuid_string in json db:%s" % (vuid_string)) 162 self.json_missing = self.json_missing + 1 163 self.error_db_dict[enum]['ext'] = 'core' 164 # TODO: Currently GL843 tracks 2 VUs that are missing from json incorrectly 165 # Fix will land in 1.0.51 spec. After that we should take some alternative 166 # action here to indicate that VUs have gone away. 167 # Can have a removed_enums set that we add to and report to user 168 #sys.exit() 169 else: 170 json_db_set.remove(vuid_string) 171 self.error_db_dict[enum]['ext'] = self.json_db[vuid_string]['ext'] 172 if 'core' == self.json_db[vuid_string]['ext'] or '!' in self.json_db[vuid_string]['ext']: 173 spec_link = "%s#%s" % (core_url, vuid_string) 174 else: 175 spec_link = "%s#%s" % (ext_url, vuid_string) 176 self.error_db_dict[enum]['error_msg'] = "%s'%s' (%s)" % (error_msg_prefix, self.json_db[vuid_string]['vu_txt'], spec_link) 177 print ("Updated error_db error_msg:%s" % (self.error_db_dict[enum]['error_msg'])) 178 #sys.exit() 179 print ("These json DB entries are not in error DB:") 180 for extra_vuid in json_db_set: 181 print ("\t%s" % (extra_vuid)) 182 # Add these missing entries into the error_db 183 # Create link into core or ext spec as needed 184 if 'core' == self.json_db[extra_vuid]['ext'] or '!' in self.json_db[extra_vuid]['ext']: 185 spec_link = "%s#%s" % (core_url, extra_vuid) 186 else: 187 spec_link = "%s#%s" % (ext_url, extra_vuid) 188 error_enum = "%s%s" % (validation_error_enum_name, get8digithex(self.json_db[extra_vuid]['number_vuid'])) 189 self.error_db_dict[error_enum] = {} 190 self.error_db_dict[error_enum]['check_implemented'] = 'N' 191 self.error_db_dict[error_enum]['testname'] = 'None' 192 self.error_db_dict[error_enum]['api'] = self.json_db[extra_vuid]['struct_func'] 193 self.error_db_dict[error_enum]['vuid_string'] = extra_vuid 194 self.error_db_dict[error_enum]['error_msg'] = "%s'%s' (%s)" % (error_msg_prefix, self.json_db[extra_vuid]['vu_txt'], spec_link) 195 self.error_db_dict[error_enum]['note'] = '' 196 self.error_db_dict[error_enum]['ext'] = self.json_db[extra_vuid]['ext'] 197 implicit = False 198 last_segment = extra_vuid.split("-")[-1] 199 if last_segment in vuid_mapping.implicit_type_map: 200 implicit = True 201 elif not last_segment.isdigit(): # Explicit ids should only have digits in last segment 202 print ("ERROR: Found last segment of val error ID that isn't in implicit map and doesn't have numbers in last segment: %s" % (last_segment)) 203 sys.exit() 204 self.error_db_dict[error_enum]['implicit'] = implicit 205 206 def genHeader(self, header_file): 207 """Generate a header file based on the contents of a parsed spec""" 208 print ("Generating header %s..." % (header_file)) 209 file_contents = [] 210 file_contents.append(self.copyright) 211 file_contents.append('\n#pragma once') 212 file_contents.append('\n// Disable auto-formatting for generated file') 213 file_contents.append('// clang-format off') 214 file_contents.append('\n#include <unordered_map>') 215 file_contents.append('\n// enum values for unique validation error codes') 216 file_contents.append('// Corresponding validation error message for each enum is given in the mapping table below') 217 file_contents.append('// When a given error occurs, these enum values should be passed to the as the messageCode') 218 file_contents.append('// parameter to the PFN_vkDebugReportCallbackEXT function') 219 enum_decl = ['enum UNIQUE_VALIDATION_ERROR_CODE {\n VALIDATION_ERROR_UNDEFINED = -1,'] 220 error_string_map = ['static std::unordered_map<int, char const *const> validation_error_map{'] 221 enum_value = 0 222 max_enum_val = 0 223 for enum in sorted(self.error_db_dict): 224 #print ("Header enum is %s" % (enum)) 225 # TMP: Use updated value 226 vuid_str = self.error_db_dict[enum]['vuid_string'] 227 if vuid_str in self.json_db: 228 enum_value = self.json_db[vuid_str]['number_vuid'] 229 else: 230 enum_value = vuid_mapping.convertVUID(vuid_str) 231 new_enum = "%s%s" % (validation_error_enum_name, get8digithex(enum_value)) 232 enum_decl.append(' %s = 0x%s,' % (new_enum, get8digithex(enum_value))) 233 error_string_map.append(' {%s, "%s"},' % (new_enum, self.error_db_dict[enum]['error_msg'].replace('"', '\\"'))) 234 max_enum_val = max(max_enum_val, enum_value) 235 enum_decl.append(' %sMAX_ENUM = %d,' % (validation_error_enum_name, max_enum_val + 1)) 236 enum_decl.append('};') 237 error_string_map.append('};\n') 238 file_contents.extend(enum_decl) 239 file_contents.append('\n// Mapping from unique validation error enum to the corresponding error message') 240 file_contents.append('// The error message should be appended to the end of a custom error message that is passed') 241 file_contents.append('// as the pMessage parameter to the PFN_vkDebugReportCallbackEXT function') 242 file_contents.extend(error_string_map) 243 #print ("File contents: %s" % (file_contents)) 244 with open(header_file, "w") as outfile: 245 outfile.write("\n".join(file_contents)) 246 def genDB(self, db_file): 247 """Generate a database of check_enum, check_coded?, testname, API, VUID_string, core|ext, error_string, notes""" 248 db_lines = [] 249 # Write header for database file 250 db_lines.append("# This is a database file with validation error check information") 251 db_lines.append("# Comments are denoted with '#' char") 252 db_lines.append("# The format of the lines is:") 253 db_lines.append("# <error_enum>%s<check_implemented>%s<testname>%s<api>%s<vuid_string>%s<core|ext>%s<errormsg>%s<note>" % (self.delimiter, self.delimiter, self.delimiter, self.delimiter, self.delimiter, self.delimiter, self.delimiter)) 254 db_lines.append("# error_enum: Unique error enum for this check of format %s<uniqueid>" % validation_error_enum_name) 255 db_lines.append("# check_implemented: 'Y' if check has been implemented in layers, or 'N' for not implemented") 256 db_lines.append("# testname: Name of validation test for this check, 'Unknown' for unknown, 'None' if not implemented, or 'NotTestable' if cannot be implemented") 257 db_lines.append("# api: Vulkan API function that this check is related to") 258 db_lines.append("# vuid_string: Unique string to identify this check") 259 db_lines.append("# core|ext: Either 'core' for core spec or some extension string that indicates the extension required for this VU to be relevant") 260 db_lines.append("# errormsg: The unique error message for this check that includes spec language and link") 261 db_lines.append("# note: Free txt field with any custom notes related to the check in question") 262 for enum in sorted(self.error_db_dict): 263 print ("Gen DB for enum %s" % (enum)) 264 implicit = self.error_db_dict[enum]['implicit'] 265 implemented = self.error_db_dict[enum]['check_implemented'] 266 testname = self.error_db_dict[enum]['testname'] 267 note = self.error_db_dict[enum]['note'] 268 core_ext = self.error_db_dict[enum]['ext'] 269 self.error_db_dict[enum]['vuid_string'] = self.error_db_dict[enum]['vuid_string'] 270 if implicit and 'implicit' not in note: # add implicit note 271 if '' != note: 272 note = "implicit, %s" % (note) 273 else: 274 note = "implicit" 275 db_lines.append("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s" % (enum, self.delimiter, implemented, self.delimiter, testname, self.delimiter, self.error_db_dict[enum]['api'], self.delimiter, self.error_db_dict[enum]['vuid_string'], self.delimiter, core_ext, self.delimiter, self.error_db_dict[enum]['error_msg'], self.delimiter, note)) 276 db_lines.append("\n") # newline at end of file 277 print ("Generating database file %s" % (db_file)) 278 with open(db_file, "w") as outfile: 279 outfile.write("\n".join(db_lines)) 280 def readDB(self, db_file): 281 """Read a db file into a dict, refer to genDB function above for format of each line""" 282 with open(db_file, "r", encoding='utf-8') as infile: 283 for line in infile: 284 line = line.strip() 285 if line.startswith('#') or '' == line: 286 continue 287 db_line = line.split(self.delimiter) 288 if len(db_line) != 8: 289 print ("ERROR: Bad database line doesn't have 8 elements: %s" % (line)) 290 error_enum = db_line[0] 291 implemented = db_line[1] 292 testname = db_line[2] 293 api = db_line[3] 294 vuid_str = db_line[4] 295 core_ext = db_line[5] 296 error_str = db_line[6] 297 note = db_line[7] 298 # Also read complete database contents into our class var for later use 299 self.error_db_dict[error_enum] = {} 300 self.error_db_dict[error_enum]['check_implemented'] = implemented 301 self.error_db_dict[error_enum]['testname'] = testname 302 self.error_db_dict[error_enum]['api'] = api 303 self.error_db_dict[error_enum]['vuid_string'] = vuid_str 304 self.error_db_dict[error_enum]['ext'] = core_ext 305 self.error_db_dict[error_enum]['error_msg'] = error_str 306 self.error_db_dict[error_enum]['note'] = note 307 implicit = False 308 last_segment = vuid_str.split("-")[-1] 309 if last_segment in vuid_mapping.implicit_type_map: 310 implicit = True 311 elif not last_segment.isdigit(): # Explicit ids should only have digits in last segment 312 print ("ERROR: Found last segment of val error ID that isn't in implicit map and doesn't have numbers in last segment: %s" % (last_segment)) 313 sys.exit() 314 self.error_db_dict[error_enum]['implicit'] = implicit 315if __name__ == "__main__": 316 i = 1 317 use_online = True # Attempt to grab spec from online by default 318 while (i < len(sys.argv)): 319 arg = sys.argv[i] 320 i = i + 1 321 if (arg == '-json-file'): 322 json_filename = sys.argv[i] 323 i = i + 1 324 elif (arg == '-json'): 325 read_json = True 326 elif (arg == '-json-compare'): 327 json_compare = True 328 elif (arg == '-out'): 329 out_filename = sys.argv[i] 330 i = i + 1 331 elif (arg == '-gendb'): 332 gen_db = True 333 # Set filename if supplied, else use default 334 if i < len(sys.argv) and not sys.argv[i].startswith('-'): 335 db_filename = sys.argv[i] 336 i = i + 1 337 elif (arg == '-update'): 338 read_json = True 339 json_compare = True 340 gen_db = True 341 elif (arg in ['-help', '-h']): 342 printHelp() 343 sys.exit() 344 spec = Specification() 345 if read_json: 346 spec.readJSON() 347 spec.parseJSON() 348 #sys.exit() 349 if (json_compare): 350 # Read in current spec info from db file 351 (orig_err_msg_dict) = spec.readDB(db_filename) 352 spec.compareJSON() 353 print ("Found %d missing db entries in json db" % (spec.json_missing)) 354 print ("Found %d duplicate json entries" % (spec.duplicate_json_key_count)) 355 spec.genDB(db_filename) 356 if (gen_db): 357 spec.genDB(db_filename) 358 print ("Writing out file (-out) to '%s'" % (out_filename)) 359 spec.genHeader(out_filename) 360