1#!/usr/bin/env python3 2# coding=utf-8 3 4#=============================================================================== 5# @brief XML tools 6# Copyright (c) 2020 HiSilicon (Shanghai) Technologies CO., LIMITED. 7# Licensed under the Apache License, Version 2.0 (the "License"); 8# you may not use this file except in compliance with the License. 9# You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, software 14# distributed under the License is distributed on an "AS IS" BASIS, 15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16# See the License for the specific language governing permissions and 17# limitations under the License. 18#=============================================================================== 19import ctypes 20import sys 21import inspect 22import os 23import re 24from xml.etree import ElementTree 25from xml.dom import minidom 26 27sys.path.append(os.path.join("build", "script", "hdbxml_custom", "parse")) 28import parse_msgdefs 29 30 31def isStackmessage(id, core): 32 if (core in ['protocol_core']) and ((id >> 26) != 0x10): 33 return True 34 return False 35 36def swap16(x): 37 return (((x << 8) & 0xFF00) | 38 ((x >> 8) & 0x00FF)) 39 40def swap32(x): 41 return (((x << 24) & 0xFF000000) | 42 ((x << 8) & 0x00FF0000) | 43 ((x >> 8) & 0x0000FF00) | 44 ((x >> 24) & 0x000000FF)) 45 46def getIdFromParts(core, msg_class, msg_module, msg_id): 47 return (((core << 30) & 0xC0000000) | ((msg_class << 29) & 0x20000000) | ((msg_module << 16) & 0x00FF0000) | (swap16(msg_id) & 0x0000FFFF)) 48 49def getIdForMessage(id, core): 50 if isStackmessage(id, core): 51 # A stack message needs to be dissasembled and put back together 52 msg_module = (((id) >> 24) & 0xFF) 53 msg_log = (((id) >> 21) & 0x1) 54 msg_verbosity = (((id) >> 22) & 0x3) 55 msg_id = (((id) & 0x3ff) | (msg_log << 13) | (msg_verbosity << 14)) 56 messageIDOut = getIdFromParts(1, 1, msg_module, msg_id) 57 else: 58 # All other messages need their low two bytes swapped 59 messageIDOut = ((id) & 0xFFFF0000) | (((id) & 0x0000FF00) >> 8) | (((id) & 0x000000FF) << 8) 60 61 messageIDOut = swap32(messageIDOut) 62 63 return messageIDOut 64 65def getFirmwareVersion(): 66 versionfile = os.path.join("build_scons", "VERSION_STRING") 67 if os.path.exists(versionfile): 68 with open(versionfile) as f: 69 firmware_version = f.read().strip() 70 else: 71 # This can happen when the messagexml generation is used outside the build. 72 # The DSP build uses this via tools/scripts/messagesxml.bat 73 firmware_version = "UNKNOWN" 74 return firmware_version 75 76# just used to include comments from rules files. 77class CommentedTreeBuilder(ElementTree.TreeBuilder): 78 def __init__(self, *args, **kwargs): 79 super(CommentedTreeBuilder, self).__init__(*args, **kwargs) 80 81 def comment(self, data): 82 self.start(ElementTree.Comment, {}) 83 self.data(data) 84 self.end(ElementTree.Comment) 85 86class messageTree(object): 87 def __init__(self, distribution, firmware_version, coreName, imageName): 88 self.parser = ElementTree.XMLParser(target=CommentedTreeBuilder()) 89 ElementTree.register_namespace("","http://tempuri.org/xmlDefinition.xsd" ) 90 debugInformationAttributes = { \ 91 'Distribution': distribution, \ 92 'FirmwareVersion':firmware_version, \ 93 'MulticoreLoggingSupported':"true", \ 94 'SocXMLVersion':"2", \ 95 'SequenceNumbersSupported':"true", \ 96 'xmlns':"http://tempuri.org/xmlDefinition.xsd", \ 97 } 98 if imageName in ['SEBoot', 'SERecovery', 'updater']: 99 isBootImage = 'true' 100 else: 101 isBootImage = 'false' 102 coreAttributes = { \ 103 'CoreName': str(coreName), \ 104 'ImageName': str(imageName), \ 105 'BootImage': isBootImage, \ 106 } 107 108 self.top = ElementTree.Element('DebugInformation', debugInformationAttributes) 109 self.tree = ElementTree.ElementTree(element=self.top, file=None) 110 loghdr = ElementTree.SubElement(self.top, 'CommonLogHeader', {'LogIndexMask': '0x0000001F'}) 111 coreEntry = ElementTree.SubElement(self.top, 'Core', coreAttributes) 112 113 self.messages = ElementTree.SubElement(coreEntry, 'Messages') 114 self.structs_dict = ElementTree.SubElement(coreEntry, 'StructureDictionary') 115 self.enums_dict = ElementTree.SubElement(coreEntry, 'EnumsDictionary') 116 117 def addMessageHeader(self, messageEnumName, messageId, core, elemName): 118 messageAttributes = { \ 119 'Name': messageEnumName, \ 120 'MessageID': "0x"+str(hex(messageId))[2:].zfill(8), \ 121 'Type': 'structure', \ 122 } 123 if isStackmessage(messageId, core): 124 messageAttributes['MessageType'] = 'StackMessage' 125 message = ElementTree.SubElement(self.messages, "Message",messageAttributes) 126 return message 127 128 def addStructure(self, nodeTypeName, fieldElementType, elementSize): 129 structureAttributes = { \ 130 'Type': nodeTypeName, \ 131 'FieldType': fieldElementType, \ 132 } 133 fieldEntry = ElementTree.SubElement(self.structs_dict, 'Structure', structureAttributes) 134 return fieldEntry 135 136 def structureInDictionary(self, structName): 137 found = False 138 for i in self.structs_dict: 139 nameEntry = i.attrib["Type"] 140 if structName == nameEntry: 141 found = True 142 return found 143 144 def addField(self, parent, fieldName, nodeTypeName, fieldElementType, elementSize, length, enumName): 145 fieldAttributes = { \ 146 'FieldName': fieldName, \ 147 'Type': nodeTypeName, \ 148 'FieldType': fieldElementType, \ 149 'Size': str(elementSize), \ 150 'Length': str(length), \ 151 } 152 if enumName is not None: 153 fieldAttributes['Enum'] = enumName 154 fieldSubEntry = ElementTree.SubElement(parent, 'Field', fieldAttributes) 155 return fieldSubEntry 156 157 def addEnum(self, enum_type_name, enumList): 158 found = False 159 for i in self.enums_dict: 160 nameEntry = i.find("Name").text 161 if enum_type_name == nameEntry: 162 found = True 163 164 if not found: 165 enumsEntry = ElementTree.SubElement(self.enums_dict, 'EnumsEntry') 166 enumName = ElementTree.SubElement(enumsEntry, 'Name') 167 enumName.text = enum_type_name 168 #enumsArray = ElementTree.SubElement(enumsEntry, 'Enums') 169 for name, val in enumList: 170 self.addEnumEntry(enumsEntry, name, val) 171 172 def addEnumEntry(self, parent, name, val): 173 enumAttributes = { \ 174 'Name': name, \ 175 'Value': str(val), \ 176 } 177 enumValueEntry = ElementTree.SubElement(parent, 'Enum', enumAttributes) 178 179 def dump(self): 180 return ElementTree.dump(self.top) 181 182 def prettify(self, elem, encoding = 'utf-8'): 183 """Return a pretty-printed XML string for the Element. 184 """ 185 initial_string = ElementTree.tostring(elem, encoding=encoding, method='xml') 186 reparsed_string = minidom.parseString(initial_string) 187 return reparsed_string.toprettyxml(indent=" ") 188 189 def write_xml_file (self, file, xml_root_element, xml_declaration=False, pretty_print=False, encoding='unicode', indent='\t'): 190 if xml_declaration: 191 file.write('<?xml version="1.0" encoding="utf-8" ?>') 192 pretty_printed_xml = self.prettify(xml_root_element, encoding = 'utf-8') 193 file.write(pretty_printed_xml) 194 195 def outputMessageTree(self, regexp, rulesfilename, outfilename): 196 if regexp: 197 matcher = re.compile(regexp) 198 199 # Filter out any messages we don't want to publish, then add the rules and write it out 200 internal_message_list = [] 201 for message in self.messages: 202 name = message.attrib['Name'] 203 internal = regexp and matcher.search(name) is not None 204 if internal: 205 internal_message_list.append(message) 206 207 for message in internal_message_list: 208 name = message.attrib['Name'] 209 self.messages.remove(message) 210 211 if rulesfilename != None: 212 rules_data = ElementTree.parse(rulesfilename, self.parser) 213 if rules_data is None: 214 raise SystemExit('Error: failed to parse %s' % rulesfilename) 215 rules_root = rules_data.getroot() 216 if rules_root is None: 217 raise SystemExit('Error: failed to find root section in %s' % rulesfilename) 218 rulesEntry = self.top.find('Core') 219 if rulesEntry is None: 220 raise SystemExit('Error: failed to find "Core" section') 221 rulesEntry.extend(rules_root) 222 223 with open(outfilename, 'w') as outfile: 224 self.write_xml_file(outfile, self.top, xml_declaration=False, pretty_print=True, encoding='unicode', indent='\t') 225 226 227# Taking some ctypes element, recursively analyse it, and build a tree of nodes representing it 228def buildTree(xmlTree, message_fields, elem): 229 if hasattr(elem, "_fields_"): 230 for field in elem._fields_: 231 structAlreadyPopulated = False 232 current_message_fields = message_fields 233 fieldType = field[1] 234 arrayItemNamePostfix='' 235 236 # This is potentiallay a multi-dimensional array that must be descended into so do so until an actual type is encountered 237 while True: 238 # If this is an array there will be some kind of length 239 length = fieldType._length_ if hasattr(fieldType, "_length_") else 1 240 # This size includes all the different elements (if this is an array) or just the size of the single field if not 241 size = ctypes.sizeof(fieldType) 242 # Deduce the size of a single element 243 elementSize = size // length 244 # Pointers have a "contents" attribute 245 isPointer = hasattr(fieldType, "contents") 246 247 # Simple single dimension arrays can be handled using the length 248 isSomeKindOfArray = issubclass(fieldType, ctypes.Array) 249 if isSomeKindOfArray: 250 # This is some kind of array so get the element within the array - the fieldType from now onwards is within the array 251 fieldType = fieldType._type_ 252 253 # Base types have no fields in their "type" (note - arrays of base types are considered to be base types) 254 isBaseType = not hasattr(fieldType, "_fields_") 255 isUnion = isinstance(fieldType, ctypes.Union) 256 isEnum = hasattr(fieldType, "members") 257 258 if isEnum: 259 for class_ in inspect.getmro(fieldType): 260 if issubclass(class_, ctypes._SimpleCData) and class_ is not fieldType: 261 fullTypeName = class_ 262 break 263 else: 264 raise TypeError("Can't find a ctypes class to use for enum %s" % fieldType) 265 else: 266 fullTypeName = fieldType 267 268 # Extract a useful type name from the verbose name in CTypes 269 (junk, separator, fullTypeName) = str(fullTypeName).partition(".") 270 (fullTypeName, separator, junk) = fullTypeName.partition("'") 271 272 # Check to see if there are further array dimensions that need to be unwrapped and if not exit to actually 273 if not issubclass(fieldType, ctypes.Array): 274 break 275 276 # This is a multi-dimensional array so use a fake struct here to hold the next dimensionality 277 arrayItemNamePostfix='_item' 278 nodeTypeName = ''.join(["struct_", fullTypeName]) 279 fieldEntry = xmlTree.addField(current_message_fields, field[0], nodeTypeName, 'struct', elementSize, length, enum_type_name) 280 281 # If the fake structure has already been populated then stop here 282 if xmlTree.structureInDictionary(nodeTypeName): 283 structAlreadyPopulated = True 284 break 285 286 # Add the fake struct definition and start populating it during the next iteration 287 current_message_fields = xmlTree.addStructure(nodeTypeName, 'struct', elementSize) 288 289 # Only need to do anything if a struct has not already been populated 290 if not structAlreadyPopulated: 291 # Now dealing with something that is not an array 292 (friendlyTypeName, separator, junk) = fullTypeName.partition("_Array_") 293 nodeTypeName = friendlyTypeName 294 if 'struct' in nodeTypeName and not isPointer: 295 fieldElementType = 'struct' 296 elif 'union' in nodeTypeName: 297 fieldElementType = 'union' 298 else: 299 fieldElementType = 'base' 300 301 if isEnum: 302 enum_type_name = inspect.getmro(fieldType)[0].__name__ 303 xmlTree.addEnum(enum_type_name, sorted(fieldType.members.items())) 304 else: 305 enum_type_name = None 306 307 # write the message entry in all cases. 308 fieldEntry = xmlTree.addField(current_message_fields, field[0] + arrayItemNamePostfix, nodeTypeName, fieldElementType, elementSize, length, enum_type_name) 309 310 # if it's a structure add to structure dict etc, if not it's a base type and go to the next one 311 if fieldElementType == 'struct': 312 if not xmlTree.structureInDictionary(nodeTypeName): 313 fields_SubEntry = xmlTree.addStructure(nodeTypeName, fieldElementType, elementSize) 314 buildTree(xmlTree, fields_SubEntry, fieldType) 315 else: 316 if not isBaseType and not isPointer: 317 buildTree(xmlTree, fieldEntry, fieldType) 318 319 320def xml_gen_main(outputBasepath, source, cfgfilename, core, image, rulesfilename): 321 messages = parse_msgdefs.parse_preprocessed_headers(source, core) 322 # Load the configuration file 323 with open(cfgfilename) as cfgfile: 324 if cfgfile == None: 325 raise SystemExit("Error: Could not find configuration file %s." % cfgfilename) 326 327 # Load the core 328 if core not in ['acore', 'protocol_core', 'security_core']: 329 raise SystemExit("Error: Invalid core %s." % core) 330 331 if rulesfilename is not None: 332 with open(rulesfilename) as rulesfile: 333 if rulesfile == None: 334 raise SystemExit("Error: Could not find rules file %s." %rulesfilename) 335 336 # List of filenames and regular expressions 337 if not os.path.isdir(outputBasepath): 338 # If given an output filename rather than base directory, 339 # strip off the filename and keep the path 340 outputBasepath = os.path.dirname(outputBasepath) 341 342 outputs = [] 343 344 # Parse the pairs of filenames and regular expressions 345 for line in cfgfile.readlines(): 346 if not line.startswith("#"): # Exclude comments 347 if line.strip() != "": # Exclude empty lines 348 # Format is: filename , regexp 349 distribution, filename, regexp = line.split(",", 2) 350 # Remove regexp whitespace 351 regexp = regexp.strip() 352 # Remove whitespace and add base path 353 filename = filename.strip() 354 filename = os.path.join(outputBasepath, filename) 355 outputs.append((distribution, filename, regexp)) 356 357 firmware_version = getFirmwareVersion() 358 359 for distribution, outfilename, regexp in outputs: 360 #with messageTree(distribution, firmware_version, core) as xmlTree: 361 xmlTree = None 362 xmlTree = messageTree(distribution, firmware_version, core, image) 363 364 # And a list of message IDs already used 365 # (These should ideally be stored in a tree for scalable search speed, but the number of messages is small so the wasted time isn't a problem) 366 messageIdList = [] 367 368 for messageEnumName, structname, messageId, struct in messages: 369 # If not a stack xml then convert the IDs to the log IDs 370 # 371 # if distribution != "stack": 372 # messageId = getIdForMessage(messageId, core) 373 fieldsEntry = xmlTree.addMessageHeader(messageEnumName, messageId, core, structname) 374 buildTree(xmlTree, fieldsEntry, struct) 375 376 xmlTree.outputMessageTree(regexp, rulesfilename, outfilename) 377 del xmlTree 378 379 380if len(sys.argv) < 6: 381 raise SystemExit("Usage: python MessageXmlGen.py <ctypes_library> <output_dir> <configuration_file> <corename> <imagename> [<rules_file>]") 382 383source = sys.argv[1] 384outputBasepath = sys.argv[2] 385cfgfilename = sys.argv[3] 386corename = sys.argv[4] 387imagename = sys.argv[5] 388 389# Optional Rules File 390if len(sys.argv) == 7: 391 rulesfilename = sys.argv[6] 392else: 393 rulesfilename = None 394 395# 删除.i文件中的#pragma预编译指令 396with open(source, "rb+") as f: 397 write_lines = [] 398 for line in f.readlines(): 399 if not re.match("#pragma", line.decode("utf-8", errors="replace").strip()): 400 write_lines.append(line) 401 f.seek(0, 0) 402 f.truncate() 403 404 for line in write_lines: 405 f.write(line) 406 407xml_gen_main(outputBasepath, source, cfgfilename, corename, imagename, rulesfilename) 408