1from __future__ import print_function 2import re, sys, os, time, glob, errno, tempfile, binascii, subprocess, shutil 3from lxml import etree 4from optparse import OptionParser 5import textwrap 6import string 7 8VERSION = '0.1' 9__all__ = ['DoxyGen2RST'] 10LINE_BREAKER = "\n" 11MAX_COLUMN = 80 12 13def is_valid_uuid(uuid_string): 14 uuid4hex = re.compile('[0-9a-f]{32}\Z', re.I) 15 return uuid4hex.match(uuid_string) != None 16 17def get_page(refid): 18 fields = refid.split("_") 19 if(is_valid_uuid(fields[-1][-32:])): 20 return ["_".join(fields[0:-1]), fields[-1]] 21 return [refid, None] 22 23def mkdir_p(path): 24 try: 25 os.makedirs(path) 26 except OSError as exc: # Python >2.5 27 if exc.errno == errno.EEXIST and os.path.isdir(path): 28 pass 29 else: 30 raise 31 32def _glob(path, *exts): 33 path = os.path.join(path, "*") if os.path.isdir(path) else path + "*" 34 return [f for files in [glob.glob(path + ext) for ext in exts] for f in files] 35 36class DoxyGen2RST(object): 37 """ 38 Customize the Doxygen XML output into RST format, then it can 39 be translated into all formats with the unified user interface. 40 The Doxygen output itself is too verbose and not hard to be 41 organized for a good documentation. 42 """ 43 44 def __init__(self, 45 src, 46 dst, 47 missing_filename = "missing.rst", 48 is_github = False, 49 enable_uml = True, 50 github_ext = ""): 51 self.doxy_output_dir = os.path.join(src, "_doxygen", "xml") 52 self.output_dir = dst 53 self.rst_dir = src 54 self.enable_uml = enable_uml 55 mkdir_p(dst) 56 self.is_github = is_github 57 if(is_github): 58 self.page_ext = github_ext 59 self.anchor_prefix = "wiki-" 60 else: 61 self.anchor_prefix = "" 62 self.page_ext = ".html" 63 self.filter = ["*.rst", "*.rest"] 64 self.re_doxy = "<doxygen2rst\s(\S*)=(\S*)>(.*?)</doxygen2rst>" 65 self.index_root = etree.parse(os.path.join(self.doxy_output_dir, "index.xml")).getroot() 66 self.references = {} 67 self.missed_types_structs = {} 68 self.name_refid_map = {} 69 self.build_references() 70 self.page_references = {} 71 self.missing_filename = missing_filename 72 self.temp_uml_path = os.path.join(tempfile.gettempdir(), "uml_" + binascii.b2a_hex(os.urandom(15))) 73 if os.path.exists(self.temp_uml_path): 74 shutil.rmtree(self.temp_uml_path) 75 os.mkdir(self.temp_uml_path) 76 77 def _find_ref_id(self, kind, name): 78 #print("_find_ref_id, %s - %s" %(kind, name)) 79 if(kind == "function"): 80 for comp in self.index_root.iter("member"): 81 if(comp.attrib["kind"].lower() == kind.lower() and 82 comp.findtext("name").lower() == name.lower()): 83 return (comp.attrib["refid"]) 84 pass 85 else: 86 for comp in self.index_root.iter("compound"): 87 if(comp.attrib["kind"].lower() == kind.lower() and 88 comp.findtext("name").lower() == name.lower()): 89 return comp.attrib["refid"] 90 return None 91 92 def strip_title_ref(self, text): 93 table = string.maketrans("","") 94 retstr = text.translate(table, string.punctuation) 95 words = retstr.split() 96 retstr = "-".join(words) 97 return retstr.lower() 98 99 def build_references(self): 100 for file in _glob(self.rst_dir, *self.filter): 101 filename = os.path.basename(file) 102 fin = open(file,'r') 103 content = fin.read() 104 it = re.finditer(self.re_doxy, content, re.DOTALL) 105 for m in it: 106 ref_id = self._find_ref_id(m.groups()[0], m.groups()[1]) 107 if(ref_id is None): 108 #print("Reference is NOT found for: %s=%s" % (m.groups()[0], m.groups()[1])) 109 continue 110 page_name = os.path.splitext(filename)[0] 111 title_ref = self.strip_title_ref(m.groups()[2]) 112 self.references[ref_id] = [m.groups()[0], m.groups()[1], page_name, filename, title_ref] 113 self.name_refid_map[m.groups()[1]] = ref_id 114 fin.close() 115 #print(self.references) 116 117 def call_plantuml(self): 118 if(not self.enable_uml): 119 return 120 121 java_bin = os.path.join(os.environ['JAVA_HOME'], "bin", "java") 122 output_path = os.path.abspath(os.path.join(self.output_dir, "images")) 123 cmds = ["\"" + java_bin + "\"", "-jar", "plantuml.jar", self.temp_uml_path + "/", "-o", output_path] 124 print(" ".join(cmds)) 125 os.system(" ".join(cmds)) 126 shutil.rmtree(self.temp_uml_path) 127 128 def _build_uml(self, uml_name, content): 129 uml_path = os.path.join(self.temp_uml_path, uml_name + ".txt") 130 fuml = open(uml_path, "w+") 131 fuml.write("@startuml\n") 132 fuml.write(content) 133 fuml.write("\n@enduml\n") 134 fuml.close() 135 return ".. image:: images/" + uml_name + ".png" + LINE_BREAKER 136 137 def _build(self, m): 138 retstr = "" 139 if(m.groups()[0] == "uml"): 140 retstr = self._build_uml(m.groups()[1], m.groups()[2]) 141 elif(m.groups()[0] == "link"): 142 link = m.groups()[1] + self.page_ext 143 retstr = ("`%s <%s>`_" % (m.groups()[2], link)) 144 else: 145 if(m.groups()[0] != "function"): 146 retstr += self._build_title(m.groups()[2]) 147 retstr += self.convert_doxy(m.groups()[0], m.groups()[1]) 148 149 return retstr 150 151 def generate(self): 152 for file in _glob(self.rst_dir, *self.filter): 153 filename = os.path.basename(file) 154 fin = open(file,'r') 155 input_txt = fin.read() 156 fin.close() 157 158 output_txt = re.sub(self.re_doxy, self._build, input_txt, 0, re.DOTALL) 159 output_txt += self._build_page_ref_notes() 160 161 fout = open(os.path.join(self.output_dir, filename), 'w+') 162 fout.write(output_txt) 163 fout.close() 164 #print("%s --- %s" %( file, os.path.join(self.output_dir, filename))) 165 166 self._build_missed_types_and_structs() 167 self.call_plantuml() 168 169 def make_para_title(self, title, indent = 4): 170 retstr = LINE_BREAKER 171 if(title): 172 retstr += "".ljust(indent, " ") + "| **" + title + "**" + LINE_BREAKER 173 return retstr 174 175 def _build_title(self, title, flag = '=', ref = None): 176 retstr = LINE_BREAKER 177 if(ref): 178 retstr += ".. _ref-" + ref + ":" + LINE_BREAKER + LINE_BREAKER 179 retstr += title + LINE_BREAKER 180 retstr += "".ljust(20, flag) + LINE_BREAKER 181 retstr += LINE_BREAKER 182 return retstr 183 184 def _build_ref(self, node): 185 text = node.text.strip() 186 retstr = "" 187 target = '`' + text + '`' 188 retstr += target + "_ " 189 if target in self.page_references: 190 reflink = self.page_references[target] 191 print("Link already added: %s == %s" % (reflink[0], node.attrib["refid"])) 192 assert(reflink[0] == node.attrib["refid"]) 193 pass 194 else: 195 self.page_references[target] = (node.attrib["refid"], node.attrib["kindref"], text) 196 197 return retstr 198 199 def _build_code_block(self, node): 200 retstr = "::" + LINE_BREAKER + LINE_BREAKER 201 for codeline in node.iter("codeline"): 202 retstr += " " 203 for phrases in codeline.iter("highlight"): 204 if(phrases.text): 205 retstr += phrases.text.strip() 206 for child in phrases: 207 if(child.text): 208 retstr += child.text.strip() 209 if(child.tag == "sp"): 210 retstr += " " 211 if(child.tag == "ref" and child.text): 212 #escape the reference in the code block 213 retstr += "" # self._build_ref(child) 214 if(child.tail): 215 retstr += child.tail.strip() 216 retstr += LINE_BREAKER 217 return retstr 218 219 def _build_itemlist(self, node): 220 retstr = "" 221 for para in node: 222 if(para.tag != "para"): 223 continue 224 if(para.text): 225 retstr += para.text.strip() 226 for child in para: 227 if(child.tag == "ref" and child.text): 228 retstr += self._build_ref(child) 229 if(child.tail): 230 retstr += child.tail.strip() 231 232 return retstr 233 234 def _build_itemizedlist(self, node): 235 retstr = LINE_BREAKER 236 if(node == None): 237 return "" 238 for item in node: 239 if(item.tag != "listitem"): 240 continue 241 retstr += " - " + self._build_itemlist(item) 242 retstr += LINE_BREAKER 243 return retstr 244 245 def _build_verbatim(self, node): 246 retstr = LINE_BREAKER 247 if(node.text): 248 lines = node.text.splitlines() 249 print(lines[0]) 250 m = re.search("{plantuml}\s(\S*)", lines[0]) 251 if(m): 252 uml_name = "uml_" + m.groups()[0] 253 retstr += self._build_uml(uml_name, "\n".join(lines[1:])) 254 else: 255 retstr += "::" + LINE_BREAKER + LINE_BREAKER 256 retstr += node.text 257 258 return retstr 259 260 def _build_para(self, para): 261 retstr = "" 262 no_new_line = False 263 if(para.text): 264 retstr += textwrap.fill(para.text.strip(), MAX_COLUMN) + LINE_BREAKER + LINE_BREAKER 265 for child in para: 266 no_new_line = False 267 if(child.tag == "simplesect"): 268 for child_para in child: 269 if(child.attrib["kind"] == "return"): 270 return_str = self._build_para(child_para) 271 retstr += "".ljust(4, " ") + "| Return:" + LINE_BREAKER 272 for line in return_str.splitlines(): 273 retstr += "".ljust(4, " ") + "| " + line + LINE_BREAKER 274 elif(child_para.tag == "title" and child_para.text): 275 lf.make_para_title(child_para.text.strip(), 4) 276 elif(child_para.tag == "para"): #for @see 277 retstr += self._build_para(child_para) 278 elif(child_para.text): 279 retstr += "".ljust(4, " ") + "| " + child_para.text.strip() + LINE_BREAKER 280 if(child.tag == "preformatted"): 281 retstr += "::" + LINE_BREAKER + LINE_BREAKER 282 if(child.text): 283 for line in child.text.splitlines(): 284 retstr += " " + line + LINE_BREAKER 285 if(child.tag == "ref" and child.text): 286 retstr = retstr.rstrip('\n') 287 retstr += " " + self._build_ref(child) 288 no_new_line = True 289 if(child.tag == "programlisting"): 290 retstr += self._build_code_block(child) 291 if(child.tag == "itemizedlist"): 292 retstr += self._build_itemizedlist(child) 293 if(child.tag == "verbatim"): 294 retstr += self._build_verbatim(child) 295 if(not no_new_line): 296 retstr += LINE_BREAKER 297 if(child.tail): 298 retstr += textwrap.fill(child.tail.strip(), MAX_COLUMN) + LINE_BREAKER + LINE_BREAKER 299 return retstr 300 301 def get_text(self, node): 302 retstr = "" 303 if(node == None): 304 return "" 305 for para in node: 306 if(para.tag != "para"): 307 continue 308 retstr += self._build_para(para) 309 310 return retstr 311 312 def _find_text_ref(self, node): 313 retstr = "" 314 if(node.text): 315 retstr += node.text.strip() 316 for child in node: 317 if(child.tag == "ref"): 318 retstr += " " + self._build_ref(child) + " " 319 if(child.tail): 320 retstr += child.tail.strip() 321 return retstr 322 323 def _build_row_breaker(self, columns): 324 retstr = "+" 325 for column in columns: 326 retstr += "".ljust(column, "-") + "+" 327 return retstr + LINE_BREAKER 328 329 def _wrap_cell(self, text, length = 30): 330 newlines = [] 331 for line in text.splitlines(): 332 newlines.extend(textwrap.wrap(line, length)) 333 return newlines 334 335 def _build_row(self, row, columns): 336 retstr = "" 337 row_lines = [] 338 max_line = 0 339 for i in range(3): 340 row_lines.append(row[i].splitlines()) 341 if(max_line < len(row_lines[i])): 342 max_line = len(row_lines[i]) 343 344 for i in range(max_line): 345 for j in range(3): 346 retstr += "|" 347 if(len(row_lines[j]) > i): 348 retstr += row_lines[j][i] 349 retstr += "".ljust(columns[j] - len(row_lines[j][i]), " ") 350 else: 351 retstr += "".ljust(columns[j], " ") 352 retstr += "|" + LINE_BREAKER 353 return retstr 354 355 def _build_table(self, rows): 356 retstr = "" 357 columns = [0, 0, 0] 358 for row in rows: 359 for i in range(3): 360 for rowline in row[i].splitlines(): 361 if(columns[i] < len(rowline) + 2): 362 columns[i] = len(rowline) + 2 363 364 #columns[0] = 40 if(columns[0] > 40) else columns[0] 365 #columns[1] = 40 if(columns[1] > 40) else columns[1] 366 #columns[2] = MAX_COLUMN - columns[0] - columns[1] 367 368 retstr += self._build_row_breaker(columns) 369 for row in rows: 370 retstr += self._build_row(row, columns) 371 retstr += self._build_row_breaker(columns) 372 return retstr; 373 374 def build_param_list(self, params, paramdescs): 375 retstr = "" 376 param_descriptions = [] 377 for desc in paramdescs: 378 param_descriptions.append(desc) 379 380 rows = [] 381 rows.append(("Name", "Type", "Descritpion")) 382 for param in params: 383 declname = param.findtext("declname") 384 paramdesc = None 385 for desc in param_descriptions: 386 paramname = desc.findtext("parameternamelist/parametername") 387 if(paramname.lower() == declname.lower()): 388 paramdesc = desc.find("parameterdescription") 389 break 390 decltype = self._find_text_ref(param.find("type")) 391 rows.append((declname, decltype, self.get_text(paramdesc))) 392 393 if(len(rows) > 1): 394 retstr += self._build_table(rows) 395 return retstr 396 397 def _build_enum(self, member): 398 enum_id = member.attrib["id"] 399 file, tag = get_page(enum_id) 400 retstr = self._build_title(member.findtext("name"), ref = tag) 401 detail_node = self.get_desc_node(member) 402 if(detail_node is not None): 403 retstr += LINE_BREAKER 404 retstr += self.get_text(detail_node) 405 406 rows = [] 407 rows.append(("Name", "Initializer", "Descritpion")) 408 for enumvalue in member.iter("enumvalue"): 409 name = enumvalue.findtext("name") 410 initializer = enumvalue.findtext("initializer") 411 if(not initializer): 412 initializer = "" 413 desc = self.get_text(enumvalue.find("briefdescription")) 414 desc += self.get_text(enumvalue.find("detaileddescription")) 415 if(not desc): 416 desc = "" 417 rows.append((name, initializer, desc)) 418 419 if(len(rows) > 1): 420 retstr += self._build_table(rows) 421 return retstr 422 423 424 def _build_struct(self, node): 425 retstr = "" 426 detail_node = self.get_desc_node(node) 427 if(detail_node is not None): 428 retstr += self.get_text(detail_node) + LINE_BREAKER 429 rows = [] 430 rows.append(("Name", "Type", "Descritpion")) 431 for member in node.iter("memberdef"): 432 if(member.attrib["kind"] == "variable"): 433 name = member.findtext("name") 434 type = self._find_text_ref(member.find("type")) 435 desc = self.get_text(member.find("briefdescription")) 436 desc += self.get_text(member.find("detaileddescription")) 437 desc += self.get_text(member.find("inbodydescription")) 438 if(not desc): 439 desc = "" 440 rows.append((name, type, desc)) 441 442 if(len(rows) > 1): 443 retstr += self._build_table(rows) 444 return retstr 445 446 def _build_class(self, node): 447 retstr = "" 448 449 for member in node.iter("memberdef"): 450 if(member.attrib["kind"] == "function"): 451 retstr += self.build_function(member) 452 return retstr 453 454 def get_desc_node(self, member): 455 detail_node = member.find("detaileddescription") 456 brief_node = member.find("briefdescription") 457 detail_txt = "" 458 if(detail_node == None and brief_node == None): 459 return None 460 461 if(detail_node is not None): 462 detail_txt = detail_node.findtext("para") 463 464 if(not detail_txt and brief_node != None): 465 detail_txt = brief_node.findtext("para") 466 detail_node = brief_node 467 468 return detail_node 469 470 def build_function(self, member): 471 retstr = "" 472 473 desc_node = self.get_desc_node(member) 474 if(desc_node is None): 475 return "" 476 detail_txt = desc_node.findtext("para") 477 if(not detail_txt or detail_txt.strip() == "{ignore}"): 478 return "" 479 480 func_id = member.attrib["id"] 481 page_id, ref_id = get_page(func_id) 482 retstr += self._build_title(member.findtext("name"), '-', ref = ref_id) 483 retstr += self.get_text(desc_node) 484 retstr += LINE_BREAKER 485 detail_node = member.find("detaileddescription") 486 if(desc_node != detail_node): 487 retstr += self.get_text(detail_node) 488 retstr += self.build_param_list(member.iter("param"), detail_node.iter("parameteritem")) 489 return retstr 490 491 def _build_missed_types_and_structs(self): 492 fout = open(os.path.join(self.output_dir, self.missing_filename), 'w+') 493 fout.write(".. contents:: " + LINE_BREAKER) 494 fout.write(" :local:" + LINE_BREAKER) 495 fout.write(" :depth: 2" + LINE_BREAKER + LINE_BREAKER) 496 497 footnote = "" 498 while (len(self.missed_types_structs) > 0): 499 for key, value in self.missed_types_structs.iteritems(): 500 fout.write(self.covert_item(value[0], key, value[1])) 501 #print(value) 502 self.missed_types_structs = {} 503 footnote += self._build_page_ref_notes() 504 505 fout.write(footnote) 506 507 fout.close() 508 509 def _build_page_ref_notes(self): 510 retstr = LINE_BREAKER 511 #TODO 512 for key, value in self.page_references.iteritems(): 513 page, tag = get_page(value[0]) 514 m = re.search("_8h_", page) 515 if(m): 516 continue; 517 518 rstname = None 519 anchor = value[2].lower() 520 if not page in self.references: 521 self.missed_types_structs[value[0]] = (page, tag) 522 rstname = os.path.splitext(self.missing_filename)[0] 523 else: 524 rstname = self.references[page][2] 525 anchor = self.references[page][4] 526 #if(tag and not self.is_github): 527 # anchor = self.anchor_prefix + "ref-" + tag 528 retstr += ".. _" + key + ": " + rstname + self.page_ext + "#" + anchor 529 retstr += LINE_BREAKER + LINE_BREAKER 530 self.page_references = {} 531 return retstr 532 533 def _build_item_by_id(self, node, id): 534 retstr = "" 535 for member in node.iter("memberdef"): 536 if(member.attrib["id"] != id): 537 continue 538 if(member.attrib["kind"] == "enum"): 539 retstr += self._build_enum(member) 540 return retstr 541 542 def covert_item(self, compound, id, tag): 543 xml_path = os.path.join(self.doxy_output_dir, "%s.xml" % compound) 544 print("covert_item: id=%s, name=%s" % (id, xml_path)) 545 obj_root = etree.parse(xml_path).getroot() 546 retstr = "" 547 compound = obj_root.find("compounddef") 548 compound_kind = compound.attrib["kind"] 549 if(not tag): 550 retstr += self._build_title(compound.findtext("compoundname")) 551 if(compound_kind == "class"): 552 retstr += self._build_class(compound) 553 elif(compound_kind == "struct"): 554 retstr += self._build_struct(compound) 555 else: 556 retstr += self._build_item_by_id(compound, id) 557 558 return retstr 559 560 def _build_page(self, compound): 561 retstr = "" 562 retstr += self.get_text(compound.find("detaileddescription")) 563 return retstr 564 565 def _build_file(self, compound, type, ref_id, name): 566 retstr = "" 567 for member in compound.iter("memberdef"): 568 if(member.attrib["kind"] == "function" and member.attrib["id"] == ref_id): 569 retstr += self.build_function(member) 570 return retstr 571 572 def convert_doxy(self, type, name): 573 #print(name) 574 file = ref_id = self.name_refid_map[name] 575 dst_kind = type 576 if(type == "function"): 577 file, tag = get_page(ref_id) 578 dst_kind = "file" 579 xml_path = os.path.join(self.doxy_output_dir, "%s.xml" % file) 580 print("convert_doxy: type=%s, name=%s" % (type, xml_path)) 581 obj_root = etree.parse(xml_path).getroot() 582 compound = obj_root.find("compounddef") 583 compound_kind = compound.attrib["kind"] 584 assert(dst_kind == compound_kind) 585 retstr = "" 586 if(compound_kind == "class"): 587 retstr += self._build_class(compound) 588 elif(compound_kind == "struct"): 589 retstr += self._build_struct(compound) 590 elif(compound_kind == "page"): 591 retstr += self._build_page(compound) 592 elif(compound_kind == "group"): 593 retstr += self._build_page(compound) 594 elif(compound_kind == "file"): 595 retstr += self._build_file(compound, type, ref_id, name) 596 return retstr 597 598 599if __name__ == '__main__': 600 import argparse 601 parser = argparse.ArgumentParser() 602 parser.add_argument("-g", "--github", action="store_true", help="Render the link in format of github wiki.") 603 parser.add_argument("-e", "--ext", default="", help="extension for github wiki") 604 parser.add_argument("-i", "--input", default="doxygen", help="Input file path of doxygen output and source rst file.") 605 parser.add_argument("-o", "--output", default="wikipage", help="Output converted restructured text files to path.") 606 parser.add_argument("-s", "--struct", default="TypesAndStructures.rest", help="Output of auto generated enum and structures.") 607 parser.add_argument("-u", "--uml", action="store_true", help="Enable UML, you need to download plantuml.jar from Plantuml and put it to here. http://plantuml.sourceforge.net/") 608 609 args = parser.parse_args() 610 ext = "" 611 if(len(args.ext) > 0): 612 ext = ("." + args.ext) 613 agent = DoxyGen2RST(args.input, 614 args.output, 615 args.struct, 616 is_github = True, 617 enable_uml = args.uml, 618 github_ext = ext) 619 agent.generate() 620