1// doc generates HTML files from the comments in header files. 2// 3// doc expects to be given the path to a JSON file via the --config option. 4// From that JSON (which is defined by the Config struct) it reads a list of 5// header file locations and generates HTML files for each in the current 6// directory. 7 8package main 9 10import ( 11 "bufio" 12 "encoding/json" 13 "errors" 14 "flag" 15 "fmt" 16 "html/template" 17 "io/ioutil" 18 "os" 19 "path/filepath" 20 "strings" 21) 22 23// Config describes the structure of the config JSON file. 24type Config struct { 25 // BaseDirectory is a path to which other paths in the file are 26 // relative. 27 BaseDirectory string 28 Sections []ConfigSection 29} 30 31type ConfigSection struct { 32 Name string 33 // Headers is a list of paths to header files. 34 Headers []string 35} 36 37// HeaderFile is the internal representation of a header file. 38type HeaderFile struct { 39 // Name is the basename of the header file (e.g. "ex_data.html"). 40 Name string 41 // Preamble contains a comment for the file as a whole. Each string 42 // is a separate paragraph. 43 Preamble []string 44 Sections []HeaderSection 45 // AllDecls maps all decls to their URL fragments. 46 AllDecls map[string]string 47} 48 49type HeaderSection struct { 50 // Preamble contains a comment for a group of functions. 51 Preamble []string 52 Decls []HeaderDecl 53 // Anchor, if non-empty, is the URL fragment to use in anchor tags. 54 Anchor string 55 // IsPrivate is true if the section contains private functions (as 56 // indicated by its name). 57 IsPrivate bool 58} 59 60type HeaderDecl struct { 61 // Comment contains a comment for a specific function. Each string is a 62 // paragraph. Some paragraph may contain \n runes to indicate that they 63 // are preformatted. 64 Comment []string 65 // Name contains the name of the function, if it could be extracted. 66 Name string 67 // Decl contains the preformatted C declaration itself. 68 Decl string 69 // Anchor, if non-empty, is the URL fragment to use in anchor tags. 70 Anchor string 71} 72 73const ( 74 cppGuard = "#if defined(__cplusplus)" 75 commentStart = "/* " 76 commentEnd = " */" 77) 78 79func extractComment(lines []string, lineNo int) (comment []string, rest []string, restLineNo int, err error) { 80 if len(lines) == 0 { 81 return nil, lines, lineNo, nil 82 } 83 84 restLineNo = lineNo 85 rest = lines 86 87 if !strings.HasPrefix(rest[0], commentStart) { 88 panic("extractComment called on non-comment") 89 } 90 commentParagraph := rest[0][len(commentStart):] 91 rest = rest[1:] 92 restLineNo++ 93 94 for len(rest) > 0 { 95 i := strings.Index(commentParagraph, commentEnd) 96 if i >= 0 { 97 if i != len(commentParagraph)-len(commentEnd) { 98 err = fmt.Errorf("garbage after comment end on line %d", restLineNo) 99 return 100 } 101 commentParagraph = commentParagraph[:i] 102 if len(commentParagraph) > 0 { 103 comment = append(comment, commentParagraph) 104 } 105 return 106 } 107 108 line := rest[0] 109 if !strings.HasPrefix(line, " *") { 110 err = fmt.Errorf("comment doesn't start with block prefix on line %d: %s", restLineNo, line) 111 return 112 } 113 if len(line) == 2 || line[2] != '/' { 114 line = line[2:] 115 } 116 if strings.HasPrefix(line, " ") { 117 /* Identing the lines of a paragraph marks them as 118 * preformatted. */ 119 if len(commentParagraph) > 0 { 120 commentParagraph += "\n" 121 } 122 line = line[3:] 123 } 124 if len(line) > 0 { 125 commentParagraph = commentParagraph + line 126 if len(commentParagraph) > 0 && commentParagraph[0] == ' ' { 127 commentParagraph = commentParagraph[1:] 128 } 129 } else { 130 comment = append(comment, commentParagraph) 131 commentParagraph = "" 132 } 133 rest = rest[1:] 134 restLineNo++ 135 } 136 137 err = errors.New("hit EOF in comment") 138 return 139} 140 141func extractDecl(lines []string, lineNo int) (decl string, rest []string, restLineNo int, err error) { 142 if len(lines) == 0 || len(lines[0]) == 0 { 143 return "", lines, lineNo, nil 144 } 145 146 rest = lines 147 restLineNo = lineNo 148 149 var stack []rune 150 for len(rest) > 0 { 151 line := rest[0] 152 for _, c := range line { 153 switch c { 154 case '(', '{', '[': 155 stack = append(stack, c) 156 case ')', '}', ']': 157 if len(stack) == 0 { 158 err = fmt.Errorf("unexpected %c on line %d", c, restLineNo) 159 return 160 } 161 var expected rune 162 switch c { 163 case ')': 164 expected = '(' 165 case '}': 166 expected = '{' 167 case ']': 168 expected = '[' 169 default: 170 panic("internal error") 171 } 172 if last := stack[len(stack)-1]; last != expected { 173 err = fmt.Errorf("found %c when expecting %c on line %d", c, last, restLineNo) 174 return 175 } 176 stack = stack[:len(stack)-1] 177 } 178 } 179 if len(decl) > 0 { 180 decl += "\n" 181 } 182 decl += line 183 rest = rest[1:] 184 restLineNo++ 185 186 if len(stack) == 0 && (len(decl) == 0 || decl[len(decl)-1] != '\\') { 187 break 188 } 189 } 190 191 return 192} 193 194func skipLine(s string) string { 195 i := strings.Index(s, "\n") 196 if i > 0 { 197 return s[i:] 198 } 199 return "" 200} 201 202func getNameFromDecl(decl string) (string, bool) { 203 for strings.HasPrefix(decl, "#if") || strings.HasPrefix(decl, "#elif") { 204 decl = skipLine(decl) 205 } 206 207 if strings.HasPrefix(decl, "typedef ") { 208 return "", false 209 } 210 211 for _, prefix := range []string{"struct ", "enum ", "#define "} { 212 if !strings.HasPrefix(decl, prefix) { 213 continue 214 } 215 216 decl = strings.TrimPrefix(decl, prefix) 217 218 for len(decl) > 0 && decl[0] == ' ' { 219 decl = decl[1:] 220 } 221 222 // struct and enum types can be the return type of a 223 // function. 224 if prefix[0] != '#' && strings.Index(decl, "{") == -1 { 225 break 226 } 227 228 i := strings.IndexAny(decl, "( ") 229 if i < 0 { 230 return "", false 231 } 232 return decl[:i], true 233 } 234 decl = strings.TrimPrefix(decl, "OPENSSL_EXPORT ") 235 decl = strings.TrimPrefix(decl, "STACK_OF(") 236 decl = strings.TrimPrefix(decl, "LHASH_OF(") 237 i := strings.Index(decl, "(") 238 if i < 0 { 239 return "", false 240 } 241 j := strings.LastIndex(decl[:i], " ") 242 if j < 0 { 243 return "", false 244 } 245 for j+1 < len(decl) && decl[j+1] == '*' { 246 j++ 247 } 248 return decl[j+1 : i], true 249} 250 251func sanitizeAnchor(name string) string { 252 return strings.Replace(name, " ", "-", -1) 253} 254 255func isPrivateSection(name string) bool { 256 return strings.HasPrefix(name, "Private functions") || strings.HasPrefix(name, "Private structures") || strings.Contains(name, "(hidden)") 257} 258 259func (config *Config) parseHeader(path string) (*HeaderFile, error) { 260 headerPath := filepath.Join(config.BaseDirectory, path) 261 262 headerFile, err := os.Open(headerPath) 263 if err != nil { 264 return nil, err 265 } 266 defer headerFile.Close() 267 268 scanner := bufio.NewScanner(headerFile) 269 var lines, oldLines []string 270 for scanner.Scan() { 271 lines = append(lines, scanner.Text()) 272 } 273 if err := scanner.Err(); err != nil { 274 return nil, err 275 } 276 277 lineNo := 1 278 found := false 279 for i, line := range lines { 280 if line == cppGuard { 281 lines = lines[i+1:] 282 lineNo += i + 1 283 found = true 284 break 285 } 286 } 287 288 if !found { 289 return nil, errors.New("no C++ guard found") 290 } 291 292 if len(lines) == 0 || lines[0] != "extern \"C\" {" { 293 return nil, errors.New("no extern \"C\" found after C++ guard") 294 } 295 lineNo += 2 296 lines = lines[2:] 297 298 header := &HeaderFile{ 299 Name: filepath.Base(path), 300 AllDecls: make(map[string]string), 301 } 302 303 for i, line := range lines { 304 if len(line) > 0 { 305 lines = lines[i:] 306 lineNo += i 307 break 308 } 309 } 310 311 oldLines = lines 312 if len(lines) > 0 && strings.HasPrefix(lines[0], commentStart) { 313 comment, rest, restLineNo, err := extractComment(lines, lineNo) 314 if err != nil { 315 return nil, err 316 } 317 318 if len(rest) > 0 && len(rest[0]) == 0 { 319 if len(rest) < 2 || len(rest[1]) != 0 { 320 return nil, errors.New("preamble comment should be followed by two blank lines") 321 } 322 header.Preamble = comment 323 lineNo = restLineNo + 2 324 lines = rest[2:] 325 } else { 326 lines = oldLines 327 } 328 } 329 330 allAnchors := make(map[string]struct{}) 331 332 for { 333 // Start of a section. 334 if len(lines) == 0 { 335 return nil, errors.New("unexpected end of file") 336 } 337 line := lines[0] 338 if line == cppGuard { 339 break 340 } 341 342 if len(line) == 0 { 343 return nil, fmt.Errorf("blank line at start of section on line %d", lineNo) 344 } 345 346 var section HeaderSection 347 348 if strings.HasPrefix(line, commentStart) { 349 comment, rest, restLineNo, err := extractComment(lines, lineNo) 350 if err != nil { 351 return nil, err 352 } 353 if len(rest) > 0 && len(rest[0]) == 0 { 354 anchor := sanitizeAnchor(firstSentence(comment)) 355 if len(anchor) > 0 { 356 if _, ok := allAnchors[anchor]; ok { 357 return nil, fmt.Errorf("duplicate anchor: %s", anchor) 358 } 359 allAnchors[anchor] = struct{}{} 360 } 361 362 section.Preamble = comment 363 section.IsPrivate = len(comment) > 0 && isPrivateSection(comment[0]) 364 section.Anchor = anchor 365 lines = rest[1:] 366 lineNo = restLineNo + 1 367 } 368 } 369 370 for len(lines) > 0 { 371 line := lines[0] 372 if len(line) == 0 { 373 lines = lines[1:] 374 lineNo++ 375 break 376 } 377 if line == cppGuard { 378 return nil, errors.New("hit ending C++ guard while in section") 379 } 380 381 var comment []string 382 var decl string 383 if strings.HasPrefix(line, commentStart) { 384 comment, lines, lineNo, err = extractComment(lines, lineNo) 385 if err != nil { 386 return nil, err 387 } 388 } 389 if len(lines) == 0 { 390 return nil, errors.New("expected decl at EOF") 391 } 392 declLineNo := lineNo 393 decl, lines, lineNo, err = extractDecl(lines, lineNo) 394 if err != nil { 395 return nil, err 396 } 397 name, ok := getNameFromDecl(decl) 398 if !ok { 399 name = "" 400 } 401 if last := len(section.Decls) - 1; len(name) == 0 && len(comment) == 0 && last >= 0 { 402 section.Decls[last].Decl += "\n" + decl 403 } else { 404 // As a matter of style, comments should start 405 // with the name of the thing that they are 406 // commenting on. We make an exception here for 407 // #defines (because we often have blocks of 408 // them) and collective comments, which are 409 // detected by starting with “The” or “These”. 410 if len(comment) > 0 && 411 !strings.HasPrefix(comment[0], name) && 412 !strings.HasPrefix(comment[0], "A "+name) && 413 !strings.HasPrefix(comment[0], "An "+name) && 414 !strings.HasPrefix(decl, "#define ") && 415 !strings.HasPrefix(comment[0], "The ") && 416 !strings.HasPrefix(comment[0], "These ") { 417 return nil, fmt.Errorf("Comment for %q doesn't seem to match line %s:%d\n", name, path, declLineNo) 418 } 419 anchor := sanitizeAnchor(name) 420 // TODO(davidben): Enforce uniqueness. This is 421 // skipped because #ifdefs currently result in 422 // duplicate table-of-contents entries. 423 allAnchors[anchor] = struct{}{} 424 425 header.AllDecls[name] = anchor 426 427 section.Decls = append(section.Decls, HeaderDecl{ 428 Comment: comment, 429 Name: name, 430 Decl: decl, 431 Anchor: anchor, 432 }) 433 } 434 435 if len(lines) > 0 && len(lines[0]) == 0 { 436 lines = lines[1:] 437 lineNo++ 438 } 439 } 440 441 header.Sections = append(header.Sections, section) 442 } 443 444 return header, nil 445} 446 447func firstSentence(paragraphs []string) string { 448 if len(paragraphs) == 0 { 449 return "" 450 } 451 s := paragraphs[0] 452 i := strings.Index(s, ". ") 453 if i >= 0 { 454 return s[:i] 455 } 456 if lastIndex := len(s) - 1; s[lastIndex] == '.' { 457 return s[:lastIndex] 458 } 459 return s 460} 461 462func markupPipeWords(allDecls map[string]string, s string) template.HTML { 463 ret := "" 464 465 for { 466 i := strings.Index(s, "|") 467 if i == -1 { 468 ret += s 469 break 470 } 471 ret += s[:i] 472 s = s[i+1:] 473 474 i = strings.Index(s, "|") 475 j := strings.Index(s, " ") 476 if i > 0 && (j == -1 || j > i) { 477 ret += "<tt>" 478 anchor, isLink := allDecls[s[:i]] 479 if isLink { 480 ret += fmt.Sprintf("<a href=\"%s\">", template.HTMLEscapeString(anchor)) 481 } 482 ret += s[:i] 483 if isLink { 484 ret += "</a>" 485 } 486 ret += "</tt>" 487 s = s[i+1:] 488 } else { 489 ret += "|" 490 } 491 } 492 493 return template.HTML(ret) 494} 495 496func markupFirstWord(s template.HTML) template.HTML { 497 start := 0 498again: 499 end := strings.Index(string(s[start:]), " ") 500 if end > 0 { 501 end += start 502 w := strings.ToLower(string(s[start:end])) 503 // The first word was already marked up as an HTML tag. Don't 504 // mark it up further. 505 if strings.ContainsRune(w, '<') { 506 return s 507 } 508 if w == "a" || w == "an" { 509 start = end + 1 510 goto again 511 } 512 return s[:start] + "<span class=\"first-word\">" + s[start:end] + "</span>" + s[end:] 513 } 514 return s 515} 516 517func newlinesToBR(html template.HTML) template.HTML { 518 s := string(html) 519 if !strings.Contains(s, "\n") { 520 return html 521 } 522 s = strings.Replace(s, "\n", "<br>", -1) 523 s = strings.Replace(s, " ", " ", -1) 524 return template.HTML(s) 525} 526 527func generate(outPath string, config *Config) (map[string]string, error) { 528 allDecls := make(map[string]string) 529 530 headerTmpl := template.New("headerTmpl") 531 headerTmpl.Funcs(template.FuncMap{ 532 "firstSentence": firstSentence, 533 "markupPipeWords": func(s string) template.HTML { return markupPipeWords(allDecls, s) }, 534 "markupFirstWord": markupFirstWord, 535 "newlinesToBR": newlinesToBR, 536 }) 537 headerTmpl, err := headerTmpl.Parse(`<!DOCTYPE html> 538<html> 539 <head> 540 <title>BoringSSL - {{.Name}}</title> 541 <meta charset="utf-8"> 542 <link rel="stylesheet" type="text/css" href="doc.css"> 543 </head> 544 545 <body> 546 <div id="main"> 547 <div class="title"> 548 <h2>{{.Name}}</h2> 549 <a href="headers.html">All headers</a> 550 </div> 551 552 {{range .Preamble}}<p>{{. | html | markupPipeWords}}</p>{{end}} 553 554 <ol> 555 {{range .Sections}} 556 {{if not .IsPrivate}} 557 {{if .Anchor}}<li class="header"><a href="#{{.Anchor}}">{{.Preamble | firstSentence | html | markupPipeWords}}</a></li>{{end}} 558 {{range .Decls}} 559 {{if .Anchor}}<li><a href="#{{.Anchor}}"><tt>{{.Name}}</tt></a></li>{{end}} 560 {{end}} 561 {{end}} 562 {{end}} 563 </ol> 564 565 {{range .Sections}} 566 {{if not .IsPrivate}} 567 <div class="section" {{if .Anchor}}id="{{.Anchor}}"{{end}}> 568 {{if .Preamble}} 569 <div class="sectionpreamble"> 570 {{range .Preamble}}<p>{{. | html | markupPipeWords}}</p>{{end}} 571 </div> 572 {{end}} 573 574 {{range .Decls}} 575 <div class="decl" {{if .Anchor}}id="{{.Anchor}}"{{end}}> 576 {{range .Comment}} 577 <p>{{. | html | markupPipeWords | newlinesToBR | markupFirstWord}}</p> 578 {{end}} 579 <pre>{{.Decl}}</pre> 580 </div> 581 {{end}} 582 </div> 583 {{end}} 584 {{end}} 585 </div> 586 </body> 587</html>`) 588 if err != nil { 589 return nil, err 590 } 591 592 headerDescriptions := make(map[string]string) 593 var headers []*HeaderFile 594 595 for _, section := range config.Sections { 596 for _, headerPath := range section.Headers { 597 header, err := config.parseHeader(headerPath) 598 if err != nil { 599 return nil, errors.New("while parsing " + headerPath + ": " + err.Error()) 600 } 601 headerDescriptions[header.Name] = firstSentence(header.Preamble) 602 headers = append(headers, header) 603 604 for name, anchor := range header.AllDecls { 605 allDecls[name] = fmt.Sprintf("%s#%s", header.Name+".html", anchor) 606 } 607 } 608 } 609 610 for _, header := range headers { 611 filename := filepath.Join(outPath, header.Name+".html") 612 file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) 613 if err != nil { 614 panic(err) 615 } 616 defer file.Close() 617 if err := headerTmpl.Execute(file, header); err != nil { 618 return nil, err 619 } 620 } 621 622 return headerDescriptions, nil 623} 624 625func generateIndex(outPath string, config *Config, headerDescriptions map[string]string) error { 626 indexTmpl := template.New("indexTmpl") 627 indexTmpl.Funcs(template.FuncMap{ 628 "baseName": filepath.Base, 629 "headerDescription": func(header string) string { 630 return headerDescriptions[header] 631 }, 632 }) 633 indexTmpl, err := indexTmpl.Parse(`<!DOCTYPE html5> 634 635 <head> 636 <title>BoringSSL - Headers</title> 637 <meta charset="utf-8"> 638 <link rel="stylesheet" type="text/css" href="doc.css"> 639 </head> 640 641 <body> 642 <div id="main"> 643 <div class="title"> 644 <h2>BoringSSL Headers</h2> 645 </div> 646 <table> 647 {{range .Sections}} 648 <tr class="header"><td colspan="2">{{.Name}}</td></tr> 649 {{range .Headers}} 650 <tr><td><a href="{{. | baseName}}.html">{{. | baseName}}</a></td><td>{{. | baseName | headerDescription}}</td></tr> 651 {{end}} 652 {{end}} 653 </table> 654 </div> 655 </body> 656</html>`) 657 658 if err != nil { 659 return err 660 } 661 662 file, err := os.OpenFile(filepath.Join(outPath, "headers.html"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) 663 if err != nil { 664 panic(err) 665 } 666 defer file.Close() 667 668 if err := indexTmpl.Execute(file, config); err != nil { 669 return err 670 } 671 672 return nil 673} 674 675func copyFile(outPath string, inFilePath string) error { 676 bytes, err := ioutil.ReadFile(inFilePath) 677 if err != nil { 678 return err 679 } 680 return ioutil.WriteFile(filepath.Join(outPath, filepath.Base(inFilePath)), bytes, 0666) 681} 682 683func main() { 684 var ( 685 configFlag *string = flag.String("config", "doc.config", "Location of config file") 686 outputDir *string = flag.String("out", ".", "Path to the directory where the output will be written") 687 config Config 688 ) 689 690 flag.Parse() 691 692 if len(*configFlag) == 0 { 693 fmt.Printf("No config file given by --config\n") 694 os.Exit(1) 695 } 696 697 if len(*outputDir) == 0 { 698 fmt.Printf("No output directory given by --out\n") 699 os.Exit(1) 700 } 701 702 configBytes, err := ioutil.ReadFile(*configFlag) 703 if err != nil { 704 fmt.Printf("Failed to open config file: %s\n", err) 705 os.Exit(1) 706 } 707 708 if err := json.Unmarshal(configBytes, &config); err != nil { 709 fmt.Printf("Failed to parse config file: %s\n", err) 710 os.Exit(1) 711 } 712 713 headerDescriptions, err := generate(*outputDir, &config) 714 if err != nil { 715 fmt.Printf("Failed to generate output: %s\n", err) 716 os.Exit(1) 717 } 718 719 if err := generateIndex(*outputDir, &config, headerDescriptions); err != nil { 720 fmt.Printf("Failed to generate index: %s\n", err) 721 os.Exit(1) 722 } 723 724 if err := copyFile(*outputDir, "doc.css"); err != nil { 725 fmt.Printf("Failed to copy static file: %s\n", err) 726 os.Exit(1) 727 } 728} 729