• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// doc generates HTML files from the comments in header files.
2//
3// doc expects to be given the path to a JSON file via the --config option.
4// From that JSON (which is defined by the Config struct) it reads a list of
5// header file locations and generates HTML files for each in the current
6// directory.
7
8package main
9
10import (
11	"bufio"
12	"encoding/json"
13	"errors"
14	"flag"
15	"fmt"
16	"html/template"
17	"io/ioutil"
18	"os"
19	"path/filepath"
20	"regexp"
21	"strings"
22)
23
24// Config describes the structure of the config JSON file.
25type Config struct {
26	// BaseDirectory is a path to which other paths in the file are
27	// relative.
28	BaseDirectory string
29	Sections      []ConfigSection
30}
31
32type ConfigSection struct {
33	Name string
34	// Headers is a list of paths to header files.
35	Headers []string
36}
37
38// HeaderFile is the internal representation of a header file.
39type HeaderFile struct {
40	// Name is the basename of the header file (e.g. "ex_data.html").
41	Name string
42	// Preamble contains a comment for the file as a whole. Each string
43	// is a separate paragraph.
44	Preamble []string
45	Sections []HeaderSection
46	// AllDecls maps all decls to their URL fragments.
47	AllDecls map[string]string
48}
49
50type HeaderSection struct {
51	// Preamble contains a comment for a group of functions.
52	Preamble []string
53	Decls    []HeaderDecl
54	// Anchor, if non-empty, is the URL fragment to use in anchor tags.
55	Anchor string
56	// IsPrivate is true if the section contains private functions (as
57	// indicated by its name).
58	IsPrivate bool
59}
60
61type HeaderDecl struct {
62	// Comment contains a comment for a specific function. Each string is a
63	// paragraph. Some paragraph may contain \n runes to indicate that they
64	// are preformatted.
65	Comment []string
66	// Name contains the name of the function, if it could be extracted.
67	Name string
68	// Decl contains the preformatted C declaration itself.
69	Decl string
70	// Anchor, if non-empty, is the URL fragment to use in anchor tags.
71	Anchor string
72}
73
74const (
75	cppGuard     = "#if defined(__cplusplus)"
76	commentStart = "/* "
77	commentEnd   = " */"
78	lineComment  = "// "
79)
80
81func isComment(line string) bool {
82	return strings.HasPrefix(line, commentStart) || strings.HasPrefix(line, lineComment)
83}
84
85func commentSubject(line string) string {
86	if strings.HasPrefix(line, "A ") {
87		line = line[len("A "):]
88	} else if strings.HasPrefix(line, "An ") {
89		line = line[len("An "):]
90	}
91	idx := strings.IndexAny(line, " ,")
92	if idx < 0 {
93		return line
94	}
95	return line[:idx]
96}
97
98func extractComment(lines []string, lineNo int) (comment []string, rest []string, restLineNo int, err error) {
99	if len(lines) == 0 {
100		return nil, lines, lineNo, nil
101	}
102
103	restLineNo = lineNo
104	rest = lines
105
106	var isBlock bool
107	if strings.HasPrefix(rest[0], commentStart) {
108		isBlock = true
109	} else if !strings.HasPrefix(rest[0], lineComment) {
110		panic("extractComment called on non-comment")
111	}
112	commentParagraph := rest[0][len(commentStart):]
113	rest = rest[1:]
114	restLineNo++
115
116	for len(rest) > 0 {
117		if isBlock {
118			i := strings.Index(commentParagraph, commentEnd)
119			if i >= 0 {
120				if i != len(commentParagraph)-len(commentEnd) {
121					err = fmt.Errorf("garbage after comment end on line %d", restLineNo)
122					return
123				}
124				commentParagraph = commentParagraph[:i]
125				if len(commentParagraph) > 0 {
126					comment = append(comment, commentParagraph)
127				}
128				return
129			}
130		}
131
132		line := rest[0]
133		if isBlock {
134			if !strings.HasPrefix(line, " *") {
135				err = fmt.Errorf("comment doesn't start with block prefix on line %d: %s", restLineNo, line)
136				return
137			}
138		} else if !strings.HasPrefix(line, "//") {
139			if len(commentParagraph) > 0 {
140				comment = append(comment, commentParagraph)
141			}
142			return
143		}
144		if len(line) == 2 || !isBlock || line[2] != '/' {
145			line = line[2:]
146		}
147		if strings.HasPrefix(line, "   ") {
148			/* Identing the lines of a paragraph marks them as
149			* preformatted. */
150			if len(commentParagraph) > 0 {
151				commentParagraph += "\n"
152			}
153			line = line[3:]
154		}
155		if len(line) > 0 {
156			commentParagraph = commentParagraph + line
157			if len(commentParagraph) > 0 && commentParagraph[0] == ' ' {
158				commentParagraph = commentParagraph[1:]
159			}
160		} else {
161			comment = append(comment, commentParagraph)
162			commentParagraph = ""
163		}
164		rest = rest[1:]
165		restLineNo++
166	}
167
168	err = errors.New("hit EOF in comment")
169	return
170}
171
172func extractDecl(lines []string, lineNo int) (decl string, rest []string, restLineNo int, err error) {
173	if len(lines) == 0 || len(lines[0]) == 0 {
174		return "", lines, lineNo, nil
175	}
176
177	rest = lines
178	restLineNo = lineNo
179
180	var stack []rune
181	for len(rest) > 0 {
182		line := rest[0]
183		for _, c := range line {
184			switch c {
185			case '(', '{', '[':
186				stack = append(stack, c)
187			case ')', '}', ']':
188				if len(stack) == 0 {
189					err = fmt.Errorf("unexpected %c on line %d", c, restLineNo)
190					return
191				}
192				var expected rune
193				switch c {
194				case ')':
195					expected = '('
196				case '}':
197					expected = '{'
198				case ']':
199					expected = '['
200				default:
201					panic("internal error")
202				}
203				if last := stack[len(stack)-1]; last != expected {
204					err = fmt.Errorf("found %c when expecting %c on line %d", c, last, restLineNo)
205					return
206				}
207				stack = stack[:len(stack)-1]
208			}
209		}
210		if len(decl) > 0 {
211			decl += "\n"
212		}
213		decl += line
214		rest = rest[1:]
215		restLineNo++
216
217		if len(stack) == 0 && (len(decl) == 0 || decl[len(decl)-1] != '\\') {
218			break
219		}
220	}
221
222	return
223}
224
225func skipLine(s string) string {
226	i := strings.Index(s, "\n")
227	if i > 0 {
228		return s[i:]
229	}
230	return ""
231}
232
233var stackOfRegexp = regexp.MustCompile(`STACK_OF\(([^)]*)\)`)
234var lhashOfRegexp = regexp.MustCompile(`LHASH_OF\(([^)]*)\)`)
235
236func getNameFromDecl(decl string) (string, bool) {
237	for strings.HasPrefix(decl, "#if") || strings.HasPrefix(decl, "#elif") {
238		decl = skipLine(decl)
239	}
240
241	if strings.HasPrefix(decl, "typedef ") {
242		return "", false
243	}
244
245	for _, prefix := range []string{"struct ", "enum ", "#define "} {
246		if !strings.HasPrefix(decl, prefix) {
247			continue
248		}
249
250		decl = strings.TrimPrefix(decl, prefix)
251
252		for len(decl) > 0 && decl[0] == ' ' {
253			decl = decl[1:]
254		}
255
256		// struct and enum types can be the return type of a
257		// function.
258		if prefix[0] != '#' && strings.Index(decl, "{") == -1 {
259			break
260		}
261
262		i := strings.IndexAny(decl, "( ")
263		if i < 0 {
264			return "", false
265		}
266		return decl[:i], true
267	}
268	decl = strings.TrimPrefix(decl, "OPENSSL_EXPORT ")
269	decl = strings.TrimPrefix(decl, "const ")
270	decl = stackOfRegexp.ReplaceAllString(decl, "STACK_OF_$1")
271	decl = lhashOfRegexp.ReplaceAllString(decl, "LHASH_OF_$1")
272	i := strings.Index(decl, "(")
273	if i < 0 {
274		return "", false
275	}
276	j := strings.LastIndex(decl[:i], " ")
277	if j < 0 {
278		return "", false
279	}
280	for j+1 < len(decl) && decl[j+1] == '*' {
281		j++
282	}
283	return decl[j+1 : i], true
284}
285
286func sanitizeAnchor(name string) string {
287	return strings.Replace(name, " ", "-", -1)
288}
289
290func isPrivateSection(name string) bool {
291	return strings.HasPrefix(name, "Private functions") || strings.HasPrefix(name, "Private structures") || strings.Contains(name, "(hidden)")
292}
293
294func (config *Config) parseHeader(path string) (*HeaderFile, error) {
295	headerPath := filepath.Join(config.BaseDirectory, path)
296
297	headerFile, err := os.Open(headerPath)
298	if err != nil {
299		return nil, err
300	}
301	defer headerFile.Close()
302
303	scanner := bufio.NewScanner(headerFile)
304	var lines, oldLines []string
305	for scanner.Scan() {
306		lines = append(lines, scanner.Text())
307	}
308	if err := scanner.Err(); err != nil {
309		return nil, err
310	}
311
312	lineNo := 1
313	found := false
314	for i, line := range lines {
315		if line == cppGuard {
316			lines = lines[i+1:]
317			lineNo += i + 1
318			found = true
319			break
320		}
321	}
322
323	if !found {
324		return nil, errors.New("no C++ guard found")
325	}
326
327	if len(lines) == 0 || lines[0] != "extern \"C\" {" {
328		return nil, errors.New("no extern \"C\" found after C++ guard")
329	}
330	lineNo += 2
331	lines = lines[2:]
332
333	header := &HeaderFile{
334		Name:     filepath.Base(path),
335		AllDecls: make(map[string]string),
336	}
337
338	for i, line := range lines {
339		if len(line) > 0 {
340			lines = lines[i:]
341			lineNo += i
342			break
343		}
344	}
345
346	oldLines = lines
347	if len(lines) > 0 && isComment(lines[0]) {
348		comment, rest, restLineNo, err := extractComment(lines, lineNo)
349		if err != nil {
350			return nil, err
351		}
352
353		if len(rest) > 0 && len(rest[0]) == 0 {
354			if len(rest) < 2 || len(rest[1]) != 0 {
355				return nil, errors.New("preamble comment should be followed by two blank lines")
356			}
357			header.Preamble = comment
358			lineNo = restLineNo + 2
359			lines = rest[2:]
360		} else {
361			lines = oldLines
362		}
363	}
364
365	allAnchors := make(map[string]struct{})
366
367	for {
368		// Start of a section.
369		if len(lines) == 0 {
370			return nil, errors.New("unexpected end of file")
371		}
372		line := lines[0]
373		if line == cppGuard {
374			break
375		}
376
377		if len(line) == 0 {
378			return nil, fmt.Errorf("blank line at start of section on line %d", lineNo)
379		}
380
381		var section HeaderSection
382
383		if isComment(line) {
384			comment, rest, restLineNo, err := extractComment(lines, lineNo)
385			if err != nil {
386				return nil, err
387			}
388			if len(rest) > 0 && len(rest[0]) == 0 {
389				anchor := sanitizeAnchor(firstSentence(comment))
390				if len(anchor) > 0 {
391					if _, ok := allAnchors[anchor]; ok {
392						return nil, fmt.Errorf("duplicate anchor: %s", anchor)
393					}
394					allAnchors[anchor] = struct{}{}
395				}
396
397				section.Preamble = comment
398				section.IsPrivate = len(comment) > 0 && isPrivateSection(comment[0])
399				section.Anchor = anchor
400				lines = rest[1:]
401				lineNo = restLineNo + 1
402			}
403		}
404
405		for len(lines) > 0 {
406			line := lines[0]
407			if len(line) == 0 {
408				lines = lines[1:]
409				lineNo++
410				break
411			}
412			if line == cppGuard {
413				return nil, errors.New("hit ending C++ guard while in section")
414			}
415
416			var comment []string
417			var decl string
418			if isComment(line) {
419				comment, lines, lineNo, err = extractComment(lines, lineNo)
420				if err != nil {
421					return nil, err
422				}
423			}
424			if len(lines) == 0 {
425				return nil, errors.New("expected decl at EOF")
426			}
427			declLineNo := lineNo
428			decl, lines, lineNo, err = extractDecl(lines, lineNo)
429			if err != nil {
430				return nil, err
431			}
432			name, ok := getNameFromDecl(decl)
433			if !ok {
434				name = ""
435			}
436			if last := len(section.Decls) - 1; len(name) == 0 && len(comment) == 0 && last >= 0 {
437				section.Decls[last].Decl += "\n" + decl
438			} else {
439				// As a matter of style, comments should start
440				// with the name of the thing that they are
441				// commenting on. We make an exception here for
442				// collective comments, which are detected by
443				// starting with “The” or “These”.
444				if len(comment) > 0 &&
445					len(name) > 0 &&
446					!strings.HasPrefix(comment[0], "The ") &&
447					!strings.HasPrefix(comment[0], "These ") {
448					subject := commentSubject(comment[0])
449					ok := subject == name
450					if l := len(subject); l > 0 && subject[l-1] == '*' {
451						// Groups of names, notably #defines, are often
452						// denoted with a wildcard.
453						ok = strings.HasPrefix(name, subject[:l-1])
454					}
455					if !ok {
456						return nil, fmt.Errorf("comment for %q doesn't seem to match line %s:%d\n", name, path, declLineNo)
457					}
458				}
459				anchor := sanitizeAnchor(name)
460				// TODO(davidben): Enforce uniqueness. This is
461				// skipped because #ifdefs currently result in
462				// duplicate table-of-contents entries.
463				allAnchors[anchor] = struct{}{}
464
465				header.AllDecls[name] = anchor
466
467				section.Decls = append(section.Decls, HeaderDecl{
468					Comment: comment,
469					Name:    name,
470					Decl:    decl,
471					Anchor:  anchor,
472				})
473			}
474
475			if len(lines) > 0 && len(lines[0]) == 0 {
476				lines = lines[1:]
477				lineNo++
478			}
479		}
480
481		header.Sections = append(header.Sections, section)
482	}
483
484	return header, nil
485}
486
487func firstSentence(paragraphs []string) string {
488	if len(paragraphs) == 0 {
489		return ""
490	}
491	s := paragraphs[0]
492	i := strings.Index(s, ". ")
493	if i >= 0 {
494		return s[:i]
495	}
496	if lastIndex := len(s) - 1; s[lastIndex] == '.' {
497		return s[:lastIndex]
498	}
499	return s
500}
501
502// markupPipeWords converts |s| into an HTML string, safe to be included outside
503// a tag, while also marking up words surrounded by |.
504func markupPipeWords(allDecls map[string]string, s string) template.HTML {
505	// It is safe to look for '|' in the HTML-escaped version of |s|
506	// below. The escaped version cannot include '|' instead tags because
507	// there are no tags by construction.
508	s = template.HTMLEscapeString(s)
509	ret := ""
510
511	for {
512		i := strings.Index(s, "|")
513		if i == -1 {
514			ret += s
515			break
516		}
517		ret += s[:i]
518		s = s[i+1:]
519
520		i = strings.Index(s, "|")
521		j := strings.Index(s, " ")
522		if i > 0 && (j == -1 || j > i) {
523			ret += "<tt>"
524			anchor, isLink := allDecls[s[:i]]
525			if isLink {
526				ret += fmt.Sprintf("<a href=\"%s\">", template.HTMLEscapeString(anchor))
527			}
528			ret += s[:i]
529			if isLink {
530				ret += "</a>"
531			}
532			ret += "</tt>"
533			s = s[i+1:]
534		} else {
535			ret += "|"
536		}
537	}
538
539	return template.HTML(ret)
540}
541
542func markupFirstWord(s template.HTML) template.HTML {
543	start := 0
544again:
545	end := strings.Index(string(s[start:]), " ")
546	if end > 0 {
547		end += start
548		w := strings.ToLower(string(s[start:end]))
549		// The first word was already marked up as an HTML tag. Don't
550		// mark it up further.
551		if strings.ContainsRune(w, '<') {
552			return s
553		}
554		if w == "a" || w == "an" {
555			start = end + 1
556			goto again
557		}
558		return s[:start] + "<span class=\"first-word\">" + s[start:end] + "</span>" + s[end:]
559	}
560	return s
561}
562
563func newlinesToBR(html template.HTML) template.HTML {
564	s := string(html)
565	if !strings.Contains(s, "\n") {
566		return html
567	}
568	s = strings.Replace(s, "\n", "<br>", -1)
569	s = strings.Replace(s, " ", "&nbsp;", -1)
570	return template.HTML(s)
571}
572
573func generate(outPath string, config *Config) (map[string]string, error) {
574	allDecls := make(map[string]string)
575
576	headerTmpl := template.New("headerTmpl")
577	headerTmpl.Funcs(template.FuncMap{
578		"firstSentence":   firstSentence,
579		"markupPipeWords": func(s string) template.HTML { return markupPipeWords(allDecls, s) },
580		"markupFirstWord": markupFirstWord,
581		"newlinesToBR":    newlinesToBR,
582	})
583	headerTmpl, err := headerTmpl.Parse(`<!DOCTYPE html>
584<html>
585  <head>
586    <title>BoringSSL - {{.Name}}</title>
587    <meta charset="utf-8">
588    <link rel="stylesheet" type="text/css" href="doc.css">
589  </head>
590
591  <body>
592    <div id="main">
593    <div class="title">
594      <h2>{{.Name}}</h2>
595      <a href="headers.html">All headers</a>
596    </div>
597
598    {{range .Preamble}}<p>{{. | markupPipeWords}}</p>{{end}}
599
600    <ol>
601      {{range .Sections}}
602        {{if not .IsPrivate}}
603          {{if .Anchor}}<li class="header"><a href="#{{.Anchor}}">{{.Preamble | firstSentence | markupPipeWords}}</a></li>{{end}}
604          {{range .Decls}}
605            {{if .Anchor}}<li><a href="#{{.Anchor}}"><tt>{{.Name}}</tt></a></li>{{end}}
606          {{end}}
607        {{end}}
608      {{end}}
609    </ol>
610
611    {{range .Sections}}
612      {{if not .IsPrivate}}
613        <div class="section" {{if .Anchor}}id="{{.Anchor}}"{{end}}>
614        {{if .Preamble}}
615          <div class="sectionpreamble">
616          {{range .Preamble}}<p>{{. | markupPipeWords}}</p>{{end}}
617          </div>
618        {{end}}
619
620        {{range .Decls}}
621          <div class="decl" {{if .Anchor}}id="{{.Anchor}}"{{end}}>
622          {{range .Comment}}
623            <p>{{. | markupPipeWords | newlinesToBR | markupFirstWord}}</p>
624          {{end}}
625          <pre>{{.Decl}}</pre>
626          </div>
627        {{end}}
628        </div>
629      {{end}}
630    {{end}}
631    </div>
632  </body>
633</html>`)
634	if err != nil {
635		return nil, err
636	}
637
638	headerDescriptions := make(map[string]string)
639	var headers []*HeaderFile
640
641	for _, section := range config.Sections {
642		for _, headerPath := range section.Headers {
643			header, err := config.parseHeader(headerPath)
644			if err != nil {
645				return nil, errors.New("while parsing " + headerPath + ": " + err.Error())
646			}
647			headerDescriptions[header.Name] = firstSentence(header.Preamble)
648			headers = append(headers, header)
649
650			for name, anchor := range header.AllDecls {
651				allDecls[name] = fmt.Sprintf("%s#%s", header.Name+".html", anchor)
652			}
653		}
654	}
655
656	for _, header := range headers {
657		filename := filepath.Join(outPath, header.Name+".html")
658		file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
659		if err != nil {
660			panic(err)
661		}
662		defer file.Close()
663		if err := headerTmpl.Execute(file, header); err != nil {
664			return nil, err
665		}
666	}
667
668	return headerDescriptions, nil
669}
670
671func generateIndex(outPath string, config *Config, headerDescriptions map[string]string) error {
672	indexTmpl := template.New("indexTmpl")
673	indexTmpl.Funcs(template.FuncMap{
674		"baseName": filepath.Base,
675		"headerDescription": func(header string) string {
676			return headerDescriptions[header]
677		},
678	})
679	indexTmpl, err := indexTmpl.Parse(`<!DOCTYPE html5>
680
681  <head>
682    <title>BoringSSL - Headers</title>
683    <meta charset="utf-8">
684    <link rel="stylesheet" type="text/css" href="doc.css">
685  </head>
686
687  <body>
688    <div id="main">
689      <div class="title">
690        <h2>BoringSSL Headers</h2>
691      </div>
692      <table>
693        {{range .Sections}}
694	  <tr class="header"><td colspan="2">{{.Name}}</td></tr>
695	  {{range .Headers}}
696	    <tr><td><a href="{{. | baseName}}.html">{{. | baseName}}</a></td><td>{{. | baseName | headerDescription}}</td></tr>
697	  {{end}}
698	{{end}}
699      </table>
700    </div>
701  </body>
702</html>`)
703
704	if err != nil {
705		return err
706	}
707
708	file, err := os.OpenFile(filepath.Join(outPath, "headers.html"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
709	if err != nil {
710		panic(err)
711	}
712	defer file.Close()
713
714	if err := indexTmpl.Execute(file, config); err != nil {
715		return err
716	}
717
718	return nil
719}
720
721func copyFile(outPath string, inFilePath string) error {
722	bytes, err := ioutil.ReadFile(inFilePath)
723	if err != nil {
724		return err
725	}
726	return ioutil.WriteFile(filepath.Join(outPath, filepath.Base(inFilePath)), bytes, 0666)
727}
728
729func main() {
730	var (
731		configFlag *string = flag.String("config", "doc.config", "Location of config file")
732		outputDir  *string = flag.String("out", ".", "Path to the directory where the output will be written")
733		config     Config
734	)
735
736	flag.Parse()
737
738	if len(*configFlag) == 0 {
739		fmt.Printf("No config file given by --config\n")
740		os.Exit(1)
741	}
742
743	if len(*outputDir) == 0 {
744		fmt.Printf("No output directory given by --out\n")
745		os.Exit(1)
746	}
747
748	configBytes, err := ioutil.ReadFile(*configFlag)
749	if err != nil {
750		fmt.Printf("Failed to open config file: %s\n", err)
751		os.Exit(1)
752	}
753
754	if err := json.Unmarshal(configBytes, &config); err != nil {
755		fmt.Printf("Failed to parse config file: %s\n", err)
756		os.Exit(1)
757	}
758
759	headerDescriptions, err := generate(*outputDir, &config)
760	if err != nil {
761		fmt.Printf("Failed to generate output: %s\n", err)
762		os.Exit(1)
763	}
764
765	if err := generateIndex(*outputDir, &config, headerDescriptions); err != nil {
766		fmt.Printf("Failed to generate index: %s\n", err)
767		os.Exit(1)
768	}
769
770	if err := copyFile(*outputDir, "doc.css"); err != nil {
771		fmt.Printf("Failed to copy static file: %s\n", err)
772		os.Exit(1)
773	}
774}
775