• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2011 Google Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.google.doclava.apicheck;
18 
19 import com.google.doclava.AnnotationInstanceInfo;
20 import com.google.doclava.ClassInfo;
21 import com.google.doclava.Converter;
22 import com.google.doclava.FieldInfo;
23 import com.google.doclava.MethodInfo;
24 import com.google.doclava.PackageInfo;
25 import com.google.doclava.ParameterInfo;
26 import com.google.doclava.SourcePositionInfo;
27 import com.google.doclava.TypeInfo;
28 
29 import java.io.IOException;
30 import java.io.InputStream;
31 import java.util.ArrayList;
32 import java.util.Collections;
33 import java.util.HashSet;
34 import java.util.LinkedList;
35 import java.util.List;
36 
37 public class ApiFile {
38 
parseApi(String filename, InputStream stream)39   public static ApiInfo parseApi(String filename, InputStream stream) throws ApiParseException {
40     final int CHUNK = 1024*1024;
41     int hint = 0;
42     try {
43       hint = stream.available() + CHUNK;
44     } catch (IOException ex) {
45     }
46     if (hint < CHUNK) {
47       hint = CHUNK;
48     }
49     byte[] buf = new byte[hint];
50     int size = 0;
51 
52     try {
53       while (true) {
54         if (size == buf.length) {
55           byte[] tmp = new byte[buf.length+CHUNK];
56           System.arraycopy(buf, 0, tmp, 0, buf.length);
57           buf = tmp;
58         }
59         int amt = stream.read(buf, size, (buf.length-size));
60         if (amt < 0) {
61           break;
62         } else {
63           size += amt;
64         }
65       }
66     } catch (IOException ex) {
67       throw new ApiParseException("Error reading API file", ex);
68     }
69 
70     final Tokenizer tokenizer = new Tokenizer(filename, (new String(buf, 0, size)).toCharArray());
71     final ApiInfo api = new ApiInfo();
72 
73     while (true) {
74       String token = tokenizer.getToken();
75       if (token == null) {
76         break;
77       }
78       if ("package".equals(token)) {
79         parsePackage(api, tokenizer);
80       } else {
81         throw new ApiParseException("expected package got " + token, tokenizer.getLine());
82       }
83     }
84 
85     api.resolveSuperclasses();
86     api.resolveInterfaces();
87 
88     return api;
89   }
90 
parsePackage(ApiInfo api, Tokenizer tokenizer)91   private static void parsePackage(ApiInfo api, Tokenizer tokenizer)
92       throws ApiParseException {
93     String token;
94     String name;
95     PackageInfo pkg;
96 
97     token = tokenizer.requireToken();
98     assertIdent(tokenizer, token);
99     name = token;
100     pkg = new PackageInfo(name, tokenizer.pos());
101     token = tokenizer.requireToken();
102     if (!"{".equals(token)) {
103       throw new ApiParseException("expected '{' got " + token, tokenizer.getLine());
104     }
105     while (true) {
106       token = tokenizer.requireToken();
107       if ("}".equals(token)) {
108         break;
109       } else {
110         parseClass(api, pkg, tokenizer, token);
111       }
112     }
113     api.addPackage(pkg);
114   }
115 
parseClass(ApiInfo api, PackageInfo pkg, Tokenizer tokenizer, String token)116   private static void parseClass(ApiInfo api, PackageInfo pkg, Tokenizer tokenizer, String token)
117       throws ApiParseException {
118     boolean pub = false;
119     boolean prot = false;
120     boolean priv = false;
121     boolean pkgpriv = false;
122     boolean stat = false;
123     boolean fin = false;
124     boolean abs = false;
125     boolean dep = false;
126     boolean iface;
127     String name;
128     String qname;
129     String ext = null;
130     ClassInfo cl;
131 
132     if ("public".equals(token)) {
133       pub = true;
134       token = tokenizer.requireToken();
135     } else if ("protected".equals(token)) {
136       prot = true;
137       token = tokenizer.requireToken();
138     } else if ("private".equals(token)) {
139       priv = true;
140       token = tokenizer.requireToken();
141     } else {
142       pkgpriv = true;
143     }
144     if ("static".equals(token)) {
145       stat = true;
146       token = tokenizer.requireToken();
147     }
148     if ("final".equals(token)) {
149       fin = true;
150       token = tokenizer.requireToken();
151     }
152     if ("abstract".equals(token)) {
153       abs = true;
154       token = tokenizer.requireToken();
155     }
156     if ("deprecated".equals(token)) {
157       dep = true;
158       token = tokenizer.requireToken();
159     }
160     if ("class".equals(token)) {
161       iface = false;
162       token = tokenizer.requireToken();
163     } else if ("interface".equals(token)) {
164       iface = true;
165       token = tokenizer.requireToken();
166     } else {
167       throw new ApiParseException("missing class or interface. got: " + token, tokenizer.getLine());
168     }
169     assertIdent(tokenizer, token);
170     name = token;
171     qname = qualifiedName(pkg.name(), name, null);
172     final TypeInfo typeInfo = Converter.obtainTypeFromString(qname);
173     // Simple type info excludes the package name (but includes enclosing class names)
174     final TypeInfo simpleTypeInfo = Converter.obtainTypeFromString(name);
175     token = tokenizer.requireToken();
176     cl = new ClassInfo(null/*classDoc*/, ""/*rawCommentText*/, tokenizer.pos(), pub, prot,
177         pkgpriv, priv, stat, iface, abs, true/*isOrdinaryClass*/,
178         false/*isException*/, false/*isError*/, false/*isEnum*/, false/*isAnnotation*/,
179         fin, false/*isIncluded*/, simpleTypeInfo.qualifiedTypeName(), typeInfo.qualifiedTypeName(),
180         null/*qualifiedTypeName*/, false/*isPrimitive*/);
181     cl.setTypeInfo(typeInfo);
182     cl.setDeprecated(dep);
183     if ("extends".equals(token)) {
184       token = tokenizer.requireToken();
185       assertIdent(tokenizer, token);
186       ext = token;
187       token = tokenizer.requireToken();
188     }
189     // Resolve superclass after done parsing
190     api.mapClassToSuper(cl, ext);
191     cl.setAnnotations(new ArrayList<AnnotationInstanceInfo>());
192     if ("implements".equals(token)) {
193       while (true) {
194         token = tokenizer.requireToken();
195         if ("{".equals(token)) {
196           break;
197         } else {
198           /// TODO
199           if (!",".equals(token)) {
200             api.mapClassToInterface(cl, token);
201           }
202         }
203       }
204     }
205     if (!"{".equals(token)) {
206       throw new ApiParseException("expected {", tokenizer.getLine());
207     }
208     token = tokenizer.requireToken();
209     while (true) {
210       if ("}".equals(token)) {
211         break;
212       } else if ("ctor".equals(token)) {
213         token = tokenizer.requireToken();
214         parseConstructor(tokenizer, cl, token);
215       } else if ("method".equals(token)) {
216         token = tokenizer.requireToken();
217         parseMethod(tokenizer, cl, token);
218       } else if ("field".equals(token)) {
219         token = tokenizer.requireToken();
220         parseField(tokenizer, cl, token, false);
221       } else if ("enum_constant".equals(token)) {
222         token = tokenizer.requireToken();
223         parseField(tokenizer, cl, token, true);
224       } else {
225         throw new ApiParseException("expected ctor, enum_constant, field or method", tokenizer.getLine());
226       }
227       token = tokenizer.requireToken();
228     }
229     pkg.addClass(cl);
230   }
231 
parseConstructor(Tokenizer tokenizer, ClassInfo cl, String token)232   private static void parseConstructor(Tokenizer tokenizer, ClassInfo cl, String token)
233       throws ApiParseException {
234     boolean pub = false;
235     boolean prot = false;
236     boolean priv = false;
237     boolean pkgpriv = false;
238     boolean dep = false;
239     String name;
240     MethodInfo method;
241 
242     if ("public".equals(token)) {
243       pub = true;
244       token = tokenizer.requireToken();
245     } else if ("protected".equals(token)) {
246       prot = true;
247       token = tokenizer.requireToken();
248     } else if ("private".equals(token)) {
249       priv = true;
250       token = tokenizer.requireToken();
251     } else {
252       pkgpriv = true;
253     }
254     if ("deprecated".equals(token)) {
255       dep = true;
256       token = tokenizer.requireToken();
257     }
258     assertIdent(tokenizer, token);
259     name = token;
260     token = tokenizer.requireToken();
261     if (!"(".equals(token)) {
262       throw new ApiParseException("expected (", tokenizer.getLine());
263     }
264     //method = new MethodInfo(name, cl.qualifiedName(), false/*static*/, false/*final*/, dep,
265     //    pub ? "public" : "protected", tokenizer.pos(), cl);
266     method = new MethodInfo(""/*rawCommentText*/, new ArrayList<TypeInfo>()/*typeParameters*/,
267         name, null/*signature*/, cl, cl, pub, prot, pkgpriv, priv, false/*isFinal*/,
268         false/*isStatic*/, false/*isSynthetic*/, false/*isAbstract*/, false/*isSynthetic*/,
269         false/*isNative*/, false/* isDefault */,
270         false /*isAnnotationElement*/, "constructor", null/*flatSignature*/,
271         null/*overriddenMethod*/, cl.asTypeInfo(), new ArrayList<ParameterInfo>(),
272         new ArrayList<ClassInfo>()/*thrownExceptions*/, tokenizer.pos(),
273         new ArrayList<AnnotationInstanceInfo>()/*annotations*/);
274     method.setDeprecated(dep);
275     token = tokenizer.requireToken();
276     parseParameterList(tokenizer, method, new HashSet<String>(), token);
277     token = tokenizer.requireToken();
278     if ("throws".equals(token)) {
279       token = parseThrows(tokenizer, method);
280     }
281     if (!";".equals(token)) {
282       throw new ApiParseException("expected ; found " + token, tokenizer.getLine());
283     }
284     cl.addConstructor(method);
285   }
286 
parseMethod(Tokenizer tokenizer, ClassInfo cl, String token)287   private static void parseMethod(Tokenizer tokenizer, ClassInfo cl, String token)
288       throws ApiParseException {
289     boolean pub = false;
290     boolean prot = false;
291     boolean priv = false;
292     boolean pkgpriv = false;
293     boolean stat = false;
294     boolean fin = false;
295     boolean abs = false;
296     boolean dep = false;
297     boolean syn = false;
298     boolean def = false;
299     ArrayList<TypeInfo> typeParameters = new ArrayList<>();
300     TypeInfo returnType;
301     HashSet<String> typeVariableNames;
302     String name;
303     String ext = null;
304     MethodInfo method;
305 
306     if ("public".equals(token)) {
307       pub = true;
308       token = tokenizer.requireToken();
309     } else if ("protected".equals(token)) {
310       prot = true;
311       token = tokenizer.requireToken();
312     } else if ("private".equals(token)) {
313       priv = true;
314       token = tokenizer.requireToken();
315     } else {
316       pkgpriv = true;
317     }
318     if ("default".equals(token)) {
319       def = true;
320       token = tokenizer.requireToken();
321     }
322     if ("static".equals(token)) {
323       stat = true;
324       token = tokenizer.requireToken();
325     }
326     if ("final".equals(token)) {
327       fin = true;
328       token = tokenizer.requireToken();
329     }
330     if ("abstract".equals(token)) {
331       abs = true;
332       token = tokenizer.requireToken();
333     }
334     if ("deprecated".equals(token)) {
335       dep = true;
336       token = tokenizer.requireToken();
337     }
338     if ("synchronized".equals(token)) {
339       syn = true;
340       token = tokenizer.requireToken();
341     }
342     if ("<".equals(token)) {
343       parseTypeParameterList(tokenizer, typeParameters, cl);
344       token = tokenizer.requireToken();
345     }
346     assertIdent(tokenizer, token);
347     returnType = Converter.obtainTypeFromString(token);
348     typeVariableNames = TypeInfo.typeVariables(typeParameters);
349     if (typeVariableNames.contains(returnType.qualifiedTypeName())) {
350       returnType.setIsTypeVariable(true);
351     }
352     token = tokenizer.requireToken();
353     assertIdent(tokenizer, token);
354     name = token;
355     method = new MethodInfo(""/*rawCommentText*/, typeParameters, name, null/*signature*/, cl, cl,
356         pub, prot, pkgpriv, priv, fin, stat, false/*isSynthetic*/, abs/*isAbstract*/,
357         syn, false/*isNative*/, def/*isDefault*/, false /*isAnnotationElement*/, "method",
358         null/*flatSignature*/, null/*overriddenMethod*/, returnType,
359         new ArrayList<ParameterInfo>(), new ArrayList<ClassInfo>()/*thrownExceptions*/,
360         tokenizer.pos(), new ArrayList<AnnotationInstanceInfo>()/*annotations*/);
361     method.setDeprecated(dep);
362     token = tokenizer.requireToken();
363     if (!"(".equals(token)) {
364       throw new ApiParseException("expected (", tokenizer.getLine());
365     }
366     token = tokenizer.requireToken();
367     parseParameterList(tokenizer, method, typeVariableNames, token);
368     token = tokenizer.requireToken();
369     if ("throws".equals(token)) {
370       token = parseThrows(tokenizer, method);
371     }
372     if (!";".equals(token)) {
373       throw new ApiParseException("expected ; found " + token, tokenizer.getLine());
374     }
375     cl.addMethod(method);
376   }
377 
parseField(Tokenizer tokenizer, ClassInfo cl, String token, boolean isEnum)378   private static void parseField(Tokenizer tokenizer, ClassInfo cl, String token, boolean isEnum)
379       throws ApiParseException {
380     boolean pub = false;
381     boolean prot = false;
382     boolean priv = false;
383     boolean pkgpriv = false;
384     boolean stat = false;
385     boolean fin = false;
386     boolean dep = false;
387     boolean trans = false;
388     boolean vol = false;
389     String type;
390     String name;
391     String val = null;
392     Object v;
393     FieldInfo field;
394 
395     if ("public".equals(token)) {
396       pub = true;
397       token = tokenizer.requireToken();
398     } else if ("protected".equals(token)) {
399       prot = true;
400       token = tokenizer.requireToken();
401     } else if ("private".equals(token)) {
402       priv = true;
403       token = tokenizer.requireToken();
404     } else {
405       pkgpriv = true;
406     }
407     if ("static".equals(token)) {
408       stat = true;
409       token = tokenizer.requireToken();
410     }
411     if ("final".equals(token)) {
412       fin = true;
413       token = tokenizer.requireToken();
414     }
415     if ("deprecated".equals(token)) {
416       dep = true;
417       token = tokenizer.requireToken();
418     }
419     if ("transient".equals(token)) {
420       trans = true;
421       token = tokenizer.requireToken();
422     }
423     if ("volatile".equals(token)) {
424       vol = true;
425       token = tokenizer.requireToken();
426     }
427     assertIdent(tokenizer, token);
428     type = token;
429     token = tokenizer.requireToken();
430     assertIdent(tokenizer, token);
431     name = token;
432     token = tokenizer.requireToken();
433     if ("=".equals(token)) {
434       token = tokenizer.requireToken(false);
435       val = token;
436       token = tokenizer.requireToken();
437     }
438     if (!";".equals(token)) {
439       throw new ApiParseException("expected ; found " + token, tokenizer.getLine());
440     }
441     try {
442       v = parseValue(type, val);
443     } catch (ApiParseException ex) {
444       ex.line = tokenizer.getLine();
445       throw ex;
446     }
447     field = new FieldInfo(name, cl, cl, pub, prot, pkgpriv, priv, fin, stat,
448         trans, vol, false, Converter.obtainTypeFromString(type), "", v, tokenizer.pos(),
449         new ArrayList<AnnotationInstanceInfo>());
450     field.setDeprecated(dep);
451     if (isEnum) {
452       cl.addEnumConstant(field);
453     } else {
454       cl.addField(field);
455     }
456   }
457 
parseValue(String type, String val)458   public static Object parseValue(String type, String val) throws ApiParseException {
459     if (val != null) {
460       if ("boolean".equals(type)) {
461         return "true".equals(val) ? Boolean.TRUE : Boolean.FALSE;
462       } else if ("byte".equals(type)) {
463         return Integer.valueOf(val);
464       } else if ("short".equals(type)) {
465         return Integer.valueOf(val);
466       } else if ("int".equals(type)) {
467         return Integer.valueOf(val);
468       } else if ("long".equals(type)) {
469         return Long.valueOf(val.substring(0, val.length()-1));
470       } else if ("float".equals(type)) {
471         if ("(1.0f/0.0f)".equals(val) || "(1.0f / 0.0f)".equals(val)) {
472           return Float.POSITIVE_INFINITY;
473         } else if ("(-1.0f/0.0f)".equals(val) || "(-1.0f / 0.0f)".equals(val)) {
474           return Float.NEGATIVE_INFINITY;
475         } else if ("(0.0f/0.0f)".equals(val) || "(0.0f / 0.0f)".equals(val)) {
476           return Float.NaN;
477         } else {
478           return Float.valueOf(val);
479         }
480       } else if ("double".equals(type)) {
481         if ("(1.0/0.0)".equals(val) || "(1.0 / 0.0)".equals(val)) {
482           return Double.POSITIVE_INFINITY;
483         } else if ("(-1.0/0.0)".equals(val) || "(-1.0 / 0.0)".equals(val)) {
484           return Double.NEGATIVE_INFINITY;
485         } else if ("(0.0/0.0)".equals(val) || "(0.0 / 0.0)".equals(val)) {
486           return Double.NaN;
487         } else {
488           return Double.valueOf(val);
489         }
490       } else if ("char".equals(type)) {
491         return new Integer((char)Integer.parseInt(val));
492       } else if ("java.lang.String".equals(type)) {
493         if ("null".equals(val)) {
494           return null;
495         } else {
496           return FieldInfo.javaUnescapeString(val.substring(1, val.length()-1));
497         }
498       }
499     }
500     if ("null".equals(val)) {
501       return null;
502     } else {
503       return val;
504     }
505   }
506 
parseTypeParameterList(Tokenizer tokenizer, List<TypeInfo> methodTypeParameters, ClassInfo cl)507   private static void parseTypeParameterList(Tokenizer tokenizer,
508       List<TypeInfo> methodTypeParameters, ClassInfo cl) throws ApiParseException {
509     String token;
510     HashSet<String> variables = cl.typeVariables();
511     do {
512       token = tokenizer.requireToken();
513       assertIdent(tokenizer, token);
514       TypeInfo type = new TypeInfo(token);
515       type.setIsTypeVariable(true);
516       variables.add(type.qualifiedTypeName());
517       ArrayList<TypeInfo> extendsBounds = new ArrayList<>();
518       token = tokenizer.requireToken();
519       if ("extends".equals(token)) {
520         do {
521           token = tokenizer.requireToken();
522           assertIdent(tokenizer, token);
523           extendsBounds.add(new TypeInfo(token));
524           token = tokenizer.requireToken();
525         } while ("&".equals(token));
526       }
527       if (!extendsBounds.isEmpty()) {
528         type.setBounds(null, extendsBounds);
529       }
530       methodTypeParameters.add(type);
531     } while (",".equals(token));
532 
533     // Type variables aren't guaranteed to be declared before they're referenced so we need to wait
534     // until after we've processed them all to figure out which ones are type variables and which
535     // ones are classes (which we may not have processed yet either).
536     for (TypeInfo type : methodTypeParameters) {
537       type.resolveTypeVariables(variables);
538     }
539 
540     if (!">".equals(token)) {
541       throw new ApiParseException("Expected '>' to end type parameter list, found "
542           + token, tokenizer.getLine());
543     }
544   }
545 
parseParameterList(Tokenizer tokenizer, AbstractMethodInfo method, HashSet<String> typeParameters, String token)546   private static void parseParameterList(Tokenizer tokenizer, AbstractMethodInfo method,
547       HashSet<String> typeParameters, String token) throws ApiParseException {
548     while (true) {
549       if (")".equals(token)) {
550         return;
551       }
552 
553       String type = token;
554       String name = null;
555       token = tokenizer.requireToken();
556       if (isIdent(token)) {
557         name = token;
558         token = tokenizer.requireToken();
559       }
560       if (",".equals(token)) {
561         token = tokenizer.requireToken();
562       } else if (")".equals(token)) {
563       } else {
564         throw new ApiParseException("expected , found " + token, tokenizer.getLine());
565       }
566       // api file does not preserve annotations.
567       List<AnnotationInstanceInfo> annotations = Collections.emptyList();
568       TypeInfo typeInfo = Converter.obtainTypeFromString(type);
569       if (typeParameters.contains(typeInfo.qualifiedTypeName())) {
570         typeInfo.setIsTypeVariable(true);
571       }
572       method.addParameter(new ParameterInfo(name, type,
573             typeInfo,
574             type.endsWith("..."),
575             tokenizer.pos(),
576             annotations));
577       if (type.endsWith("...")) {
578         method.setVarargs(true);
579       }
580     }
581   }
582 
parseThrows(Tokenizer tokenizer, AbstractMethodInfo method)583   private static String parseThrows(Tokenizer tokenizer, AbstractMethodInfo method)
584       throws ApiParseException {
585     String token = tokenizer.requireToken();
586     boolean comma = true;
587     while (true) {
588       if (";".equals(token)) {
589         return token;
590       } else if (",".equals(token)) {
591         if (comma) {
592           throw new ApiParseException("Expected exception, got ','", tokenizer.getLine());
593         }
594         comma = true;
595       } else {
596         if (!comma) {
597           throw new ApiParseException("Expected ',' or ';' got " + token, tokenizer.getLine());
598         }
599         comma = false;
600         method.addException(token);
601       }
602       token = tokenizer.requireToken();
603     }
604   }
605 
qualifiedName(String pkg, String className, ClassInfo parent)606   private static String qualifiedName(String pkg, String className, ClassInfo parent) {
607     String parentQName = (parent != null) ? (parent.qualifiedName() + ".") : "";
608     return pkg + "." + parentQName + className;
609   }
610 
isIdent(String token)611   public static boolean isIdent(String token) {
612     return isident(token.charAt(0));
613   }
614 
assertIdent(Tokenizer tokenizer, String token)615   public static void assertIdent(Tokenizer tokenizer, String token) throws ApiParseException {
616     if (!isident(token.charAt(0))) {
617       throw new ApiParseException("Expected identifier: " + token, tokenizer.getLine());
618     }
619   }
620 
621   static class Tokenizer {
622     char[] mBuf;
623     String mFilename;
624     int mPos;
625     int mLine = 1;
Tokenizer(String filename, char[] buf)626     Tokenizer(String filename, char[] buf) {
627       mFilename = filename;
628       mBuf = buf;
629     }
630 
pos()631     public SourcePositionInfo pos() {
632       return new SourcePositionInfo(mFilename, mLine, 0);
633     }
634 
getLine()635     public int getLine() {
636       return mLine;
637     }
638 
eatWhitespace()639     boolean eatWhitespace() {
640       boolean ate = false;
641       while (mPos < mBuf.length && isspace(mBuf[mPos])) {
642         if (mBuf[mPos] == '\n') {
643           mLine++;
644         }
645         mPos++;
646         ate = true;
647       }
648       return ate;
649     }
650 
eatComment()651     boolean eatComment() {
652       if (mPos+1 < mBuf.length) {
653         if (mBuf[mPos] == '/' && mBuf[mPos+1] == '/') {
654           mPos += 2;
655           while (mPos < mBuf.length && !isnewline(mBuf[mPos])) {
656             mPos++;
657           }
658           return true;
659         }
660       }
661       return false;
662     }
663 
eatWhitespaceAndComments()664     void eatWhitespaceAndComments() {
665       while (eatWhitespace() || eatComment()) {
666       }
667     }
668 
requireToken()669     public String requireToken() throws ApiParseException {
670       return requireToken(true);
671     }
672 
requireToken(boolean parenIsSep)673     public String requireToken(boolean parenIsSep) throws ApiParseException {
674       final String token = getToken(parenIsSep);
675       if (token != null) {
676         return token;
677       } else {
678         throw new ApiParseException("Unexpected end of file", mLine);
679       }
680     }
681 
getToken()682     public String getToken() throws ApiParseException {
683       return getToken(true);
684     }
685 
getToken(boolean parenIsSep)686     public String getToken(boolean parenIsSep) throws ApiParseException {
687       eatWhitespaceAndComments();
688       if (mPos >= mBuf.length) {
689         return null;
690       }
691       final int line = mLine;
692       final char c = mBuf[mPos];
693       final int start = mPos;
694       mPos++;
695       if (c == '"') {
696         final int STATE_BEGIN = 0;
697         final int STATE_ESCAPE = 1;
698         int state = STATE_BEGIN;
699         while (true) {
700           if (mPos >= mBuf.length) {
701             throw new ApiParseException("Unexpected end of file for \" starting at " + line, mLine);
702           }
703           final char k = mBuf[mPos];
704           if (k == '\n' || k == '\r') {
705             throw new ApiParseException("Unexpected newline for \" starting at " + line, mLine);
706           }
707           mPos++;
708           switch (state) {
709             case STATE_BEGIN:
710               switch (k) {
711                 case '\\':
712                   state = STATE_ESCAPE;
713                   mPos++;
714                   break;
715                 case '"':
716                   return new String(mBuf, start, mPos-start);
717               }
718             case STATE_ESCAPE:
719               state = STATE_BEGIN;
720               break;
721           }
722         }
723       } else if (issep(c, parenIsSep)) {
724         return "" + c;
725       } else {
726         int genericDepth = 0;
727         do {
728           while (mPos < mBuf.length && !isspace(mBuf[mPos]) && !issep(mBuf[mPos], parenIsSep)) {
729             mPos++;
730           }
731           if (mPos < mBuf.length) {
732             if (mBuf[mPos] == '<') {
733               genericDepth++;
734               mPos++;
735             } else if (genericDepth != 0) {
736               if (mBuf[mPos] == '>') {
737                 genericDepth--;
738               }
739               mPos++;
740             }
741           }
742         } while (mPos < mBuf.length
743             && ((!isspace(mBuf[mPos]) && !issep(mBuf[mPos], parenIsSep)) || genericDepth != 0));
744         if (mPos >= mBuf.length) {
745           throw new ApiParseException("Unexpected end of file for \" starting at " + line, mLine);
746         }
747         return new String(mBuf, start, mPos-start);
748       }
749     }
750   }
751 
isspace(char c)752   static boolean isspace(char c) {
753     return c == ' ' || c == '\t' || c == '\n' || c == '\r';
754   }
755 
isnewline(char c)756   static boolean isnewline(char c) {
757     return c == '\n' || c == '\r';
758   }
759 
issep(char c, boolean parenIsSep)760   static boolean issep(char c, boolean parenIsSep) {
761     if (parenIsSep) {
762       if (c == '(' || c == ')') {
763         return true;
764       }
765     }
766     return c == '{' || c == '}' || c == ',' || c == ';' || c == '<' || c == '>';
767   }
768 
isident(char c)769   static boolean isident(char c) {
770     if (c == '"' || issep(c, true)) {
771       return false;
772     }
773     return true;
774   }
775 }
776