1 /* 2 * Copyright (C) 2011 Google Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tools.metalava.doclava1; 18 19 import com.android.tools.lint.checks.infrastructure.ClassNameKt; 20 import com.android.tools.metalava.FileFormat; 21 import com.android.tools.metalava.model.AnnotationItem; 22 import com.android.tools.metalava.model.DefaultModifierList; 23 import com.android.tools.metalava.model.TypeParameterList; 24 import com.android.tools.metalava.model.VisibilityLevel; 25 import com.android.tools.metalava.model.text.TextClassItem; 26 import com.android.tools.metalava.model.text.TextConstructorItem; 27 import com.android.tools.metalava.model.text.TextFieldItem; 28 import com.android.tools.metalava.model.text.TextMethodItem; 29 import com.android.tools.metalava.model.text.TextModifiers; 30 import com.android.tools.metalava.model.text.TextPackageItem; 31 import com.android.tools.metalava.model.text.TextParameterItem; 32 import com.android.tools.metalava.model.text.TextParameterItemKt; 33 import com.android.tools.metalava.model.text.TextPropertyItem; 34 import com.android.tools.metalava.model.text.TextTypeItem; 35 import com.android.tools.metalava.model.text.TextTypeParameterList; 36 import com.google.common.annotations.VisibleForTesting; 37 import com.google.common.io.Files; 38 import kotlin.Pair; 39 import kotlin.text.StringsKt; 40 import org.jetbrains.annotations.Nullable; 41 42 import javax.annotation.Nonnull; 43 import java.io.File; 44 import java.io.IOException; 45 import java.util.ArrayList; 46 import java.util.List; 47 48 import static com.android.tools.metalava.ConstantsKt.ANDROIDX_NONNULL; 49 import static com.android.tools.metalava.ConstantsKt.ANDROIDX_NULLABLE; 50 import static com.android.tools.metalava.ConstantsKt.JAVA_LANG_ANNOTATION; 51 import static com.android.tools.metalava.ConstantsKt.JAVA_LANG_ENUM; 52 import static com.android.tools.metalava.ConstantsKt.JAVA_LANG_STRING; 53 import static com.android.tools.metalava.model.FieldItemKt.javaUnescapeString; 54 import static kotlin.text.Charsets.UTF_8; 55 56 // 57 // Copied from doclava1, but adapted to metalava's code model (plus tweaks to handle 58 // metalava's richer files, e.g. annotations) 59 // 60 public class ApiFile { 61 /** 62 * Same as {@link #parseApi(List, boolean)}}, but take a single file for convenience. 63 * 64 * @param file input signature file 65 * @param kotlinStyleNulls if true, we assume the input has a kotlin style nullability markers (e.g. "?"). 66 * Even if false, we'll allow them if the file format supports them/ 67 */ parseApi(@onnull File file, boolean kotlinStyleNulls)68 public static TextCodebase parseApi(@Nonnull File file, boolean kotlinStyleNulls) throws ApiParseException { 69 final List<File> files = new ArrayList<>(1); 70 files.add(file); 71 return parseApi(files, kotlinStyleNulls); 72 } 73 74 /** 75 * Read API signature files into a {@link TextCodebase}. 76 * 77 * Note: when reading from them multiple files, {@link TextCodebase#getLocation} would refer to the first 78 * file specified. each {@link com.android.tools.metalava.model.text.TextItem#getPosition} would correctly 79 * point out the source file of each item. 80 * 81 * @param files input signature files 82 * @param kotlinStyleNulls if true, we assume the input has a kotlin style nullability markers (e.g. "?"). 83 * Even if false, we'll allow them if the file format supports them/ 84 */ parseApi(@onnull List<File> files, boolean kotlinStyleNulls)85 public static TextCodebase parseApi(@Nonnull List<File> files, boolean kotlinStyleNulls) 86 throws ApiParseException { 87 if (files.size() == 0) { 88 throw new IllegalArgumentException("files must not be empty"); 89 } 90 final TextCodebase api = new TextCodebase(files.get(0)); 91 final StringBuilder description = new StringBuilder("Codebase loaded from "); 92 93 boolean first = true; 94 for (File file : files) { 95 if (!first) { 96 description.append(", "); 97 } 98 description.append(file.getPath()); 99 100 final String apiText; 101 try { 102 apiText = Files.asCharSource(file, UTF_8).read(); 103 } catch (IOException ex) { 104 throw new ApiParseException("Error reading API file", file.getPath(), ex); 105 } 106 parseApiSingleFile(api, !first, file.getPath(), apiText, kotlinStyleNulls); 107 first = false; 108 } 109 api.setDescription(description.toString()); 110 api.postProcess(); 111 return api; 112 } 113 114 /** @deprecated Exists only for external callers. */ 115 @Deprecated parseApi(@onnull String filename, @Nonnull String apiText, Boolean kotlinStyleNulls)116 public static TextCodebase parseApi(@Nonnull String filename, @Nonnull String apiText, 117 Boolean kotlinStyleNulls) throws ApiParseException { 118 return parseApi(filename, apiText, kotlinStyleNulls != null && kotlinStyleNulls); 119 } 120 121 /** 122 * Entry point fo test. Take a filename and content separately. 123 */ 124 @VisibleForTesting parseApi(@onnull String filename, @Nonnull String apiText, boolean kotlinStyleNulls)125 public static TextCodebase parseApi(@Nonnull String filename, @Nonnull String apiText, 126 boolean kotlinStyleNulls) throws ApiParseException { 127 final TextCodebase api = new TextCodebase(new File(filename)); 128 api.setDescription("Codebase loaded from " + filename); 129 parseApiSingleFile(api, false, filename, apiText, kotlinStyleNulls); 130 api.postProcess(); 131 return api; 132 } 133 parseApiSingleFile(TextCodebase api, boolean appending, String filename, String apiText, boolean kotlinStyleNulls)134 private static void parseApiSingleFile(TextCodebase api, boolean appending, String filename, String apiText, 135 boolean kotlinStyleNulls) throws ApiParseException { 136 // Infer the format. 137 FileFormat format = FileFormat.Companion.parseHeader(apiText); 138 139 // If it's the first file, set the format. Otherwise, make sure the format is the same as the prior files. 140 if (!appending) { 141 // This is the first file to process. 142 api.setFormat(format); 143 } else { 144 // If we're appending to another API file, make sure the format is the same. 145 if (!format.equals(api.getFormat())) { 146 throw new ApiParseException(String.format( 147 "Cannot merge different formats of signature files. First file format=%s, current file format=%s: file=%s", 148 api.getFormat(), format, filename)); 149 } 150 // When we're appending, and the content is empty, nothing to do. 151 if (StringsKt.isBlank(apiText)) { 152 return; 153 } 154 } 155 156 // Even if kotlinStyleNulls is false, still allow kotlin nullability markers, if the format allows them. 157 if (format.isSignatureFormat()) { 158 if (!kotlinStyleNulls) { 159 kotlinStyleNulls = format.useKotlinStyleNulls(); 160 } 161 } else if (StringsKt.isBlank(apiText)) { 162 // Sometimes, signature files are empty, and we do want to accept them. 163 } else { 164 throw new ApiParseException("Unknown file format of " + filename); 165 } 166 167 if (kotlinStyleNulls) { 168 api.setKotlinStyleNulls(true); 169 } 170 171 // Remove the block comments. 172 if (apiText.contains("/*")) { 173 apiText = ClassNameKt.stripComments(apiText, false); // line comments are used to stash field constants 174 } 175 176 final Tokenizer tokenizer = new Tokenizer(filename, apiText.toCharArray()); 177 while (true) { 178 String token = tokenizer.getToken(); 179 if (token == null) { 180 break; 181 } 182 // TODO: Accept annotations on packages. 183 if ("package".equals(token)) { 184 parsePackage(api, tokenizer); 185 } else { 186 throw new ApiParseException("expected package got " + token, tokenizer); 187 } 188 } 189 } 190 parsePackage(TextCodebase api, Tokenizer tokenizer)191 private static void parsePackage(TextCodebase api, Tokenizer tokenizer) 192 throws ApiParseException { 193 String token; 194 String name; 195 TextPackageItem pkg; 196 197 token = tokenizer.requireToken(); 198 199 // Metalava: including annotations in file now 200 List<String> annotations = getAnnotations(tokenizer, token); 201 TextModifiers modifiers = new TextModifiers(api, DefaultModifierList.PUBLIC, null); 202 if (annotations != null) { 203 modifiers.addAnnotations(annotations); 204 } 205 206 token = tokenizer.getCurrent(); 207 208 assertIdent(tokenizer, token); 209 name = token; 210 211 // If the same package showed up multiple times, make sure they have the same modifiers. 212 // (Packages can't have public/private/etc, but they can have annotations, which are part of ModifierList.) 213 // ModifierList doesn't provide equals(), neither does AnnotationItem which ModifilerList contains, 214 // so we just use toString() here for equality comparison. 215 // However, ModifierList.toString() throws if the owner is not yet set, so we have to instantiate an 216 // (owner) TextPackageItem here. 217 // If it's a duplicate package, then we'll replace pkg with the existing one in the following if block. 218 219 // TODO: However, currently this parser can't handle annotations on packages, so we will never hit this case. 220 // Once the parser supports that, we should add a test case for this too. 221 pkg = new TextPackageItem(api, name, modifiers, tokenizer.pos()); 222 223 final TextPackageItem existing = api.findPackage(name); 224 if (existing != null) { 225 if (!pkg.getModifiers().toString().equals(existing.getModifiers().toString())) { 226 throw new ApiParseException(String.format( 227 "Contradicting declaration of package %s. Previously seen with modifiers \"%s\", but now with \"%s\"", 228 name, pkg.getModifiers(), modifiers), tokenizer); 229 } 230 pkg = existing; 231 } 232 233 token = tokenizer.requireToken(); 234 if (!"{".equals(token)) { 235 throw new ApiParseException("expected '{' got " + token, tokenizer); 236 } 237 while (true) { 238 token = tokenizer.requireToken(); 239 if ("}".equals(token)) { 240 break; 241 } else { 242 parseClass(api, pkg, tokenizer, token); 243 } 244 } 245 api.addPackage(pkg); 246 } 247 parseClass(TextCodebase api, TextPackageItem pkg, Tokenizer tokenizer, String token)248 private static void parseClass(TextCodebase api, TextPackageItem pkg, Tokenizer tokenizer, String token) 249 throws ApiParseException { 250 boolean isInterface = false; 251 boolean isAnnotation = false; 252 boolean isEnum = false; 253 String name; 254 String qualifiedName; 255 String ext = null; 256 TextClassItem cl; 257 258 // Metalava: including annotations in file now 259 List<String> annotations = getAnnotations(tokenizer, token); 260 token = tokenizer.getCurrent(); 261 262 TextModifiers modifiers = parseModifiers(api, tokenizer, token, annotations); 263 token = tokenizer.getCurrent(); 264 265 if ("class".equals(token)) { 266 token = tokenizer.requireToken(); 267 } else if ("interface".equals(token)) { 268 isInterface = true; 269 modifiers.setAbstract(true); 270 token = tokenizer.requireToken(); 271 } else if ("@interface".equals(token)) { 272 // Annotation 273 modifiers.setAbstract(true); 274 isAnnotation = true; 275 token = tokenizer.requireToken(); 276 } else if ("enum".equals(token)) { 277 isEnum = true; 278 modifiers.setFinal(true); 279 modifiers.setStatic(true); 280 ext = JAVA_LANG_ENUM; 281 token = tokenizer.requireToken(); 282 } else { 283 throw new ApiParseException("missing class or interface. got: " + token, tokenizer); 284 } 285 assertIdent(tokenizer, token); 286 name = token; 287 qualifiedName = qualifiedName(pkg.name(), name); 288 289 if (api.findClass(qualifiedName) != null) { 290 throw new ApiParseException("Duplicate class found: " + qualifiedName, tokenizer); 291 } 292 293 final TextTypeItem typeInfo = api.obtainTypeFromString(qualifiedName); 294 // Simple type info excludes the package name (but includes enclosing class names) 295 296 String rawName = name; 297 int variableIndex = rawName.indexOf('<'); 298 if (variableIndex != -1) { 299 rawName = rawName.substring(0, variableIndex); 300 } 301 302 token = tokenizer.requireToken(); 303 304 cl = new TextClassItem(api, tokenizer.pos(), modifiers, isInterface, isEnum, isAnnotation, 305 typeInfo.toErasedTypeString(null), typeInfo.qualifiedTypeName(), 306 rawName, annotations); 307 cl.setContainingPackage(pkg); 308 cl.setTypeInfo(typeInfo); 309 cl.setDeprecated(modifiers.isDeprecated()); 310 if ("extends".equals(token)) { 311 token = tokenizer.requireToken(); 312 assertIdent(tokenizer, token); 313 ext = token; 314 token = tokenizer.requireToken(); 315 } 316 // Resolve superclass after done parsing 317 api.mapClassToSuper(cl, ext); 318 if ("implements".equals(token) || "extends".equals(token) || 319 isInterface && ext != null && !token.equals("{")) { 320 if (!token.equals("implements") && !token.equals("extends")) { 321 api.mapClassToInterface(cl, token); 322 } 323 while (true) { 324 token = tokenizer.requireToken(); 325 if ("{".equals(token)) { 326 break; 327 } else { 328 /// TODO 329 if (!",".equals(token)) { 330 api.mapClassToInterface(cl, token); 331 } 332 } 333 } 334 } 335 if (JAVA_LANG_ENUM.equals(ext)) { 336 cl.setIsEnum(true); 337 // Above we marked all enums as static but for a top level class it's implicit 338 if (!cl.fullName().contains(".")) { 339 cl.getModifiers().setStatic(false); 340 } 341 } else if (isAnnotation) { 342 api.mapClassToInterface(cl, JAVA_LANG_ANNOTATION); 343 } else if (api.implementsInterface(cl, JAVA_LANG_ANNOTATION)) { 344 cl.setIsAnnotationType(true); 345 } 346 if (!"{".equals(token)) { 347 throw new ApiParseException("expected {, was " + token, tokenizer); 348 } 349 token = tokenizer.requireToken(); 350 while (true) { 351 if ("}".equals(token)) { 352 break; 353 } else if ("ctor".equals(token)) { 354 token = tokenizer.requireToken(); 355 parseConstructor(api, tokenizer, cl, token); 356 } else if ("method".equals(token)) { 357 token = tokenizer.requireToken(); 358 parseMethod(api, tokenizer, cl, token); 359 } else if ("field".equals(token)) { 360 token = tokenizer.requireToken(); 361 parseField(api, tokenizer, cl, token, false); 362 } else if ("enum_constant".equals(token)) { 363 token = tokenizer.requireToken(); 364 parseField(api, tokenizer, cl, token, true); 365 } else if ("property".equals(token)) { 366 token = tokenizer.requireToken(); 367 parseProperty(api, tokenizer, cl, token); 368 } else { 369 throw new ApiParseException("expected ctor, enum_constant, field or method", tokenizer); 370 } 371 token = tokenizer.requireToken(); 372 } 373 pkg.addClass(cl); 374 } 375 processKotlinTypeSuffix(TextCodebase api, String type, List<String> annotations)376 private static Pair<String, List<String>> processKotlinTypeSuffix(TextCodebase api, String type, List<String> annotations) throws ApiParseException { 377 boolean varArgs = false; 378 if (type.endsWith("...")) { 379 type = type.substring(0, type.length() - 3); 380 varArgs = true; 381 } 382 if (api.getKotlinStyleNulls()) { 383 if (type.endsWith("?")) { 384 type = type.substring(0, type.length() - 1); 385 annotations = mergeAnnotations(annotations, ANDROIDX_NULLABLE); 386 } else if (type.endsWith("!")) { 387 type = type.substring(0, type.length() - 1); 388 } else if (!type.endsWith("!")) { 389 if (!TextTypeItem.Companion.isPrimitive(type)) { // Don't add nullness on primitive types like void 390 annotations = mergeAnnotations(annotations, ANDROIDX_NONNULL); 391 } 392 } 393 } else if (type.endsWith("?") || type.endsWith("!")) { 394 throw new ApiParseException("Did you forget to supply --input-kotlin-nulls? Found Kotlin-style null type suffix when parser was not configured " + 395 "to interpret signature file that way: " + type); 396 } 397 if (varArgs) { 398 type = type + "..."; 399 } 400 return new Pair<>(type, annotations); 401 } 402 getAnnotations(Tokenizer tokenizer, String token)403 private static List<String> getAnnotations(Tokenizer tokenizer, String token) throws ApiParseException { 404 List<String> annotations = null; 405 406 while (true) { 407 if (token.startsWith("@")) { 408 // Annotation 409 String annotation = token; 410 411 // Restore annotations that were shortened on export 412 annotation = AnnotationItem.Companion.unshortenAnnotation(annotation); 413 414 token = tokenizer.requireToken(); 415 if (token.equals("(")) { 416 // Annotation arguments; potentially nested 417 int balance = 0; 418 int start = tokenizer.offset() - 1; 419 while (true) { 420 if (token.equals("(")) { 421 balance++; 422 } else if (token.equals(")")) { 423 balance--; 424 if (balance == 0) { 425 break; 426 } 427 } 428 token = tokenizer.requireToken(); 429 } 430 annotation += tokenizer.getStringFromOffset(start); 431 token = tokenizer.requireToken(); 432 } 433 if (annotations == null) { 434 annotations = new ArrayList<>(); 435 } 436 annotations.add(annotation); 437 } else { 438 break; 439 } 440 } 441 442 return annotations; 443 } 444 parseConstructor(TextCodebase api, Tokenizer tokenizer, TextClassItem cl, String token)445 private static void parseConstructor(TextCodebase api, Tokenizer tokenizer, TextClassItem cl, String token) 446 throws ApiParseException { 447 String name; 448 TextConstructorItem method; 449 450 // Metalava: including annotations in file now 451 List<String> annotations = getAnnotations(tokenizer, token); 452 token = tokenizer.getCurrent(); 453 454 TextModifiers modifiers = parseModifiers(api, tokenizer, token, annotations); 455 token = tokenizer.getCurrent(); 456 457 assertIdent(tokenizer, token); 458 name = token.substring(token.lastIndexOf('.') + 1); // For inner classes, strip outer classes from name 459 token = tokenizer.requireToken(); 460 if (!"(".equals(token)) { 461 throw new ApiParseException("expected (", tokenizer); 462 } 463 method = new TextConstructorItem(api, name, cl, modifiers, cl.asTypeInfo(), tokenizer.pos()); 464 method.setDeprecated(modifiers.isDeprecated()); 465 parseParameterList(api, tokenizer, method); 466 token = tokenizer.requireToken(); 467 if ("throws".equals(token)) { 468 token = parseThrows(tokenizer, method); 469 } 470 if (!";".equals(token)) { 471 throw new ApiParseException("expected ; found " + token, tokenizer); 472 } 473 cl.addConstructor(method); 474 } 475 parseMethod(TextCodebase api, Tokenizer tokenizer, TextClassItem cl, String token)476 private static void parseMethod(TextCodebase api, Tokenizer tokenizer, TextClassItem cl, String token) 477 throws ApiParseException { 478 TextTypeItem returnType; 479 String name; 480 TextMethodItem method; 481 TypeParameterList typeParameterList = TypeParameterList.Companion.getNONE(); 482 483 // Metalava: including annotations in file now 484 List<String> annotations = getAnnotations(tokenizer, token); 485 token = tokenizer.getCurrent(); 486 487 TextModifiers modifiers = parseModifiers(api, tokenizer, token, null); 488 token = tokenizer.getCurrent(); 489 490 if ("<".equals(token)) { 491 typeParameterList = parseTypeParameterList(api, tokenizer); 492 token = tokenizer.requireToken(); 493 } 494 assertIdent(tokenizer, token); 495 496 Pair<String, List<String>> kotlinTypeSuffix = processKotlinTypeSuffix(api, token, annotations); 497 token = kotlinTypeSuffix.getFirst(); 498 annotations = kotlinTypeSuffix.getSecond(); 499 modifiers.addAnnotations(annotations); 500 String returnTypeString = token; 501 502 token = tokenizer.requireToken(); 503 504 if (returnTypeString.contains("@") && (returnTypeString.indexOf('<') == -1 || 505 returnTypeString.indexOf('@') < returnTypeString.indexOf('<'))) { 506 returnTypeString += " " + token; 507 token = tokenizer.requireToken(); 508 } 509 while (true) { 510 if (token.contains("@") && (token.indexOf('<') == -1 || 511 token.indexOf('@') < token.indexOf('<'))) { 512 // Type-use annotations in type; keep accumulating 513 returnTypeString += " " + token; 514 token = tokenizer.requireToken(); 515 if (token.startsWith("[")) { // TODO: This isn't general purpose; make requireToken smarter! 516 returnTypeString += " " + token; 517 token = tokenizer.requireToken(); 518 } 519 } else { 520 break; 521 } 522 } 523 524 returnType = api.obtainTypeFromString(returnTypeString, cl, typeParameterList); 525 526 assertIdent(tokenizer, token); 527 name = token; 528 method = new TextMethodItem(api, name, cl, modifiers, returnType, tokenizer.pos()); 529 method.setDeprecated(modifiers.isDeprecated()); 530 if (cl.isInterface() && !modifiers.isDefault() && !modifiers.isStatic()) { 531 modifiers.setAbstract(true); 532 } 533 method.setTypeParameterList(typeParameterList); 534 if (typeParameterList instanceof TextTypeParameterList) { 535 ((TextTypeParameterList) typeParameterList).setOwner(method); 536 } 537 token = tokenizer.requireToken(); 538 if (!"(".equals(token)) { 539 throw new ApiParseException("expected (, was " + token, tokenizer); 540 } 541 parseParameterList(api, tokenizer, method); 542 token = tokenizer.requireToken(); 543 if ("throws".equals(token)) { 544 token = parseThrows(tokenizer, method); 545 } 546 if ("default".equals(token)) { 547 token = parseDefault(tokenizer, method); 548 } 549 if (!";".equals(token)) { 550 throw new ApiParseException("expected ; found " + token, tokenizer); 551 } 552 cl.addMethod(method); 553 } 554 mergeAnnotations(List<String> annotations, String annotation)555 private static List<String> mergeAnnotations(List<String> annotations, String annotation) { 556 if (annotations == null) { 557 annotations = new ArrayList<>(); 558 } 559 // Reverse effect of TypeItem.shortenTypes(...) 560 String qualifiedName = annotation.indexOf('.') == -1 561 ? "@androidx.annotation" + annotation 562 : "@" + annotation; 563 564 annotations.add(qualifiedName); 565 return annotations; 566 } 567 parseField(TextCodebase api, Tokenizer tokenizer, TextClassItem cl, String token, boolean isEnum)568 private static void parseField(TextCodebase api, Tokenizer tokenizer, TextClassItem cl, String token, boolean isEnum) 569 throws ApiParseException { 570 List<String> annotations = getAnnotations(tokenizer, token); 571 token = tokenizer.getCurrent(); 572 573 TextModifiers modifiers = parseModifiers(api, tokenizer, token, null); 574 token = tokenizer.getCurrent(); 575 assertIdent(tokenizer, token); 576 577 Pair<String, List<String>> kotlinTypeSuffix = processKotlinTypeSuffix(api, token, annotations); 578 token = kotlinTypeSuffix.getFirst(); 579 annotations = kotlinTypeSuffix.getSecond(); 580 modifiers.addAnnotations(annotations); 581 582 String type = token; 583 TextTypeItem typeInfo = api.obtainTypeFromString(type); 584 585 token = tokenizer.requireToken(); 586 assertIdent(tokenizer, token); 587 String name = token; 588 token = tokenizer.requireToken(); 589 Object value = null; 590 if ("=".equals(token)) { 591 token = tokenizer.requireToken(false); 592 value = parseValue(type, token); 593 token = tokenizer.requireToken(); 594 } 595 if (!";".equals(token)) { 596 throw new ApiParseException("expected ; found " + token, tokenizer); 597 } 598 TextFieldItem field = new TextFieldItem(api, name, cl, modifiers, typeInfo, value, tokenizer.pos()); 599 field.setDeprecated(modifiers.isDeprecated()); 600 if (isEnum) { 601 cl.addEnumConstant(field); 602 } else { 603 cl.addField(field); 604 } 605 } 606 parseModifiers( TextCodebase api, Tokenizer tokenizer, String token, List<String> annotations)607 private static TextModifiers parseModifiers( 608 TextCodebase api, 609 Tokenizer tokenizer, 610 String token, 611 List<String> annotations) throws ApiParseException { 612 613 TextModifiers modifiers = new TextModifiers(api, DefaultModifierList.PACKAGE_PRIVATE, null); 614 615 processModifiers: 616 while (true) { 617 switch (token) { 618 case "public": 619 modifiers.setVisibilityLevel(VisibilityLevel.PUBLIC); 620 token = tokenizer.requireToken(); 621 break; 622 case "protected": 623 modifiers.setVisibilityLevel(VisibilityLevel.PROTECTED); 624 token = tokenizer.requireToken(); 625 break; 626 case "private": 627 modifiers.setVisibilityLevel(VisibilityLevel.PRIVATE); 628 token = tokenizer.requireToken(); 629 break; 630 case "internal": 631 modifiers.setVisibilityLevel(VisibilityLevel.INTERNAL); 632 token = tokenizer.requireToken(); 633 break; 634 case "static": 635 modifiers.setStatic(true); 636 token = tokenizer.requireToken(); 637 break; 638 case "final": 639 modifiers.setFinal(true); 640 token = tokenizer.requireToken(); 641 break; 642 case "deprecated": 643 modifiers.setDeprecated(true); 644 token = tokenizer.requireToken(); 645 break; 646 case "abstract": 647 modifiers.setAbstract(true); 648 token = tokenizer.requireToken(); 649 break; 650 case "transient": 651 modifiers.setTransient(true); 652 token = tokenizer.requireToken(); 653 break; 654 case "volatile": 655 modifiers.setVolatile(true); 656 token = tokenizer.requireToken(); 657 break; 658 case "sealed": 659 modifiers.setSealed(true); 660 token = tokenizer.requireToken(); 661 break; 662 case "default": 663 modifiers.setDefault(true); 664 token = tokenizer.requireToken(); 665 break; 666 case "synchronized": 667 modifiers.setSynchronized(true); 668 token = tokenizer.requireToken(); 669 break; 670 case "native": 671 modifiers.setNative(true); 672 token = tokenizer.requireToken(); 673 break; 674 case "strictfp": 675 modifiers.setStrictFp(true); 676 token = tokenizer.requireToken(); 677 break; 678 case "infix": 679 modifiers.setInfix(true); 680 token = tokenizer.requireToken(); 681 break; 682 case "operator": 683 modifiers.setOperator(true); 684 token = tokenizer.requireToken(); 685 break; 686 case "inline": 687 modifiers.setInline(true); 688 token = tokenizer.requireToken(); 689 break; 690 case "suspend": 691 modifiers.setSuspend(true); 692 token = tokenizer.requireToken(); 693 break; 694 case "vararg": 695 modifiers.setVarArg(true); 696 token = tokenizer.requireToken(); 697 break; 698 default: 699 break processModifiers; 700 } 701 } 702 703 if (annotations != null) { 704 modifiers.addAnnotations(annotations); 705 } 706 707 return modifiers; 708 } 709 parseValue(String type, String val)710 private static Object parseValue(String type, String val) { 711 if (val != null) { 712 switch (type) { 713 case "boolean": 714 return "true".equals(val) ? Boolean.TRUE : Boolean.FALSE; 715 case "byte": 716 return Integer.valueOf(val); 717 case "short": 718 return Integer.valueOf(val); 719 case "int": 720 return Integer.valueOf(val); 721 case "long": 722 return Long.valueOf(val.substring(0, val.length() - 1)); 723 case "float": 724 switch (val) { 725 case "(1.0f/0.0f)": 726 case "(1.0f / 0.0f)": 727 return Float.POSITIVE_INFINITY; 728 case "(-1.0f/0.0f)": 729 case "(-1.0f / 0.0f)": 730 return Float.NEGATIVE_INFINITY; 731 case "(0.0f/0.0f)": 732 case "(0.0f / 0.0f)": 733 return Float.NaN; 734 default: 735 return Float.valueOf(val); 736 } 737 case "double": 738 switch (val) { 739 case "(1.0/0.0)": 740 case "(1.0 / 0.0)": 741 return Double.POSITIVE_INFINITY; 742 case "(-1.0/0.0)": 743 case "(-1.0 / 0.0)": 744 return Double.NEGATIVE_INFINITY; 745 case "(0.0/0.0)": 746 case "(0.0 / 0.0)": 747 return Double.NaN; 748 default: 749 return Double.valueOf(val); 750 } 751 case "char": 752 return (char) Integer.parseInt(val); 753 case JAVA_LANG_STRING: 754 case "String": 755 if ("null".equals(val)) { 756 return null; 757 } else { 758 return javaUnescapeString(val.substring(1, val.length() - 1)); 759 } 760 case "null": 761 return null; 762 default: 763 return val; 764 } 765 } 766 return null; 767 } 768 parseProperty(TextCodebase api, Tokenizer tokenizer, TextClassItem cl, String token)769 private static void parseProperty(TextCodebase api, Tokenizer tokenizer, TextClassItem cl, String token) 770 throws ApiParseException { 771 String type; 772 String name; 773 774 // Metalava: including annotations in file now 775 List<String> annotations = getAnnotations(tokenizer, token); 776 token = tokenizer.getCurrent(); 777 778 TextModifiers modifiers = parseModifiers(api, tokenizer, token, null); 779 token = tokenizer.getCurrent(); 780 assertIdent(tokenizer, token); 781 782 Pair<String, List<String>> kotlinTypeSuffix = processKotlinTypeSuffix(api, token, annotations); 783 token = kotlinTypeSuffix.getFirst(); 784 annotations = kotlinTypeSuffix.getSecond(); 785 modifiers.addAnnotations(annotations); 786 type = token; 787 TextTypeItem typeInfo = api.obtainTypeFromString(type); 788 789 token = tokenizer.requireToken(); 790 assertIdent(tokenizer, token); 791 name = token; 792 token = tokenizer.requireToken(); 793 if (!";".equals(token)) { 794 throw new ApiParseException("expected ; found " + token, tokenizer); 795 } 796 797 TextPropertyItem property = new TextPropertyItem(api, name, cl, modifiers, typeInfo, tokenizer.pos()); 798 property.setDeprecated(modifiers.isDeprecated()); 799 cl.addProperty(property); 800 } 801 parseTypeParameterList(TextCodebase codebase, Tokenizer tokenizer)802 private static TypeParameterList parseTypeParameterList(TextCodebase codebase, Tokenizer tokenizer) throws ApiParseException { 803 String token; 804 805 int start = tokenizer.offset() - 1; 806 int balance = 1; 807 while (balance > 0) { 808 token = tokenizer.requireToken(); 809 if (token.equals("<")) { 810 balance++; 811 } else if (token.equals(">")) { 812 balance--; 813 } 814 } 815 816 String typeParameterList = tokenizer.getStringFromOffset(start); 817 if (typeParameterList.isEmpty()) { 818 return TypeParameterList.Companion.getNONE(); 819 } else { 820 return TextTypeParameterList.Companion.create(codebase, null, typeParameterList); 821 } 822 } 823 parseParameterList(TextCodebase api, Tokenizer tokenizer, TextMethodItem method)824 private static void parseParameterList(TextCodebase api, Tokenizer tokenizer, TextMethodItem method) 825 throws ApiParseException { 826 String token = tokenizer.requireToken(); 827 int index = 0; 828 while (true) { 829 if (")".equals(token)) { 830 return; 831 } 832 833 // Each item can be 834 // annotations optional-modifiers type-with-use-annotations-and-generics optional-name optional-equals-default-value 835 836 // Metalava: including annotations in file now 837 List<String> annotations = getAnnotations(tokenizer, token); 838 token = tokenizer.getCurrent(); 839 840 TextModifiers modifiers = parseModifiers(api, tokenizer, token, null); 841 token = tokenizer.getCurrent(); 842 843 // Token should now represent the type 844 String type = token; 845 token = tokenizer.requireToken(); 846 if (token.startsWith("@")) { 847 // Type use annotations within the type, which broke up the tokenizer; 848 // put it back together 849 type += " " + token; 850 token = tokenizer.requireToken(); 851 if (token.startsWith("[")) { // TODO: This isn't general purpose; make requireToken smarter! 852 type += " " + token; 853 token = tokenizer.requireToken(); 854 } 855 } 856 857 Pair<String, List<String>> kotlinTypeSuffix = processKotlinTypeSuffix(api, type, annotations); 858 String typeString = kotlinTypeSuffix.getFirst(); 859 annotations = kotlinTypeSuffix.getSecond(); 860 modifiers.addAnnotations(annotations); 861 if (typeString.endsWith("...")) { 862 modifiers.setVarArg(true); 863 } 864 TextTypeItem typeInfo = api.obtainTypeFromString(typeString, 865 (TextClassItem) method.containingClass(), 866 method.typeParameterList()); 867 868 String name; 869 String publicName; 870 if (isIdent(token) && !token.equals("=")) { 871 name = token; 872 publicName = name; 873 token = tokenizer.requireToken(); 874 } else { 875 name = "arg" + (index + 1); 876 publicName = null; 877 } 878 879 String defaultValue = TextParameterItemKt.NO_DEFAULT_VALUE; 880 if ("=".equals(token)) { 881 defaultValue = tokenizer.requireToken(true); 882 StringBuilder sb = new StringBuilder(defaultValue); 883 if (defaultValue.equals("{")) { 884 int balance = 1; 885 while (balance > 0) { 886 token = tokenizer.requireToken(false, false); 887 sb.append(token); 888 if (token.equals("{")) { 889 balance++; 890 } else if (token.equals("}")) { 891 balance--; 892 if (balance == 0) { 893 break; 894 } 895 } 896 } 897 token = tokenizer.requireToken(); 898 } else { 899 int balance = defaultValue.equals("(") ? 1 : 0; 900 while (true) { 901 token = tokenizer.requireToken(true, false); 902 if ((token.endsWith(",") || token.endsWith(")")) && balance <= 0) { 903 if (token.length() > 1) { 904 sb.append(token, 0, token.length() - 1); 905 token = Character.toString(token.charAt(token.length() - 1)); 906 } 907 break; 908 } 909 sb.append(token); 910 if (token.equals("(")) { 911 balance++; 912 } else if (token.equals(")")) { 913 balance--; 914 } 915 } 916 } 917 defaultValue = sb.toString(); 918 } 919 920 if (",".equals(token)) { 921 token = tokenizer.requireToken(); 922 } else if (")".equals(token)) { 923 } else { 924 throw new ApiParseException("expected , or ), found " + token, tokenizer); 925 } 926 927 method.addParameter(new TextParameterItem(api, method, name, publicName, defaultValue, index, 928 typeInfo, modifiers, tokenizer.pos())); 929 if (modifiers.isVarArg()) { 930 method.setVarargs(true); 931 } 932 index++; 933 } 934 } 935 parseDefault(Tokenizer tokenizer, TextMethodItem method)936 private static String parseDefault(Tokenizer tokenizer, TextMethodItem method) 937 throws ApiParseException { 938 StringBuilder sb = new StringBuilder(); 939 while (true) { 940 String token = tokenizer.requireToken(); 941 if (";".equals(token)) { 942 method.setAnnotationDefault(sb.toString()); 943 return token; 944 } else { 945 sb.append(token); 946 } 947 } 948 } 949 parseThrows(Tokenizer tokenizer, TextMethodItem method)950 private static String parseThrows(Tokenizer tokenizer, TextMethodItem method) 951 throws ApiParseException { 952 String token = tokenizer.requireToken(); 953 boolean comma = true; 954 while (true) { 955 if (";".equals(token)) { 956 return token; 957 } else if (",".equals(token)) { 958 if (comma) { 959 throw new ApiParseException("Expected exception, got ','", tokenizer); 960 } 961 comma = true; 962 } else { 963 if (!comma) { 964 throw new ApiParseException("Expected ',' or ';' got " + token, tokenizer); 965 } 966 comma = false; 967 method.addException(token); 968 } 969 token = tokenizer.requireToken(); 970 } 971 } 972 qualifiedName(String pkg, String className)973 private static String qualifiedName(String pkg, String className) { 974 return pkg + "." + className; 975 } 976 isIdent(String token)977 private static boolean isIdent(String token) { 978 return isIdent(token.charAt(0)); 979 } 980 assertIdent(Tokenizer tokenizer, String token)981 private static void assertIdent(Tokenizer tokenizer, String token) throws ApiParseException { 982 if (!isIdent(token.charAt(0))) { 983 throw new ApiParseException("Expected identifier: " + token, tokenizer); 984 } 985 } 986 987 static class Tokenizer { 988 final char[] mBuf; 989 final String mFilename; 990 int mPos; 991 int mLine = 1; 992 Tokenizer(String filename, char[] buf)993 Tokenizer(String filename, char[] buf) { 994 mFilename = filename; 995 mBuf = buf; 996 } 997 pos()998 SourcePositionInfo pos() { 999 return new SourcePositionInfo(mFilename, mLine); 1000 } 1001 getLine()1002 public int getLine() { 1003 return mLine; 1004 } 1005 eatWhitespace()1006 boolean eatWhitespace() { 1007 boolean ate = false; 1008 while (mPos < mBuf.length && isSpace(mBuf[mPos])) { 1009 if (mBuf[mPos] == '\n') { 1010 mLine++; 1011 } 1012 mPos++; 1013 ate = true; 1014 } 1015 return ate; 1016 } 1017 eatComment()1018 boolean eatComment() { 1019 if (mPos + 1 < mBuf.length) { 1020 if (mBuf[mPos] == '/' && mBuf[mPos + 1] == '/') { 1021 mPos += 2; 1022 while (mPos < mBuf.length && !isNewline(mBuf[mPos])) { 1023 mPos++; 1024 } 1025 return true; 1026 } 1027 } 1028 return false; 1029 } 1030 eatWhitespaceAndComments()1031 void eatWhitespaceAndComments() { 1032 while (eatWhitespace() || eatComment()) { 1033 } 1034 } 1035 requireToken()1036 String requireToken() throws ApiParseException { 1037 return requireToken(true); 1038 } 1039 requireToken(boolean parenIsSep)1040 String requireToken(boolean parenIsSep) throws ApiParseException { 1041 return requireToken(parenIsSep, true); 1042 } 1043 requireToken(boolean parenIsSep, boolean eatWhitespace)1044 String requireToken(boolean parenIsSep, boolean eatWhitespace) throws ApiParseException { 1045 final String token = getToken(parenIsSep, eatWhitespace); 1046 if (token != null) { 1047 return token; 1048 } else { 1049 throw new ApiParseException("Unexpected end of file", this); 1050 } 1051 } 1052 getToken()1053 String getToken() throws ApiParseException { 1054 return getToken(true); 1055 } 1056 offset()1057 int offset() { 1058 return mPos; 1059 } 1060 getStringFromOffset(int offset)1061 String getStringFromOffset(int offset) { 1062 return new String(mBuf, offset, mPos - offset); 1063 } 1064 getToken(boolean parenIsSep)1065 String getToken(boolean parenIsSep) throws ApiParseException { 1066 return getToken(parenIsSep, true); 1067 } 1068 getCurrent()1069 String getCurrent() { 1070 return mCurrent; 1071 } 1072 1073 private String mCurrent = null; 1074 getToken(boolean parenIsSep, boolean eatWhitespace)1075 String getToken(boolean parenIsSep, boolean eatWhitespace) throws ApiParseException { 1076 if (eatWhitespace) { 1077 eatWhitespaceAndComments(); 1078 } 1079 if (mPos >= mBuf.length) { 1080 return null; 1081 } 1082 final int line = mLine; 1083 final char c = mBuf[mPos]; 1084 final int start = mPos; 1085 mPos++; 1086 if (c == '"') { 1087 final int STATE_BEGIN = 0; 1088 final int STATE_ESCAPE = 1; 1089 int state = STATE_BEGIN; 1090 while (true) { 1091 if (mPos >= mBuf.length) { 1092 throw new ApiParseException("Unexpected end of file for \" starting at " + line, this); 1093 } 1094 final char k = mBuf[mPos]; 1095 if (k == '\n' || k == '\r') { 1096 throw new ApiParseException("Unexpected newline for \" starting at " + line +" in " + mFilename, this); 1097 } 1098 mPos++; 1099 switch (state) { 1100 case STATE_BEGIN: 1101 switch (k) { 1102 case '\\': 1103 state = STATE_ESCAPE; 1104 break; 1105 case '"': 1106 mCurrent = new String(mBuf, start, mPos - start); 1107 return mCurrent; 1108 } 1109 break; 1110 case STATE_ESCAPE: 1111 state = STATE_BEGIN; 1112 break; 1113 } 1114 } 1115 } else if (isSeparator(c, parenIsSep)) { 1116 mCurrent = Character.toString(c); 1117 return mCurrent; 1118 } else { 1119 int genericDepth = 0; 1120 do { 1121 while (mPos < mBuf.length) { 1122 char d = mBuf[mPos]; 1123 if (isSpace(d) || isSeparator(d, parenIsSep)) { 1124 break; 1125 } else if (d == '"') { 1126 // String literal in token: skip the full thing 1127 mPos++; 1128 while (mPos < mBuf.length) { 1129 if (mBuf[mPos] == '"') { 1130 mPos++; 1131 break; 1132 } else if (mBuf[mPos] == '\\') { 1133 mPos++; 1134 } 1135 mPos++; 1136 } 1137 continue; 1138 } 1139 mPos++; 1140 } 1141 if (mPos < mBuf.length) { 1142 if (mBuf[mPos] == '<') { 1143 genericDepth++; 1144 mPos++; 1145 } else if (genericDepth != 0) { 1146 if (mBuf[mPos] == '>') { 1147 genericDepth--; 1148 } 1149 mPos++; 1150 } 1151 } 1152 } while (mPos < mBuf.length 1153 && ((!isSpace(mBuf[mPos]) && !isSeparator(mBuf[mPos], parenIsSep)) || genericDepth != 0)); 1154 if (mPos >= mBuf.length) { 1155 throw new ApiParseException("Unexpected end of file for \" starting at " + line, this); 1156 } 1157 mCurrent = new String(mBuf, start, mPos - start); 1158 return mCurrent; 1159 } 1160 } 1161 1162 @Nullable getFileName()1163 public String getFileName() { 1164 return mFilename; 1165 } 1166 } 1167 isSpace(char c)1168 private static boolean isSpace(char c) { 1169 return c == ' ' || c == '\t' || c == '\n' || c == '\r'; 1170 } 1171 isNewline(char c)1172 private static boolean isNewline(char c) { 1173 return c == '\n' || c == '\r'; 1174 } 1175 isSeparator(char c, boolean parenIsSep)1176 private static boolean isSeparator(char c, boolean parenIsSep) { 1177 if (parenIsSep) { 1178 if (c == '(' || c == ')') { 1179 return true; 1180 } 1181 } 1182 return c == '{' || c == '}' || c == ',' || c == ';' || c == '<' || c == '>'; 1183 } 1184 isIdent(char c)1185 private static boolean isIdent(char c) { 1186 return c != '"' && !isSeparator(c, true); 1187 } 1188 } 1189