1 /* 2 * Copyright 2016 Google Inc. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.google.turbine.main; 18 19 import static com.google.common.base.StandardSystemProperty.JAVA_SPECIFICATION_VERSION; 20 import static java.nio.charset.StandardCharsets.UTF_8; 21 import static java.util.Objects.requireNonNull; 22 23 import com.google.auto.value.AutoValue; 24 import com.google.common.collect.ImmutableList; 25 import com.google.common.collect.ImmutableMap; 26 import com.google.common.hash.Hashing; 27 import com.google.common.io.MoreFiles; 28 import com.google.errorprone.annotations.CanIgnoreReturnValue; 29 import com.google.turbine.binder.Binder; 30 import com.google.turbine.binder.Binder.BindingResult; 31 import com.google.turbine.binder.Binder.Statistics; 32 import com.google.turbine.binder.ClassPath; 33 import com.google.turbine.binder.ClassPathBinder; 34 import com.google.turbine.binder.CtSymClassBinder; 35 import com.google.turbine.binder.JimageClassBinder; 36 import com.google.turbine.binder.Processing; 37 import com.google.turbine.binder.bound.SourceTypeBoundClass; 38 import com.google.turbine.binder.sym.ClassSymbol; 39 import com.google.turbine.deps.Dependencies; 40 import com.google.turbine.deps.Transitive; 41 import com.google.turbine.diag.SourceFile; 42 import com.google.turbine.diag.TurbineError; 43 import com.google.turbine.lower.Lower; 44 import com.google.turbine.lower.Lower.Lowered; 45 import com.google.turbine.options.TurbineOptions; 46 import com.google.turbine.options.TurbineOptions.ReducedClasspathMode; 47 import com.google.turbine.options.TurbineOptionsParser; 48 import com.google.turbine.parse.Parser; 49 import com.google.turbine.proto.DepsProto; 50 import com.google.turbine.proto.ManifestProto; 51 import com.google.turbine.proto.ManifestProto.CompilationUnit; 52 import com.google.turbine.tree.Tree.CompUnit; 53 import com.google.turbine.zip.Zip; 54 import java.io.BufferedOutputStream; 55 import java.io.ByteArrayOutputStream; 56 import java.io.IOException; 57 import java.io.OutputStream; 58 import java.nio.file.Files; 59 import java.nio.file.Path; 60 import java.nio.file.Paths; 61 import java.time.LocalDateTime; 62 import java.util.Arrays; 63 import java.util.Collection; 64 import java.util.Map; 65 import java.util.Optional; 66 import java.util.OptionalInt; 67 import java.util.jar.Attributes; 68 import java.util.jar.JarEntry; 69 import java.util.jar.JarFile; 70 import java.util.jar.JarOutputStream; 71 import java.util.jar.Manifest; 72 import java.util.zip.ZipEntry; 73 74 /** Main entry point for the turbine CLI. */ 75 public final class Main { 76 77 private static final int BUFFER_SIZE = 65536; 78 79 // These attributes are used by JavaBuilder, Turbine, and ijar. 80 // They must all be kept in sync. 81 static final String MANIFEST_DIR = "META-INF/"; 82 static final String MANIFEST_NAME = JarFile.MANIFEST_NAME; 83 static final Attributes.Name TARGET_LABEL = new Attributes.Name("Target-Label"); 84 static final Attributes.Name INJECTING_RULE_KIND = new Attributes.Name("Injecting-Rule-Kind"); 85 main(String[] args)86 public static void main(String[] args) { 87 boolean ok; 88 try { 89 compile(args); 90 ok = true; 91 } catch (TurbineError | UsageException e) { 92 System.err.println(e.getMessage()); 93 ok = false; 94 } catch (Throwable turbineCrash) { 95 turbineCrash.printStackTrace(); 96 ok = false; 97 } 98 System.exit(ok ? 0 : 1); 99 } 100 101 /** The result of a turbine invocation. */ 102 @AutoValue 103 public abstract static class Result { 104 /** Returns {@code true} if transitive classpath fallback occurred. */ transitiveClasspathFallback()105 public abstract boolean transitiveClasspathFallback(); 106 107 /** The length of the transitive classpath. */ transitiveClasspathLength()108 public abstract int transitiveClasspathLength(); 109 110 /** 111 * The length of the reduced classpath, or {@link #transitiveClasspathLength} if classpath 112 * reduction is not supported. 113 */ reducedClasspathLength()114 public abstract int reducedClasspathLength(); 115 processorStatistics()116 public abstract Statistics processorStatistics(); 117 create( boolean transitiveClasspathFallback, int transitiveClasspathLength, int reducedClasspathLength, Statistics processorStatistics)118 static Result create( 119 boolean transitiveClasspathFallback, 120 int transitiveClasspathLength, 121 int reducedClasspathLength, 122 Statistics processorStatistics) { 123 return new AutoValue_Main_Result( 124 transitiveClasspathFallback, 125 transitiveClasspathLength, 126 reducedClasspathLength, 127 processorStatistics); 128 } 129 } 130 131 @CanIgnoreReturnValue compile(String[] args)132 public static Result compile(String[] args) throws IOException { 133 return compile(TurbineOptionsParser.parse(Arrays.asList(args))); 134 } 135 136 @CanIgnoreReturnValue compile(TurbineOptions options)137 public static Result compile(TurbineOptions options) throws IOException { 138 usage(options); 139 140 ImmutableList<CompUnit> units = parseAll(options); 141 142 ClassPath bootclasspath = bootclasspath(options); 143 144 BindingResult bound; 145 ReducedClasspathMode reducedClasspathMode = options.reducedClasspathMode(); 146 if (reducedClasspathMode == ReducedClasspathMode.JAVABUILDER_REDUCED 147 && options.directJars().isEmpty()) { 148 // the compilation doesn't support reduced classpaths 149 // TODO(cushon): make this a usage error, see TODO in Dependencies.reduceClasspath 150 reducedClasspathMode = ReducedClasspathMode.NONE; 151 } 152 boolean transitiveClasspathFallback = false; 153 ImmutableList<String> classPath = options.classPath(); 154 int transitiveClasspathLength = classPath.size(); 155 int reducedClasspathLength = classPath.size(); 156 switch (reducedClasspathMode) { 157 case NONE: 158 bound = bind(options, units, bootclasspath, classPath); 159 break; 160 case BAZEL_FALLBACK: 161 reducedClasspathLength = options.reducedClasspathLength(); 162 bound = bind(options, units, bootclasspath, classPath); 163 transitiveClasspathFallback = true; 164 break; 165 case JAVABUILDER_REDUCED: 166 Collection<String> reducedClasspath = 167 Dependencies.reduceClasspath(classPath, options.directJars(), options.depsArtifacts()); 168 reducedClasspathLength = reducedClasspath.size(); 169 try { 170 bound = bind(options, units, bootclasspath, reducedClasspath); 171 } catch (TurbineError e) { 172 bound = fallback(options, units, bootclasspath, classPath); 173 transitiveClasspathFallback = true; 174 } 175 break; 176 case BAZEL_REDUCED: 177 transitiveClasspathLength = options.fullClasspathLength(); 178 try { 179 bound = bind(options, units, bootclasspath, classPath); 180 } catch (TurbineError e) { 181 writeJdepsForFallback(options); 182 return Result.create( 183 /* transitiveClasspathFallback= */ true, 184 /* transitiveClasspathLength= */ transitiveClasspathLength, 185 /* reducedClasspathLength= */ reducedClasspathLength, 186 Statistics.empty()); 187 } 188 break; 189 default: 190 throw new AssertionError(reducedClasspathMode); 191 } 192 193 if (options.outputDeps().isPresent() 194 || options.output().isPresent() 195 || options.outputManifest().isPresent()) { 196 // TODO(cushon): parallelize 197 Lowered lowered = 198 Lower.lowerAll( 199 Lower.LowerOptions.builder() 200 .languageVersion(options.languageVersion()) 201 .emitPrivateFields(options.javacOpts().contains("-XDturbine.emitPrivateFields")) 202 .build(), 203 bound.units(), 204 bound.modules(), 205 bound.classPathEnv()); 206 207 if (options.outputDeps().isPresent()) { 208 DepsProto.Dependencies deps = 209 Dependencies.collectDeps(options.targetLabel(), bootclasspath, bound, lowered); 210 Path path = Paths.get(options.outputDeps().get()); 211 /* 212 * TODO: cpovirk - Consider checking outputDeps for validity earlier so that anyone who 213 * `--output_deps=/` or similar will get a proper error instead of NPE. 214 */ 215 Files.createDirectories(requireNonNull(path.getParent())); 216 try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) { 217 deps.writeTo(os); 218 } 219 } 220 if (options.output().isPresent()) { 221 ImmutableMap<String, byte[]> transitive = Transitive.collectDeps(bootclasspath, bound); 222 writeOutput(options, bound.generatedClasses(), lowered.bytes(), transitive); 223 } 224 if (options.outputManifest().isPresent()) { 225 writeManifestProto(options, bound.units(), bound.generatedSources()); 226 } 227 } 228 229 writeSources(options, bound.generatedSources()); 230 writeResources(options, bound.generatedClasses()); 231 return Result.create( 232 /* transitiveClasspathFallback= */ transitiveClasspathFallback, 233 /* transitiveClasspathLength= */ transitiveClasspathLength, 234 /* reducedClasspathLength= */ reducedClasspathLength, 235 bound.statistics()); 236 } 237 238 // don't inline this; we want it to show up in profiles fallback( TurbineOptions options, ImmutableList<CompUnit> units, ClassPath bootclasspath, ImmutableList<String> classPath)239 private static BindingResult fallback( 240 TurbineOptions options, 241 ImmutableList<CompUnit> units, 242 ClassPath bootclasspath, 243 ImmutableList<String> classPath) 244 throws IOException { 245 return bind(options, units, bootclasspath, classPath); 246 } 247 248 /** 249 * Writes a jdeps proto that indiciates to Blaze that the transitive classpath compilation failed, 250 * and it should fall back to the transitive classpath. Used only when {@link 251 * ReducedClasspathMode#BAZEL_REDUCED}. 252 */ writeJdepsForFallback(TurbineOptions options)253 public static void writeJdepsForFallback(TurbineOptions options) throws IOException { 254 try (OutputStream os = 255 new BufferedOutputStream(Files.newOutputStream(Paths.get(options.outputDeps().get())))) { 256 DepsProto.Dependencies.newBuilder() 257 .setRuleLabel(options.targetLabel().get()) 258 .setRequiresReducedClasspathFallback(true) 259 .build() 260 .writeTo(os); 261 } 262 } 263 bind( TurbineOptions options, ImmutableList<CompUnit> units, ClassPath bootclasspath, Collection<String> classpath)264 private static BindingResult bind( 265 TurbineOptions options, 266 ImmutableList<CompUnit> units, 267 ClassPath bootclasspath, 268 Collection<String> classpath) 269 throws IOException { 270 return Binder.bind( 271 units, 272 ClassPathBinder.bindClasspath(toPaths(classpath)), 273 Processing.initializeProcessors( 274 /* sourceVersion= */ options.languageVersion().sourceVersion(), 275 /* javacopts= */ options.javacOpts(), 276 /* processorNames= */ options.processors(), 277 Processing.processorLoader( 278 /* processorPath= */ options.processorPath(), 279 /* builtinProcessors= */ options.builtinProcessors())), 280 bootclasspath, 281 /* moduleVersion= */ Optional.empty()); 282 } 283 usage(TurbineOptions options)284 private static void usage(TurbineOptions options) { 285 if (options.help()) { 286 throw new UsageException(); 287 } 288 if (!options.output().isPresent() 289 && !options.gensrcOutput().isPresent() 290 && !options.resourceOutput().isPresent()) { 291 throw new UsageException( 292 "at least one of --output, --gensrc_output, or --resource_output is required"); 293 } 294 } 295 bootclasspath(TurbineOptions options)296 private static ClassPath bootclasspath(TurbineOptions options) throws IOException { 297 // if both --release and --bootclasspath are specified, --release wins 298 OptionalInt release = options.languageVersion().release(); 299 if (release.isPresent() && options.system().isPresent()) { 300 throw new UsageException("expected at most one of --release and --system"); 301 } 302 303 if (release.isPresent()) { 304 return release(release.getAsInt()); 305 } 306 307 if (options.system().isPresent()) { 308 // look for a jimage in the given JDK 309 return JimageClassBinder.bind(options.system().get()); 310 } 311 312 // the bootclasspath might be empty, e.g. when compiling java.lang 313 return ClassPathBinder.bindClasspath(toPaths(options.bootClassPath())); 314 } 315 release(int release)316 private static ClassPath release(int release) throws IOException { 317 // Search ct.sym for a matching release 318 ClassPath bootclasspath = CtSymClassBinder.bind(release); 319 if (bootclasspath != null) { 320 return bootclasspath; 321 } 322 if (release == Integer.parseInt(JAVA_SPECIFICATION_VERSION.value())) { 323 // if --release matches the host JDK, use its jimage 324 return JimageClassBinder.bindDefault(); 325 } 326 throw new UsageException("not a supported release: " + release); 327 } 328 329 /** Parse all source files and source jars. */ 330 // TODO(cushon): parallelize parseAll(TurbineOptions options)331 private static ImmutableList<CompUnit> parseAll(TurbineOptions options) throws IOException { 332 return parseAll(options.sources(), options.sourceJars()); 333 } 334 parseAll(Iterable<String> sources, Iterable<String> sourceJars)335 static ImmutableList<CompUnit> parseAll(Iterable<String> sources, Iterable<String> sourceJars) 336 throws IOException { 337 ImmutableList.Builder<CompUnit> units = ImmutableList.builder(); 338 for (String source : sources) { 339 Path path = Paths.get(source); 340 units.add(Parser.parse(new SourceFile(source, MoreFiles.asCharSource(path, UTF_8).read()))); 341 } 342 for (String sourceJar : sourceJars) { 343 try (Zip.ZipIterable iterable = new Zip.ZipIterable(Paths.get(sourceJar))) { 344 for (Zip.Entry ze : iterable) { 345 if (ze.name().endsWith(".java")) { 346 String name = ze.name(); 347 String source = new String(ze.data(), UTF_8); 348 units.add(Parser.parse(new SourceFile(name, source))); 349 } 350 } 351 } 352 } 353 return units.build(); 354 } 355 356 /** Writes source files generated by annotation processors. */ writeSources( TurbineOptions options, ImmutableMap<String, SourceFile> generatedSources)357 private static void writeSources( 358 TurbineOptions options, ImmutableMap<String, SourceFile> generatedSources) 359 throws IOException { 360 if (!options.gensrcOutput().isPresent()) { 361 return; 362 } 363 Path path = Paths.get(options.gensrcOutput().get()); 364 if (Files.isDirectory(path)) { 365 for (SourceFile source : generatedSources.values()) { 366 Path to = path.resolve(source.path()); 367 // TODO: cpovirk - Consider checking gensrcOutput, similar to outputDeps. 368 Files.createDirectories(requireNonNull(to.getParent())); 369 Files.writeString(to, source.source()); 370 } 371 return; 372 } 373 try (OutputStream os = Files.newOutputStream(path); 374 BufferedOutputStream bos = new BufferedOutputStream(os, BUFFER_SIZE); 375 JarOutputStream jos = new JarOutputStream(bos)) { 376 writeManifest(jos, manifest()); 377 for (SourceFile source : generatedSources.values()) { 378 addEntry(jos, source.path(), source.source().getBytes(UTF_8)); 379 } 380 } 381 } 382 383 /** Writes resource files generated by annotation processors. */ writeResources( TurbineOptions options, ImmutableMap<String, byte[]> generatedResources)384 private static void writeResources( 385 TurbineOptions options, ImmutableMap<String, byte[]> generatedResources) throws IOException { 386 if (!options.resourceOutput().isPresent()) { 387 return; 388 } 389 Path path = Paths.get(options.resourceOutput().get()); 390 if (Files.isDirectory(path)) { 391 for (Map.Entry<String, byte[]> resource : generatedResources.entrySet()) { 392 Path to = path.resolve(resource.getKey()); 393 // TODO: cpovirk - Consider checking resourceOutput, similar to outputDeps. 394 Files.createDirectories(requireNonNull(to.getParent())); 395 Files.write(to, resource.getValue()); 396 } 397 return; 398 } 399 try (OutputStream os = Files.newOutputStream(path); 400 BufferedOutputStream bos = new BufferedOutputStream(os, BUFFER_SIZE); 401 JarOutputStream jos = new JarOutputStream(bos)) { 402 for (Map.Entry<String, byte[]> resource : generatedResources.entrySet()) { 403 addEntry(jos, resource.getKey(), resource.getValue()); 404 } 405 } 406 } 407 408 /** Writes bytecode to the output jar. */ writeOutput( TurbineOptions options, Map<String, byte[]> generated, Map<String, byte[]> lowered, Map<String, byte[]> transitive)409 private static void writeOutput( 410 TurbineOptions options, 411 Map<String, byte[]> generated, 412 Map<String, byte[]> lowered, 413 Map<String, byte[]> transitive) 414 throws IOException { 415 Path path = Paths.get(options.output().get()); 416 try (OutputStream os = Files.newOutputStream(path); 417 BufferedOutputStream bos = new BufferedOutputStream(os, BUFFER_SIZE); 418 JarOutputStream jos = new JarOutputStream(bos)) { 419 if (options.targetLabel().isPresent()) { 420 writeManifest(jos, manifest(options)); 421 } 422 for (Map.Entry<String, byte[]> entry : transitive.entrySet()) { 423 addEntry( 424 jos, 425 ClassPathBinder.TRANSITIVE_PREFIX + entry.getKey() + ClassPathBinder.TRANSITIVE_SUFFIX, 426 entry.getValue()); 427 } 428 for (Map.Entry<String, byte[]> entry : lowered.entrySet()) { 429 addEntry(jos, entry.getKey() + ".class", entry.getValue()); 430 } 431 for (Map.Entry<String, byte[]> entry : generated.entrySet()) { 432 addEntry(jos, entry.getKey(), entry.getValue()); 433 } 434 } 435 } 436 writeManifestProto( TurbineOptions options, ImmutableMap<ClassSymbol, SourceTypeBoundClass> units, ImmutableMap<String, SourceFile> generatedSources)437 private static void writeManifestProto( 438 TurbineOptions options, 439 ImmutableMap<ClassSymbol, SourceTypeBoundClass> units, 440 ImmutableMap<String, SourceFile> generatedSources) 441 throws IOException { 442 ManifestProto.Manifest.Builder manifest = ManifestProto.Manifest.newBuilder(); 443 for (Map.Entry<ClassSymbol, SourceTypeBoundClass> e : units.entrySet()) { 444 manifest.addCompilationUnit( 445 CompilationUnit.newBuilder() 446 .setPath(e.getValue().source().path()) 447 .setPkg(e.getKey().packageName()) 448 .addTopLevel(e.getKey().simpleName()) 449 .setGeneratedByAnnotationProcessor( 450 generatedSources.containsKey(e.getValue().source().path())) 451 .build()); 452 } 453 try (OutputStream os = 454 new BufferedOutputStream( 455 Files.newOutputStream(Paths.get(options.outputManifest().get())))) { 456 manifest.build().writeTo(os); 457 } 458 } 459 460 /** Normalize timestamps. */ 461 static final LocalDateTime DEFAULT_TIMESTAMP = LocalDateTime.of(2010, 1, 1, 0, 0, 0); 462 addEntry(JarOutputStream jos, String name, byte[] bytes)463 private static void addEntry(JarOutputStream jos, String name, byte[] bytes) throws IOException { 464 JarEntry je = new JarEntry(name); 465 je.setTimeLocal(DEFAULT_TIMESTAMP); 466 je.setMethod(ZipEntry.STORED); 467 je.setSize(bytes.length); 468 je.setCrc(Hashing.crc32().hashBytes(bytes).padToLong()); 469 jos.putNextEntry(je); 470 jos.write(bytes); 471 } 472 writeManifest(JarOutputStream jos, Manifest manifest)473 private static void writeManifest(JarOutputStream jos, Manifest manifest) throws IOException { 474 addEntry(jos, MANIFEST_DIR, new byte[] {}); 475 ByteArrayOutputStream out = new ByteArrayOutputStream(); 476 manifest.write(out); 477 addEntry(jos, MANIFEST_NAME, out.toByteArray()); 478 } 479 480 /** Creates a default {@link Manifest}. */ manifest()481 private static Manifest manifest() { 482 Manifest manifest = new Manifest(); 483 Attributes attributes = manifest.getMainAttributes(); 484 attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); 485 Attributes.Name createdBy = new Attributes.Name("Created-By"); 486 if (attributes.getValue(createdBy) == null) { 487 attributes.put(createdBy, "bazel"); 488 } 489 return manifest; 490 } 491 492 /** Creates a {@link Manifest} that includes the target label and injecting rule kind. */ manifest(TurbineOptions turbineOptions)493 private static Manifest manifest(TurbineOptions turbineOptions) { 494 Manifest manifest = manifest(); 495 Attributes attributes = manifest.getMainAttributes(); 496 if (turbineOptions.targetLabel().isPresent()) { 497 attributes.put(TARGET_LABEL, turbineOptions.targetLabel().get()); 498 } 499 if (turbineOptions.injectingRuleKind().isPresent()) { 500 attributes.put(INJECTING_RULE_KIND, turbineOptions.injectingRuleKind().get()); 501 } 502 return manifest; 503 } 504 toPaths(Iterable<String> paths)505 private static ImmutableList<Path> toPaths(Iterable<String> paths) { 506 ImmutableList.Builder<Path> result = ImmutableList.builder(); 507 for (String path : paths) { 508 result.add(Paths.get(path)); 509 } 510 return result.build(); 511 } 512 Main()513 private Main() {} 514 } 515