1 /* 2 * XZOutputStream 3 * 4 * Author: Lasse Collin <lasse.collin@tukaani.org> 5 * 6 * This file has been put into the public domain. 7 * You can do whatever you want with this file. 8 */ 9 10 package org.tukaani.xz; 11 12 import java.io.OutputStream; 13 import java.io.IOException; 14 import org.tukaani.xz.common.EncoderUtil; 15 import org.tukaani.xz.common.StreamFlags; 16 import org.tukaani.xz.check.Check; 17 import org.tukaani.xz.index.IndexEncoder; 18 19 /** 20 * Compresses into the .xz file format. 21 * 22 * <h4>Examples</h4> 23 * <p> 24 * Getting an output stream to compress with LZMA2 using the default 25 * settings and the default integrity check type (CRC64): 26 * <p><blockquote><pre> 27 * FileOutputStream outfile = new FileOutputStream("foo.xz"); 28 * XZOutputStream outxz = new XZOutputStream(outfile, new LZMA2Options()); 29 * </pre></blockquote> 30 * <p> 31 * Using the preset level <code>8</code> for LZMA2 (the default 32 * is <code>6</code>) and SHA-256 instead of CRC64 for integrity checking: 33 * <p><blockquote><pre> 34 * XZOutputStream outxz = new XZOutputStream(outfile, new LZMA2Options(8), 35 * XZ.CHECK_SHA256); 36 * </pre></blockquote> 37 * <p> 38 * Using the x86 BCJ filter together with LZMA2 to compress x86 executables 39 * and printing the memory usage information before creating the 40 * XZOutputStream: 41 * <p><blockquote><pre> 42 * X86Options x86 = new X86Options(); 43 * LZMA2Options lzma2 = new LZMA2Options(); 44 * FilterOptions[] options = { x86, lzma2 }; 45 * System.out.println("Encoder memory usage: " 46 * + FilterOptions.getEncoderMemoryUsage(options) 47 * + " KiB"); 48 * System.out.println("Decoder memory usage: " 49 * + FilterOptions.getDecoderMemoryUsage(options) 50 * + " KiB"); 51 * XZOutputStream outxz = new XZOutputStream(outfile, options); 52 * </pre></blockquote> 53 */ 54 public class XZOutputStream extends FinishableOutputStream { 55 private final ArrayCache arrayCache; 56 57 private OutputStream out; 58 private final StreamFlags streamFlags = new StreamFlags(); 59 private final Check check; 60 private final IndexEncoder index = new IndexEncoder(); 61 62 private BlockOutputStream blockEncoder = null; 63 private FilterEncoder[] filters; 64 65 /** 66 * True if the current filter chain supports flushing. 67 * If it doesn't support flushing, <code>flush()</code> 68 * will use <code>endBlock()</code> as a fallback. 69 */ 70 private boolean filtersSupportFlushing; 71 72 private IOException exception = null; 73 private boolean finished = false; 74 75 private final byte[] tempBuf = new byte[1]; 76 77 /** 78 * Creates a new XZ compressor using one filter and CRC64 as 79 * the integrity check. This constructor is equivalent to passing 80 * a single-member FilterOptions array to 81 * <code>XZOutputStream(OutputStream, FilterOptions[])</code>. 82 * 83 * @param out output stream to which the compressed data 84 * will be written 85 * 86 * @param filterOptions 87 * filter options to use 88 * 89 * @throws UnsupportedOptionsException 90 * invalid filter chain 91 * 92 * @throws IOException may be thrown from <code>out</code> 93 */ XZOutputStream(OutputStream out, FilterOptions filterOptions)94 public XZOutputStream(OutputStream out, FilterOptions filterOptions) 95 throws IOException { 96 this(out, filterOptions, XZ.CHECK_CRC64); 97 } 98 99 /** 100 * Creates a new XZ compressor using one filter and CRC64 as 101 * the integrity check. This constructor is equivalent to passing 102 * a single-member FilterOptions array to 103 * <code>XZOutputStream(OutputStream, FilterOptions[], ArrayCache)</code>. 104 * 105 * @param out output stream to which the compressed data 106 * will be written 107 * 108 * @param filterOptions 109 * filter options to use 110 * 111 * @param arrayCache cache to be used for allocating large arrays 112 * 113 * @throws UnsupportedOptionsException 114 * invalid filter chain 115 * 116 * @throws IOException may be thrown from <code>out</code> 117 * 118 * @since 1.7 119 */ XZOutputStream(OutputStream out, FilterOptions filterOptions, ArrayCache arrayCache)120 public XZOutputStream(OutputStream out, FilterOptions filterOptions, 121 ArrayCache arrayCache) 122 throws IOException { 123 this(out, filterOptions, XZ.CHECK_CRC64, arrayCache); 124 } 125 126 /** 127 * Creates a new XZ compressor using one filter and the specified 128 * integrity check type. This constructor is equivalent to 129 * passing a single-member FilterOptions array to 130 * <code>XZOutputStream(OutputStream, FilterOptions[], int)</code>. 131 * 132 * @param out output stream to which the compressed data 133 * will be written 134 * 135 * @param filterOptions 136 * filter options to use 137 * 138 * @param checkType type of the integrity check, 139 * for example XZ.CHECK_CRC32 140 * 141 * @throws UnsupportedOptionsException 142 * invalid filter chain 143 * 144 * @throws IOException may be thrown from <code>out</code> 145 */ XZOutputStream(OutputStream out, FilterOptions filterOptions, int checkType)146 public XZOutputStream(OutputStream out, FilterOptions filterOptions, 147 int checkType) throws IOException { 148 this(out, new FilterOptions[] { filterOptions }, checkType); 149 } 150 151 /** 152 * Creates a new XZ compressor using one filter and the specified 153 * integrity check type. This constructor is equivalent to 154 * passing a single-member FilterOptions array to 155 * <code>XZOutputStream(OutputStream, FilterOptions[], int, 156 * ArrayCache)</code>. 157 * 158 * @param out output stream to which the compressed data 159 * will be written 160 * 161 * @param filterOptions 162 * filter options to use 163 * 164 * @param checkType type of the integrity check, 165 * for example XZ.CHECK_CRC32 166 * 167 * @param arrayCache cache to be used for allocating large arrays 168 * 169 * @throws UnsupportedOptionsException 170 * invalid filter chain 171 * 172 * @throws IOException may be thrown from <code>out</code> 173 * 174 * @since 1.7 175 */ XZOutputStream(OutputStream out, FilterOptions filterOptions, int checkType, ArrayCache arrayCache)176 public XZOutputStream(OutputStream out, FilterOptions filterOptions, 177 int checkType, ArrayCache arrayCache) 178 throws IOException { 179 this(out, new FilterOptions[] { filterOptions }, checkType, 180 arrayCache); 181 } 182 183 /** 184 * Creates a new XZ compressor using 1-4 filters and CRC64 as 185 * the integrity check. This constructor is equivalent 186 * <code>XZOutputStream(out, filterOptions, XZ.CHECK_CRC64)</code>. 187 * 188 * @param out output stream to which the compressed data 189 * will be written 190 * 191 * @param filterOptions 192 * array of filter options to use 193 * 194 * @throws UnsupportedOptionsException 195 * invalid filter chain 196 * 197 * @throws IOException may be thrown from <code>out</code> 198 */ XZOutputStream(OutputStream out, FilterOptions[] filterOptions)199 public XZOutputStream(OutputStream out, FilterOptions[] filterOptions) 200 throws IOException { 201 this(out, filterOptions, XZ.CHECK_CRC64); 202 } 203 204 /** 205 * Creates a new XZ compressor using 1-4 filters and CRC64 as 206 * the integrity check. This constructor is equivalent 207 * <code>XZOutputStream(out, filterOptions, XZ.CHECK_CRC64, 208 * arrayCache)</code>. 209 * 210 * @param out output stream to which the compressed data 211 * will be written 212 * 213 * @param filterOptions 214 * array of filter options to use 215 * 216 * @param arrayCache cache to be used for allocating large arrays 217 * 218 * @throws UnsupportedOptionsException 219 * invalid filter chain 220 * 221 * @throws IOException may be thrown from <code>out</code> 222 * 223 * @since 1.7 224 */ XZOutputStream(OutputStream out, FilterOptions[] filterOptions, ArrayCache arrayCache)225 public XZOutputStream(OutputStream out, FilterOptions[] filterOptions, 226 ArrayCache arrayCache) 227 throws IOException { 228 this(out, filterOptions, XZ.CHECK_CRC64, arrayCache); 229 } 230 231 /** 232 * Creates a new XZ compressor using 1-4 filters and the specified 233 * integrity check type. 234 * 235 * @param out output stream to which the compressed data 236 * will be written 237 * 238 * @param filterOptions 239 * array of filter options to use 240 * 241 * @param checkType type of the integrity check, 242 * for example XZ.CHECK_CRC32 243 * 244 * @throws UnsupportedOptionsException 245 * invalid filter chain 246 * 247 * @throws IOException may be thrown from <code>out</code> 248 */ XZOutputStream(OutputStream out, FilterOptions[] filterOptions, int checkType)249 public XZOutputStream(OutputStream out, FilterOptions[] filterOptions, 250 int checkType) throws IOException { 251 this(out, filterOptions, checkType, ArrayCache.getDefaultCache()); 252 } 253 254 /** 255 * Creates a new XZ compressor using 1-4 filters and the specified 256 * integrity check type. 257 * 258 * @param out output stream to which the compressed data 259 * will be written 260 * 261 * @param filterOptions 262 * array of filter options to use 263 * 264 * @param checkType type of the integrity check, 265 * for example XZ.CHECK_CRC32 266 * 267 * @param arrayCache cache to be used for allocating large arrays 268 * 269 * @throws UnsupportedOptionsException 270 * invalid filter chain 271 * 272 * @throws IOException may be thrown from <code>out</code> 273 * 274 * @since 1.7 275 */ XZOutputStream(OutputStream out, FilterOptions[] filterOptions, int checkType, ArrayCache arrayCache)276 public XZOutputStream(OutputStream out, FilterOptions[] filterOptions, 277 int checkType, ArrayCache arrayCache) 278 throws IOException { 279 this.arrayCache = arrayCache; 280 this.out = out; 281 updateFilters(filterOptions); 282 283 streamFlags.checkType = checkType; 284 check = Check.getInstance(checkType); 285 286 encodeStreamHeader(); 287 } 288 289 /** 290 * Updates the filter chain with a single filter. 291 * This is equivalent to passing a single-member FilterOptions array 292 * to <code>updateFilters(FilterOptions[])</code>. 293 * 294 * @param filterOptions 295 * new filter to use 296 * 297 * @throws UnsupportedOptionsException 298 * unsupported filter chain, or trying to change 299 * the filter chain in the middle of a Block 300 */ updateFilters(FilterOptions filterOptions)301 public void updateFilters(FilterOptions filterOptions) 302 throws XZIOException { 303 FilterOptions[] opts = new FilterOptions[1]; 304 opts[0] = filterOptions; 305 updateFilters(opts); 306 } 307 308 /** 309 * Updates the filter chain with 1-4 filters. 310 * <p> 311 * Currently this cannot be used to update e.g. LZMA2 options in the 312 * middle of a XZ Block. Use <code>endBlock()</code> to finish the 313 * current XZ Block before calling this function. The new filter chain 314 * will then be used for the next XZ Block. 315 * 316 * @param filterOptions 317 * new filter chain to use 318 * 319 * @throws UnsupportedOptionsException 320 * unsupported filter chain, or trying to change 321 * the filter chain in the middle of a Block 322 */ updateFilters(FilterOptions[] filterOptions)323 public void updateFilters(FilterOptions[] filterOptions) 324 throws XZIOException { 325 if (blockEncoder != null) 326 throw new UnsupportedOptionsException("Changing filter options " 327 + "in the middle of a XZ Block not implemented"); 328 329 if (filterOptions.length < 1 || filterOptions.length > 4) 330 throw new UnsupportedOptionsException( 331 "XZ filter chain must be 1-4 filters"); 332 333 filtersSupportFlushing = true; 334 FilterEncoder[] newFilters = new FilterEncoder[filterOptions.length]; 335 for (int i = 0; i < filterOptions.length; ++i) { 336 newFilters[i] = filterOptions[i].getFilterEncoder(); 337 filtersSupportFlushing &= newFilters[i].supportsFlushing(); 338 } 339 340 RawCoder.validate(newFilters); 341 filters = newFilters; 342 } 343 344 /** 345 * Writes one byte to be compressed. 346 * 347 * @throws XZIOException 348 * XZ Stream has grown too big 349 * 350 * @throws XZIOException 351 * <code>finish()</code> or <code>close()</code> 352 * was already called 353 * 354 * @throws IOException may be thrown by the underlying output stream 355 */ write(int b)356 public void write(int b) throws IOException { 357 tempBuf[0] = (byte)b; 358 write(tempBuf, 0, 1); 359 } 360 361 /** 362 * Writes an array of bytes to be compressed. 363 * The compressors tend to do internal buffering and thus the written 364 * data won't be readable from the compressed output immediately. 365 * Use <code>flush()</code> to force everything written so far to 366 * be written to the underlaying output stream, but be aware that 367 * flushing reduces compression ratio. 368 * 369 * @param buf buffer of bytes to be written 370 * @param off start offset in <code>buf</code> 371 * @param len number of bytes to write 372 * 373 * @throws XZIOException 374 * XZ Stream has grown too big: total file size 375 * about 8 EiB or the Index field exceeds 376 * 16 GiB; you shouldn't reach these sizes 377 * in practice 378 * 379 * @throws XZIOException 380 * <code>finish()</code> or <code>close()</code> 381 * was already called and len > 0 382 * 383 * @throws IOException may be thrown by the underlying output stream 384 */ write(byte[] buf, int off, int len)385 public void write(byte[] buf, int off, int len) throws IOException { 386 if (off < 0 || len < 0 || off + len < 0 || off + len > buf.length) 387 throw new IndexOutOfBoundsException(); 388 389 if (exception != null) 390 throw exception; 391 392 if (finished) 393 throw new XZIOException("Stream finished or closed"); 394 395 try { 396 if (blockEncoder == null) 397 blockEncoder = new BlockOutputStream(out, filters, check, 398 arrayCache); 399 400 blockEncoder.write(buf, off, len); 401 } catch (IOException e) { 402 exception = e; 403 throw e; 404 } 405 } 406 407 /** 408 * Finishes the current XZ Block (but not the whole XZ Stream). 409 * This doesn't flush the stream so it's possible that not all data will 410 * be decompressible from the output stream when this function returns. 411 * Call also <code>flush()</code> if flushing is wanted in addition to 412 * finishing the current XZ Block. 413 * <p> 414 * If there is no unfinished Block open, this function will do nothing. 415 * (No empty XZ Block will be created.) 416 * <p> 417 * This function can be useful, for example, to create 418 * random-accessible .xz files. 419 * <p> 420 * Starting a new XZ Block means that the encoder state is reset. 421 * Doing this very often will increase the size of the compressed 422 * file a lot (more than plain <code>flush()</code> would do). 423 * 424 * @throws XZIOException 425 * XZ Stream has grown too big 426 * 427 * @throws XZIOException 428 * stream finished or closed 429 * 430 * @throws IOException may be thrown by the underlying output stream 431 */ endBlock()432 public void endBlock() throws IOException { 433 if (exception != null) 434 throw exception; 435 436 if (finished) 437 throw new XZIOException("Stream finished or closed"); 438 439 // NOTE: Once there is threading with multiple Blocks, it's possible 440 // that this function will be more like a barrier that returns 441 // before the last Block has been finished. 442 if (blockEncoder != null) { 443 try { 444 blockEncoder.finish(); 445 index.add(blockEncoder.getUnpaddedSize(), 446 blockEncoder.getUncompressedSize()); 447 blockEncoder = null; 448 } catch (IOException e) { 449 exception = e; 450 throw e; 451 } 452 } 453 } 454 455 /** 456 * Flushes the encoder and calls <code>out.flush()</code>. 457 * All buffered pending data will then be decompressible from 458 * the output stream. 459 * <p> 460 * Calling this function very often may increase the compressed 461 * file size a lot. The filter chain options may affect the size 462 * increase too. For example, with LZMA2 the HC4 match finder has 463 * smaller penalty with flushing than BT4. 464 * <p> 465 * Some filters don't support flushing. If the filter chain has 466 * such a filter, <code>flush()</code> will call <code>endBlock()</code> 467 * before flushing. 468 * 469 * @throws XZIOException 470 * XZ Stream has grown too big 471 * 472 * @throws XZIOException 473 * stream finished or closed 474 * 475 * @throws IOException may be thrown by the underlying output stream 476 */ flush()477 public void flush() throws IOException { 478 if (exception != null) 479 throw exception; 480 481 if (finished) 482 throw new XZIOException("Stream finished or closed"); 483 484 try { 485 if (blockEncoder != null) { 486 if (filtersSupportFlushing) { 487 // This will eventually call out.flush() so 488 // no need to do it here again. 489 blockEncoder.flush(); 490 } else { 491 endBlock(); 492 out.flush(); 493 } 494 } else { 495 out.flush(); 496 } 497 } catch (IOException e) { 498 exception = e; 499 throw e; 500 } 501 } 502 503 /** 504 * Finishes compression without closing the underlying stream. 505 * No more data can be written to this stream after finishing 506 * (calling <code>write</code> with an empty buffer is OK). 507 * <p> 508 * Repeated calls to <code>finish()</code> do nothing unless 509 * an exception was thrown by this stream earlier. In that case 510 * the same exception is thrown again. 511 * <p> 512 * After finishing, the stream may be closed normally with 513 * <code>close()</code>. If the stream will be closed anyway, there 514 * usually is no need to call <code>finish()</code> separately. 515 * 516 * @throws XZIOException 517 * XZ Stream has grown too big 518 * 519 * @throws IOException may be thrown by the underlying output stream 520 */ finish()521 public void finish() throws IOException { 522 if (!finished) { 523 // This checks for pending exceptions so we don't need to 524 // worry about it here. 525 endBlock(); 526 527 try { 528 index.encode(out); 529 encodeStreamFooter(); 530 } catch (IOException e) { 531 exception = e; 532 throw e; 533 } 534 535 // Set it to true only if everything goes fine. Setting it earlier 536 // would cause repeated calls to finish() do nothing instead of 537 // throwing an exception to indicate an earlier error. 538 finished = true; 539 } 540 } 541 542 /** 543 * Finishes compression and closes the underlying stream. 544 * The underlying stream <code>out</code> is closed even if finishing 545 * fails. If both finishing and closing fail, the exception thrown 546 * by <code>finish()</code> is thrown and the exception from the failed 547 * <code>out.close()</code> is lost. 548 * 549 * @throws XZIOException 550 * XZ Stream has grown too big 551 * 552 * @throws IOException may be thrown by the underlying output stream 553 */ close()554 public void close() throws IOException { 555 if (out != null) { 556 // If finish() throws an exception, it stores the exception to 557 // the variable "exception". So we can ignore the possible 558 // exception here. 559 try { 560 finish(); 561 } catch (IOException e) {} 562 563 try { 564 out.close(); 565 } catch (IOException e) { 566 // Remember the exception but only if there is no previous 567 // pending exception. 568 if (exception == null) 569 exception = e; 570 } 571 572 out = null; 573 } 574 575 if (exception != null) 576 throw exception; 577 } 578 encodeStreamFlags(byte[] buf, int off)579 private void encodeStreamFlags(byte[] buf, int off) { 580 buf[off] = 0x00; 581 buf[off + 1] = (byte)streamFlags.checkType; 582 } 583 encodeStreamHeader()584 private void encodeStreamHeader() throws IOException { 585 out.write(XZ.HEADER_MAGIC); 586 587 byte[] buf = new byte[2]; 588 encodeStreamFlags(buf, 0); 589 out.write(buf); 590 591 EncoderUtil.writeCRC32(out, buf); 592 } 593 encodeStreamFooter()594 private void encodeStreamFooter() throws IOException { 595 byte[] buf = new byte[6]; 596 long backwardSize = index.getIndexSize() / 4 - 1; 597 for (int i = 0; i < 4; ++i) 598 buf[i] = (byte)(backwardSize >>> (i * 8)); 599 600 encodeStreamFlags(buf, 4); 601 602 EncoderUtil.writeCRC32(out, buf); 603 out.write(buf); 604 out.write(XZ.FOOTER_MAGIC); 605 } 606 } 607