1 /* XzEnc.c -- Xz Encode
2 2014-12-30 : Igor Pavlov : Public domain */
3
4 #include "Precomp.h"
5
6 #include <stdlib.h>
7 #include <string.h>
8
9 #include "7zCrc.h"
10 #include "Alloc.h"
11 #include "Bra.h"
12 #include "CpuArch.h"
13 #ifdef USE_SUBBLOCK
14 #include "Bcj3Enc.c"
15 #include "SbFind.c"
16 #include "SbEnc.c"
17 #endif
18
19 #include "XzEnc.h"
20
SzBigAlloc(void * p,size_t size)21 static void *SzBigAlloc(void *p, size_t size) { p = p; return BigAlloc(size); }
SzBigFree(void * p,void * address)22 static void SzBigFree(void *p, void *address) { p = p; BigFree(address); }
23 static ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
24
SzAlloc(void * p,size_t size)25 static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
SzFree(void * p,void * address)26 static void SzFree(void *p, void *address) { p = p; MyFree(address); }
27 static ISzAlloc g_Alloc = { SzAlloc, SzFree };
28
29 #define XzBlock_ClearFlags(p) (p)->flags = 0;
30 #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);
31 #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE;
32 #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;
33
WriteBytes(ISeqOutStream * s,const void * buf,UInt32 size)34 static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size)
35 {
36 return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;
37 }
38
WriteBytesAndCrc(ISeqOutStream * s,const void * buf,UInt32 size,UInt32 * crc)39 static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc)
40 {
41 *crc = CrcUpdate(*crc, buf, size);
42 return WriteBytes(s, buf, size);
43 }
44
Xz_WriteHeader(CXzStreamFlags f,ISeqOutStream * s)45 SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)
46 {
47 UInt32 crc;
48 Byte header[XZ_STREAM_HEADER_SIZE];
49 memcpy(header, XZ_SIG, XZ_SIG_SIZE);
50 header[XZ_SIG_SIZE] = (Byte)(f >> 8);
51 header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);
52 crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);
53 SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);
54 return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);
55 }
56
XzBlock_WriteHeader(const CXzBlock * p,ISeqOutStream * s)57 SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)
58 {
59 Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
60
61 unsigned pos = 1;
62 int numFilters, i;
63 header[pos++] = p->flags;
64
65 if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);
66 if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);
67 numFilters = XzBlock_GetNumFilters(p);
68 for (i = 0; i < numFilters; i++)
69 {
70 const CXzFilter *f = &p->filters[i];
71 pos += Xz_WriteVarInt(header + pos, f->id);
72 pos += Xz_WriteVarInt(header + pos, f->propsSize);
73 memcpy(header + pos, f->props, f->propsSize);
74 pos += f->propsSize;
75 }
76 while((pos & 3) != 0)
77 header[pos++] = 0;
78 header[0] = (Byte)(pos >> 2);
79 SetUi32(header + pos, CrcCalc(header, pos));
80 return WriteBytes(s, header, pos + 4);
81 }
82
Xz_WriteFooter(CXzStream * p,ISeqOutStream * s)83 SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s)
84 {
85 Byte buf[32];
86 UInt64 globalPos;
87 {
88 UInt32 crc = CRC_INIT_VAL;
89 unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);
90 size_t i;
91
92 globalPos = pos;
93 buf[0] = 0;
94 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
95 for (i = 0; i < p->numBlocks; i++)
96 {
97 const CXzBlockSizes *block = &p->blocks[i];
98 pos = Xz_WriteVarInt(buf, block->totalSize);
99 pos += Xz_WriteVarInt(buf + pos, block->unpackSize);
100 globalPos += pos;
101 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
102 }
103 pos = ((unsigned)globalPos & 3);
104 if (pos != 0)
105 {
106 buf[0] = buf[1] = buf[2] = 0;
107 RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc));
108 globalPos += 4 - pos;
109 }
110 {
111 SetUi32(buf, CRC_GET_DIGEST(crc));
112 RINOK(WriteBytes(s, buf, 4));
113 globalPos += 4;
114 }
115 }
116
117 {
118 UInt32 indexSize = (UInt32)((globalPos >> 2) - 1);
119 SetUi32(buf + 4, indexSize);
120 buf[8] = (Byte)(p->flags >> 8);
121 buf[9] = (Byte)(p->flags & 0xFF);
122 SetUi32(buf, CrcCalc(buf + 4, 6));
123 memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE);
124 return WriteBytes(s, buf, 12);
125 }
126 }
127
Xz_AddIndexRecord(CXzStream * p,UInt64 unpackSize,UInt64 totalSize,ISzAlloc * alloc)128 SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc)
129 {
130 if (p->blocks == 0 || p->numBlocksAllocated == p->numBlocks)
131 {
132 size_t num = (p->numBlocks + 1) * 2;
133 size_t newSize = sizeof(CXzBlockSizes) * num;
134 CXzBlockSizes *blocks;
135 if (newSize / sizeof(CXzBlockSizes) != num)
136 return SZ_ERROR_MEM;
137 blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize);
138 if (blocks == 0)
139 return SZ_ERROR_MEM;
140 if (p->numBlocks != 0)
141 {
142 memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes));
143 Xz_Free(p, alloc);
144 }
145 p->blocks = blocks;
146 p->numBlocksAllocated = num;
147 }
148 {
149 CXzBlockSizes *block = &p->blocks[p->numBlocks++];
150 block->totalSize = totalSize;
151 block->unpackSize = unpackSize;
152 }
153 return SZ_OK;
154 }
155
156 /* ---------- CSeqCheckInStream ---------- */
157
158 typedef struct
159 {
160 ISeqInStream p;
161 ISeqInStream *realStream;
162 UInt64 processed;
163 CXzCheck check;
164 } CSeqCheckInStream;
165
SeqCheckInStream_Init(CSeqCheckInStream * p,int mode)166 void SeqCheckInStream_Init(CSeqCheckInStream *p, int mode)
167 {
168 p->processed = 0;
169 XzCheck_Init(&p->check, mode);
170 }
171
SeqCheckInStream_GetDigest(CSeqCheckInStream * p,Byte * digest)172 void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)
173 {
174 XzCheck_Final(&p->check, digest);
175 }
176
SeqCheckInStream_Read(void * pp,void * data,size_t * size)177 static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size)
178 {
179 CSeqCheckInStream *p = (CSeqCheckInStream *)pp;
180 SRes res = p->realStream->Read(p->realStream, data, size);
181 XzCheck_Update(&p->check, data, *size);
182 p->processed += *size;
183 return res;
184 }
185
186 /* ---------- CSeqSizeOutStream ---------- */
187
188 typedef struct
189 {
190 ISeqOutStream p;
191 ISeqOutStream *realStream;
192 UInt64 processed;
193 } CSeqSizeOutStream;
194
MyWrite(void * pp,const void * data,size_t size)195 static size_t MyWrite(void *pp, const void *data, size_t size)
196 {
197 CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp;
198 size = p->realStream->Write(p->realStream, data, size);
199 p->processed += size;
200 return size;
201 }
202
203 /* ---------- CSeqInFilter ---------- */
204
205 #define FILTER_BUF_SIZE (1 << 20)
206
207 typedef struct
208 {
209 ISeqInStream p;
210 ISeqInStream *realStream;
211 IStateCoder StateCoder;
212 Byte *buf;
213 size_t curPos;
214 size_t endPos;
215 int srcWasFinished;
216 } CSeqInFilter;
217
SeqInFilter_Read(void * pp,void * data,size_t * size)218 static SRes SeqInFilter_Read(void *pp, void *data, size_t *size)
219 {
220 CSeqInFilter *p = (CSeqInFilter *)pp;
221 size_t sizeOriginal = *size;
222 if (sizeOriginal == 0)
223 return SZ_OK;
224 *size = 0;
225 for (;;)
226 {
227 if (!p->srcWasFinished && p->curPos == p->endPos)
228 {
229 p->curPos = 0;
230 p->endPos = FILTER_BUF_SIZE;
231 RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos));
232 if (p->endPos == 0)
233 p->srcWasFinished = 1;
234 }
235 {
236 SizeT srcLen = p->endPos - p->curPos;
237 int wasFinished;
238 SRes res;
239 *size = sizeOriginal;
240 res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen,
241 p->srcWasFinished, CODER_FINISH_ANY, &wasFinished);
242 p->curPos += srcLen;
243 if (*size != 0 || srcLen == 0 || res != 0)
244 return res;
245 }
246 }
247 }
248
SeqInFilter_Construct(CSeqInFilter * p)249 static void SeqInFilter_Construct(CSeqInFilter *p)
250 {
251 p->buf = NULL;
252 p->p.Read = SeqInFilter_Read;
253 }
254
SeqInFilter_Free(CSeqInFilter * p)255 static void SeqInFilter_Free(CSeqInFilter *p)
256 {
257 if (p->buf)
258 {
259 g_Alloc.Free(&g_Alloc, p->buf);
260 p->buf = NULL;
261 }
262 }
263
264 SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc);
265
SeqInFilter_Init(CSeqInFilter * p,const CXzFilter * props)266 static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props)
267 {
268 if (!p->buf)
269 {
270 p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE);
271 if (!p->buf)
272 return SZ_ERROR_MEM;
273 }
274 p->curPos = p->endPos = 0;
275 p->srcWasFinished = 0;
276 RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc));
277 RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc));
278 p->StateCoder.Init(p->StateCoder.p);
279 return SZ_OK;
280 }
281
282 /* ---------- CSbEncInStream ---------- */
283
284 #ifdef USE_SUBBLOCK
285
286 typedef struct
287 {
288 ISeqInStream p;
289 ISeqInStream *inStream;
290 CSbEnc enc;
291 } CSbEncInStream;
292
SbEncInStream_Read(void * pp,void * data,size_t * size)293 static SRes SbEncInStream_Read(void *pp, void *data, size_t *size)
294 {
295 CSbEncInStream *p = (CSbEncInStream *)pp;
296 size_t sizeOriginal = *size;
297 if (sizeOriginal == 0)
298 return S_OK;
299 for (;;)
300 {
301 if (p->enc.needRead && !p->enc.readWasFinished)
302 {
303 size_t processed = p->enc.needReadSizeMax;
304 RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed));
305 p->enc.readPos += processed;
306 if (processed == 0)
307 {
308 p->enc.readWasFinished = True;
309 p->enc.isFinalFinished = True;
310 }
311 p->enc.needRead = False;
312 }
313 *size = sizeOriginal;
314 RINOK(SbEnc_Read(&p->enc, data, size));
315 if (*size != 0 || !p->enc.needRead)
316 return S_OK;
317 }
318 }
319
SbEncInStream_Construct(CSbEncInStream * p,ISzAlloc * alloc)320 void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc)
321 {
322 SbEnc_Construct(&p->enc, alloc);
323 p->p.Read = SbEncInStream_Read;
324 }
325
SbEncInStream_Init(CSbEncInStream * p)326 SRes SbEncInStream_Init(CSbEncInStream *p)
327 {
328 return SbEnc_Init(&p->enc);
329 }
330
SbEncInStream_Free(CSbEncInStream * p)331 void SbEncInStream_Free(CSbEncInStream *p)
332 {
333 SbEnc_Free(&p->enc);
334 }
335
336 #endif
337
338
339 typedef struct
340 {
341 CLzma2EncHandle lzma2;
342 #ifdef USE_SUBBLOCK
343 CSbEncInStream sb;
344 #endif
345 CSeqInFilter filter;
346 ISzAlloc *alloc;
347 ISzAlloc *bigAlloc;
348 } CLzma2WithFilters;
349
350
Lzma2WithFilters_Construct(CLzma2WithFilters * p,ISzAlloc * alloc,ISzAlloc * bigAlloc)351 static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc)
352 {
353 p->alloc = alloc;
354 p->bigAlloc = bigAlloc;
355 p->lzma2 = NULL;
356 #ifdef USE_SUBBLOCK
357 SbEncInStream_Construct(&p->sb, alloc);
358 #endif
359 SeqInFilter_Construct(&p->filter);
360 }
361
Lzma2WithFilters_Create(CLzma2WithFilters * p)362 static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p)
363 {
364 p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc);
365 if (p->lzma2 == 0)
366 return SZ_ERROR_MEM;
367 return SZ_OK;
368 }
369
Lzma2WithFilters_Free(CLzma2WithFilters * p)370 static void Lzma2WithFilters_Free(CLzma2WithFilters *p)
371 {
372 SeqInFilter_Free(&p->filter);
373 #ifdef USE_SUBBLOCK
374 SbEncInStream_Free(&p->sb);
375 #endif
376 if (p->lzma2)
377 {
378 Lzma2Enc_Destroy(p->lzma2);
379 p->lzma2 = NULL;
380 }
381 }
382
XzProps_Init(CXzProps * p)383 void XzProps_Init(CXzProps *p)
384 {
385 p->lzma2Props = 0;
386 p->filterProps = 0;
387 p->checkId = XZ_CHECK_CRC32;
388 }
389
XzFilterProps_Init(CXzFilterProps * p)390 void XzFilterProps_Init(CXzFilterProps *p)
391 {
392 p->id = 0;
393 p->delta = 0;
394 p->ip= 0;
395 p->ipDefined = False;
396 }
397
Xz_Compress(CXzStream * xz,CLzma2WithFilters * lzmaf,ISeqOutStream * outStream,ISeqInStream * inStream,const CXzProps * props,ICompressProgress * progress)398 static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf,
399 ISeqOutStream *outStream, ISeqInStream *inStream,
400 const CXzProps *props, ICompressProgress *progress)
401 {
402 xz->flags = (Byte)props->checkId;
403
404 RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props));
405 RINOK(Xz_WriteHeader(xz->flags, outStream));
406
407 {
408 CSeqCheckInStream checkInStream;
409 CSeqSizeOutStream seqSizeOutStream;
410 CXzBlock block;
411 int filterIndex = 0;
412 CXzFilter *filter = NULL;
413 const CXzFilterProps *fp = props->filterProps;
414
415 XzBlock_ClearFlags(&block);
416 XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0));
417
418 if (fp)
419 {
420 filter = &block.filters[filterIndex++];
421 filter->id = fp->id;
422 filter->propsSize = 0;
423 if (fp->id == XZ_ID_Delta)
424 {
425 filter->props[0] = (Byte)(fp->delta - 1);
426 filter->propsSize = 1;
427 }
428 else if (fp->ipDefined)
429 {
430 SetUi32(filter->props, fp->ip);
431 filter->propsSize = 4;
432 }
433 }
434
435 {
436 CXzFilter *f = &block.filters[filterIndex++];
437 f->id = XZ_ID_LZMA2;
438 f->propsSize = 1;
439 f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);
440 }
441
442 seqSizeOutStream.p.Write = MyWrite;
443 seqSizeOutStream.realStream = outStream;
444 seqSizeOutStream.processed = 0;
445
446 RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p));
447
448 checkInStream.p.Read = SeqCheckInStream_Read;
449 checkInStream.realStream = inStream;
450 SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags));
451
452 if (fp)
453 {
454 #ifdef USE_SUBBLOCK
455 if (fp->id == XZ_ID_Subblock)
456 {
457 lzmaf->sb.inStream = &checkInStream.p;
458 RINOK(SbEncInStream_Init(&lzmaf->sb));
459 }
460 else
461 #endif
462 {
463 lzmaf->filter.realStream = &checkInStream.p;
464 RINOK(SeqInFilter_Init(&lzmaf->filter, filter));
465 }
466 }
467
468 {
469 UInt64 packPos = seqSizeOutStream.processed;
470 SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p,
471 fp ?
472 #ifdef USE_SUBBLOCK
473 (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p:
474 #endif
475 &lzmaf->filter.p:
476 &checkInStream.p,
477 progress);
478 RINOK(res);
479 block.unpackSize = checkInStream.processed;
480 block.packSize = seqSizeOutStream.processed - packPos;
481 }
482
483 {
484 unsigned padSize = 0;
485 Byte buf[128];
486 while((((unsigned)block.packSize + padSize) & 3) != 0)
487 buf[padSize++] = 0;
488 SeqCheckInStream_GetDigest(&checkInStream, buf + padSize);
489 RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags)));
490 RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc));
491 }
492 }
493 return Xz_WriteFooter(xz, outStream);
494 }
495
Xz_Encode(ISeqOutStream * outStream,ISeqInStream * inStream,const CXzProps * props,ICompressProgress * progress)496 SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
497 const CXzProps *props, ICompressProgress *progress)
498 {
499 SRes res;
500 CXzStream xz;
501 CLzma2WithFilters lzmaf;
502 Xz_Construct(&xz);
503 Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc);
504 res = Lzma2WithFilters_Create(&lzmaf);
505 if (res == SZ_OK)
506 res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress);
507 Lzma2WithFilters_Free(&lzmaf);
508 Xz_Free(&xz, &g_Alloc);
509 return res;
510 }
511
Xz_EncodeEmpty(ISeqOutStream * outStream)512 SRes Xz_EncodeEmpty(ISeqOutStream *outStream)
513 {
514 SRes res;
515 CXzStream xz;
516 Xz_Construct(&xz);
517 res = Xz_WriteHeader(xz.flags, outStream);
518 if (res == SZ_OK)
519 res = Xz_WriteFooter(&xz, outStream);
520 Xz_Free(&xz, &g_Alloc);
521 return res;
522 }
523