1 /* Lzma2Enc.c -- LZMA2 Encoder
2 2015-10-04 : Igor Pavlov : Public domain */
3
4 #include "Precomp.h"
5
6 /* #include <stdio.h> */
7 #include <string.h>
8
9 /* #define _7ZIP_ST */
10
11 #include "Lzma2Enc.h"
12
13 #ifndef _7ZIP_ST
14 #include "MtCoder.h"
15 #else
16 #define NUM_MT_CODER_THREADS_MAX 1
17 #endif
18
19 #define LZMA2_CONTROL_LZMA (1 << 7)
20 #define LZMA2_CONTROL_COPY_NO_RESET 2
21 #define LZMA2_CONTROL_COPY_RESET_DIC 1
22 #define LZMA2_CONTROL_EOF 0
23
24 #define LZMA2_LCLP_MAX 4
25
26 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
27
28 #define LZMA2_PACK_SIZE_MAX (1 << 16)
29 #define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX
30 #define LZMA2_UNPACK_SIZE_MAX (1 << 21)
31 #define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX
32
33 #define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16)
34
35
36 #define PRF(x) /* x */
37
38 /* ---------- CLzma2EncInt ---------- */
39
40 typedef struct
41 {
42 CLzmaEncHandle enc;
43 UInt64 srcPos;
44 Byte props;
45 Bool needInitState;
46 Bool needInitProp;
47 } CLzma2EncInt;
48
Lzma2EncInt_Init(CLzma2EncInt * p,const CLzma2EncProps * props)49 static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)
50 {
51 Byte propsEncoded[LZMA_PROPS_SIZE];
52 SizeT propsSize = LZMA_PROPS_SIZE;
53 RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
54 RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
55 p->srcPos = 0;
56 p->props = propsEncoded[0];
57 p->needInitState = True;
58 p->needInitProp = True;
59 return SZ_OK;
60 }
61
62 SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,
63 ISzAlloc *alloc, ISzAlloc *allocBig);
64 SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
65 UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig);
66 SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
67 Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize);
68 const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp);
69 void LzmaEnc_Finish(CLzmaEncHandle pp);
70 void LzmaEnc_SaveState(CLzmaEncHandle pp);
71 void LzmaEnc_RestoreState(CLzmaEncHandle pp);
72
73
Lzma2EncInt_EncodeSubblock(CLzma2EncInt * p,Byte * outBuf,size_t * packSizeRes,ISeqOutStream * outStream)74 static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
75 size_t *packSizeRes, ISeqOutStream *outStream)
76 {
77 size_t packSizeLimit = *packSizeRes;
78 size_t packSize = packSizeLimit;
79 UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX;
80 unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0);
81 Bool useCopyBlock;
82 SRes res;
83
84 *packSizeRes = 0;
85 if (packSize < lzHeaderSize)
86 return SZ_ERROR_OUTPUT_EOF;
87 packSize -= lzHeaderSize;
88
89 LzmaEnc_SaveState(p->enc);
90 res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState,
91 outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize);
92
93 PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize));
94
95 if (unpackSize == 0)
96 return res;
97
98 if (res == SZ_OK)
99 useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16));
100 else
101 {
102 if (res != SZ_ERROR_OUTPUT_EOF)
103 return res;
104 res = SZ_OK;
105 useCopyBlock = True;
106 }
107
108 if (useCopyBlock)
109 {
110 size_t destPos = 0;
111 PRF(printf("################# COPY "));
112
113 while (unpackSize > 0)
114 {
115 UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE;
116 if (packSizeLimit - destPos < u + 3)
117 return SZ_ERROR_OUTPUT_EOF;
118 outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET);
119 outBuf[destPos++] = (Byte)((u - 1) >> 8);
120 outBuf[destPos++] = (Byte)(u - 1);
121 memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u);
122 unpackSize -= u;
123 destPos += u;
124 p->srcPos += u;
125
126 if (outStream)
127 {
128 *packSizeRes += destPos;
129 if (outStream->Write(outStream, outBuf, destPos) != destPos)
130 return SZ_ERROR_WRITE;
131 destPos = 0;
132 }
133 else
134 *packSizeRes = destPos;
135 /* needInitState = True; */
136 }
137
138 LzmaEnc_RestoreState(p->enc);
139 return SZ_OK;
140 }
141
142 {
143 size_t destPos = 0;
144 UInt32 u = unpackSize - 1;
145 UInt32 pm = (UInt32)(packSize - 1);
146 unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0);
147
148 PRF(printf(" "));
149
150 outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F));
151 outBuf[destPos++] = (Byte)(u >> 8);
152 outBuf[destPos++] = (Byte)u;
153 outBuf[destPos++] = (Byte)(pm >> 8);
154 outBuf[destPos++] = (Byte)pm;
155
156 if (p->needInitProp)
157 outBuf[destPos++] = p->props;
158
159 p->needInitProp = False;
160 p->needInitState = False;
161 destPos += packSize;
162 p->srcPos += unpackSize;
163
164 if (outStream)
165 if (outStream->Write(outStream, outBuf, destPos) != destPos)
166 return SZ_ERROR_WRITE;
167
168 *packSizeRes = destPos;
169 return SZ_OK;
170 }
171 }
172
173
174 /* ---------- Lzma2 Props ---------- */
175
Lzma2EncProps_Init(CLzma2EncProps * p)176 void Lzma2EncProps_Init(CLzma2EncProps *p)
177 {
178 LzmaEncProps_Init(&p->lzmaProps);
179 p->numTotalThreads = -1;
180 p->numBlockThreads = -1;
181 p->blockSize = 0;
182 }
183
Lzma2EncProps_Normalize(CLzma2EncProps * p)184 void Lzma2EncProps_Normalize(CLzma2EncProps *p)
185 {
186 int t1, t1n, t2, t3;
187 {
188 CLzmaEncProps lzmaProps = p->lzmaProps;
189 LzmaEncProps_Normalize(&lzmaProps);
190 t1n = lzmaProps.numThreads;
191 }
192
193 t1 = p->lzmaProps.numThreads;
194 t2 = p->numBlockThreads;
195 t3 = p->numTotalThreads;
196
197 if (t2 > NUM_MT_CODER_THREADS_MAX)
198 t2 = NUM_MT_CODER_THREADS_MAX;
199
200 if (t3 <= 0)
201 {
202 if (t2 <= 0)
203 t2 = 1;
204 t3 = t1n * t2;
205 }
206 else if (t2 <= 0)
207 {
208 t2 = t3 / t1n;
209 if (t2 == 0)
210 {
211 t1 = 1;
212 t2 = t3;
213 }
214 if (t2 > NUM_MT_CODER_THREADS_MAX)
215 t2 = NUM_MT_CODER_THREADS_MAX;
216 }
217 else if (t1 <= 0)
218 {
219 t1 = t3 / t2;
220 if (t1 == 0)
221 t1 = 1;
222 }
223 else
224 t3 = t1n * t2;
225
226 p->lzmaProps.numThreads = t1;
227
228 LzmaEncProps_Normalize(&p->lzmaProps);
229
230 t1 = p->lzmaProps.numThreads;
231
232 if (p->blockSize == 0)
233 {
234 UInt32 dictSize = p->lzmaProps.dictSize;
235 UInt64 blockSize = (UInt64)dictSize << 2;
236 const UInt32 kMinSize = (UInt32)1 << 20;
237 const UInt32 kMaxSize = (UInt32)1 << 28;
238 if (blockSize < kMinSize) blockSize = kMinSize;
239 if (blockSize > kMaxSize) blockSize = kMaxSize;
240 if (blockSize < dictSize) blockSize = dictSize;
241 p->blockSize = (size_t)blockSize;
242 }
243
244 if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1)
245 {
246 UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1;
247 if (temp > p->lzmaProps.reduceSize)
248 {
249 UInt64 numBlocks = temp / p->blockSize;
250 if (numBlocks < (unsigned)t2)
251 {
252 t2 = (unsigned)numBlocks;
253 if (t2 == 0)
254 t2 = 1;
255 t3 = t1 * t2;
256 }
257 }
258 }
259
260 p->numBlockThreads = t2;
261 p->numTotalThreads = t3;
262 }
263
264
Progress(ICompressProgress * p,UInt64 inSize,UInt64 outSize)265 static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
266 {
267 return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
268 }
269
270
271 /* ---------- Lzma2 ---------- */
272
273 typedef struct
274 {
275 Byte propEncoded;
276 CLzma2EncProps props;
277
278 Byte *outBuf;
279
280 ISzAlloc *alloc;
281 ISzAlloc *allocBig;
282
283 CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];
284
285 #ifndef _7ZIP_ST
286 CMtCoder mtCoder;
287 #endif
288
289 } CLzma2Enc;
290
291
292 /* ---------- Lzma2EncThread ---------- */
293
Lzma2Enc_EncodeMt1(CLzma2EncInt * p,CLzma2Enc * mainEncoder,ISeqOutStream * outStream,ISeqInStream * inStream,ICompressProgress * progress)294 static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,
295 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
296 {
297 UInt64 packTotal = 0;
298 SRes res = SZ_OK;
299
300 if (!mainEncoder->outBuf)
301 {
302 mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
303 if (!mainEncoder->outBuf)
304 return SZ_ERROR_MEM;
305 }
306
307 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
308 RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,
309 mainEncoder->alloc, mainEncoder->allocBig));
310
311 for (;;)
312 {
313 size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
314 res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);
315 if (res != SZ_OK)
316 break;
317 packTotal += packSize;
318 res = Progress(progress, p->srcPos, packTotal);
319 if (res != SZ_OK)
320 break;
321 if (packSize == 0)
322 break;
323 }
324
325 LzmaEnc_Finish(p->enc);
326
327 if (res == SZ_OK)
328 {
329 Byte b = 0;
330 if (outStream->Write(outStream, &b, 1) != 1)
331 return SZ_ERROR_WRITE;
332 }
333
334 return res;
335 }
336
337
338 #ifndef _7ZIP_ST
339
340 typedef struct
341 {
342 IMtCoderCallback funcTable;
343 CLzma2Enc *lzma2Enc;
344 } CMtCallbackImp;
345
MtCallbackImp_Code(void * pp,unsigned index,Byte * dest,size_t * destSize,const Byte * src,size_t srcSize,int finished)346 static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize,
347 const Byte *src, size_t srcSize, int finished)
348 {
349 CMtCallbackImp *imp = (CMtCallbackImp *)pp;
350 CLzma2Enc *mainEncoder = imp->lzma2Enc;
351 CLzma2EncInt *p = &mainEncoder->coders[index];
352
353 SRes res = SZ_OK;
354 {
355 size_t destLim = *destSize;
356 *destSize = 0;
357
358 if (srcSize != 0)
359 {
360 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
361
362 RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,
363 mainEncoder->alloc, mainEncoder->allocBig));
364
365 while (p->srcPos < srcSize)
366 {
367 size_t packSize = destLim - *destSize;
368 res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);
369 if (res != SZ_OK)
370 break;
371 *destSize += packSize;
372
373 if (packSize == 0)
374 {
375 res = SZ_ERROR_FAIL;
376 break;
377 }
378
379 if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)
380 {
381 res = SZ_ERROR_PROGRESS;
382 break;
383 }
384 }
385
386 LzmaEnc_Finish(p->enc);
387 if (res != SZ_OK)
388 return res;
389 }
390
391 if (finished)
392 {
393 if (*destSize == destLim)
394 return SZ_ERROR_OUTPUT_EOF;
395 dest[(*destSize)++] = 0;
396 }
397 }
398 return res;
399 }
400
401 #endif
402
403
404 /* ---------- Lzma2Enc ---------- */
405
Lzma2Enc_Create(ISzAlloc * alloc,ISzAlloc * allocBig)406 CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig)
407 {
408 CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc));
409 if (!p)
410 return NULL;
411 Lzma2EncProps_Init(&p->props);
412 Lzma2EncProps_Normalize(&p->props);
413 p->outBuf = 0;
414 p->alloc = alloc;
415 p->allocBig = allocBig;
416 {
417 unsigned i;
418 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
419 p->coders[i].enc = 0;
420 }
421
422 #ifndef _7ZIP_ST
423 MtCoder_Construct(&p->mtCoder);
424 #endif
425
426 return p;
427 }
428
Lzma2Enc_Destroy(CLzma2EncHandle pp)429 void Lzma2Enc_Destroy(CLzma2EncHandle pp)
430 {
431 CLzma2Enc *p = (CLzma2Enc *)pp;
432 unsigned i;
433 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
434 {
435 CLzma2EncInt *t = &p->coders[i];
436 if (t->enc)
437 {
438 LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);
439 t->enc = 0;
440 }
441 }
442
443 #ifndef _7ZIP_ST
444 MtCoder_Destruct(&p->mtCoder);
445 #endif
446
447 IAlloc_Free(p->alloc, p->outBuf);
448 IAlloc_Free(p->alloc, pp);
449 }
450
Lzma2Enc_SetProps(CLzma2EncHandle pp,const CLzma2EncProps * props)451 SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
452 {
453 CLzma2Enc *p = (CLzma2Enc *)pp;
454 CLzmaEncProps lzmaProps = props->lzmaProps;
455 LzmaEncProps_Normalize(&lzmaProps);
456 if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX)
457 return SZ_ERROR_PARAM;
458 p->props = *props;
459 Lzma2EncProps_Normalize(&p->props);
460 return SZ_OK;
461 }
462
Lzma2Enc_WriteProperties(CLzma2EncHandle pp)463 Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
464 {
465 CLzma2Enc *p = (CLzma2Enc *)pp;
466 unsigned i;
467 UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps);
468 for (i = 0; i < 40; i++)
469 if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i))
470 break;
471 return (Byte)i;
472 }
473
Lzma2Enc_Encode(CLzma2EncHandle pp,ISeqOutStream * outStream,ISeqInStream * inStream,ICompressProgress * progress)474 SRes Lzma2Enc_Encode(CLzma2EncHandle pp,
475 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
476 {
477 CLzma2Enc *p = (CLzma2Enc *)pp;
478 int i;
479
480 for (i = 0; i < p->props.numBlockThreads; i++)
481 {
482 CLzma2EncInt *t = &p->coders[(unsigned)i];
483 if (!t->enc)
484 {
485 t->enc = LzmaEnc_Create(p->alloc);
486 if (!t->enc)
487 return SZ_ERROR_MEM;
488 }
489 }
490
491 #ifndef _7ZIP_ST
492 if (p->props.numBlockThreads > 1)
493 {
494 CMtCallbackImp mtCallback;
495
496 mtCallback.funcTable.Code = MtCallbackImp_Code;
497 mtCallback.lzma2Enc = p;
498
499 p->mtCoder.progress = progress;
500 p->mtCoder.inStream = inStream;
501 p->mtCoder.outStream = outStream;
502 p->mtCoder.alloc = p->alloc;
503 p->mtCoder.mtCallback = &mtCallback.funcTable;
504
505 p->mtCoder.blockSize = p->props.blockSize;
506 p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;
507 if (p->mtCoder.destBlockSize < p->props.blockSize)
508 {
509 p->mtCoder.destBlockSize = (size_t)0 - 1;
510 if (p->mtCoder.destBlockSize < p->props.blockSize)
511 return SZ_ERROR_FAIL;
512 }
513 p->mtCoder.numThreads = p->props.numBlockThreads;
514
515 return MtCoder_Code(&p->mtCoder);
516 }
517 #endif
518
519 return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);
520 }
521