1 /* Lzma2Enc.c -- LZMA2 Encoder
2 2012-06-19 : Igor Pavlov : Public domain */
3
4 #include "Precomp.h"
5
6 /* #include <stdio.h> */
7 #include <string.h>
8
9 /* #define _7ZIP_ST */
10
11 #include "Lzma2Enc.h"
12
13 #ifndef _7ZIP_ST
14 #include "MtCoder.h"
15 #else
16 #define NUM_MT_CODER_THREADS_MAX 1
17 #endif
18
19 #define LZMA2_CONTROL_LZMA (1 << 7)
20 #define LZMA2_CONTROL_COPY_NO_RESET 2
21 #define LZMA2_CONTROL_COPY_RESET_DIC 1
22 #define LZMA2_CONTROL_EOF 0
23
24 #define LZMA2_LCLP_MAX 4
25
26 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
27
28 #define LZMA2_PACK_SIZE_MAX (1 << 16)
29 #define LZMA2_COPY_CHUNK_SIZE LZMA2_PACK_SIZE_MAX
30 #define LZMA2_UNPACK_SIZE_MAX (1 << 21)
31 #define LZMA2_KEEP_WINDOW_SIZE LZMA2_UNPACK_SIZE_MAX
32
33 #define LZMA2_CHUNK_SIZE_COMPRESSED_MAX ((1 << 16) + 16)
34
35
36 #define PRF(x) /* x */
37
38 /* ---------- CLzma2EncInt ---------- */
39
40 typedef struct
41 {
42 CLzmaEncHandle enc;
43 UInt64 srcPos;
44 Byte props;
45 Bool needInitState;
46 Bool needInitProp;
47 } CLzma2EncInt;
48
Lzma2EncInt_Init(CLzma2EncInt * p,const CLzma2EncProps * props)49 static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)
50 {
51 Byte propsEncoded[LZMA_PROPS_SIZE];
52 SizeT propsSize = LZMA_PROPS_SIZE;
53 RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
54 RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
55 p->srcPos = 0;
56 p->props = propsEncoded[0];
57 p->needInitState = True;
58 p->needInitProp = True;
59 return SZ_OK;
60 }
61
62 SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,
63 ISzAlloc *alloc, ISzAlloc *allocBig);
64 SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
65 UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig);
66 SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
67 Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize);
68 const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp);
69 void LzmaEnc_Finish(CLzmaEncHandle pp);
70 void LzmaEnc_SaveState(CLzmaEncHandle pp);
71 void LzmaEnc_RestoreState(CLzmaEncHandle pp);
72
73
Lzma2EncInt_EncodeSubblock(CLzma2EncInt * p,Byte * outBuf,size_t * packSizeRes,ISeqOutStream * outStream)74 static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
75 size_t *packSizeRes, ISeqOutStream *outStream)
76 {
77 size_t packSizeLimit = *packSizeRes;
78 size_t packSize = packSizeLimit;
79 UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX;
80 unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0);
81 Bool useCopyBlock;
82 SRes res;
83
84 *packSizeRes = 0;
85 if (packSize < lzHeaderSize)
86 return SZ_ERROR_OUTPUT_EOF;
87 packSize -= lzHeaderSize;
88
89 LzmaEnc_SaveState(p->enc);
90 res = LzmaEnc_CodeOneMemBlock(p->enc, p->needInitState,
91 outBuf + lzHeaderSize, &packSize, LZMA2_PACK_SIZE_MAX, &unpackSize);
92
93 PRF(printf("\npackSize = %7d unpackSize = %7d ", packSize, unpackSize));
94
95 if (unpackSize == 0)
96 return res;
97
98 if (res == SZ_OK)
99 useCopyBlock = (packSize + 2 >= unpackSize || packSize > (1 << 16));
100 else
101 {
102 if (res != SZ_ERROR_OUTPUT_EOF)
103 return res;
104 res = SZ_OK;
105 useCopyBlock = True;
106 }
107
108 if (useCopyBlock)
109 {
110 size_t destPos = 0;
111 PRF(printf("################# COPY "));
112 while (unpackSize > 0)
113 {
114 UInt32 u = (unpackSize < LZMA2_COPY_CHUNK_SIZE) ? unpackSize : LZMA2_COPY_CHUNK_SIZE;
115 if (packSizeLimit - destPos < u + 3)
116 return SZ_ERROR_OUTPUT_EOF;
117 outBuf[destPos++] = (Byte)(p->srcPos == 0 ? LZMA2_CONTROL_COPY_RESET_DIC : LZMA2_CONTROL_COPY_NO_RESET);
118 outBuf[destPos++] = (Byte)((u - 1) >> 8);
119 outBuf[destPos++] = (Byte)(u - 1);
120 memcpy(outBuf + destPos, LzmaEnc_GetCurBuf(p->enc) - unpackSize, u);
121 unpackSize -= u;
122 destPos += u;
123 p->srcPos += u;
124 if (outStream)
125 {
126 *packSizeRes += destPos;
127 if (outStream->Write(outStream, outBuf, destPos) != destPos)
128 return SZ_ERROR_WRITE;
129 destPos = 0;
130 }
131 else
132 *packSizeRes = destPos;
133 /* needInitState = True; */
134 }
135 LzmaEnc_RestoreState(p->enc);
136 return SZ_OK;
137 }
138 {
139 size_t destPos = 0;
140 UInt32 u = unpackSize - 1;
141 UInt32 pm = (UInt32)(packSize - 1);
142 unsigned mode = (p->srcPos == 0) ? 3 : (p->needInitState ? (p->needInitProp ? 2 : 1) : 0);
143
144 PRF(printf(" "));
145
146 outBuf[destPos++] = (Byte)(LZMA2_CONTROL_LZMA | (mode << 5) | ((u >> 16) & 0x1F));
147 outBuf[destPos++] = (Byte)(u >> 8);
148 outBuf[destPos++] = (Byte)u;
149 outBuf[destPos++] = (Byte)(pm >> 8);
150 outBuf[destPos++] = (Byte)pm;
151
152 if (p->needInitProp)
153 outBuf[destPos++] = p->props;
154
155 p->needInitProp = False;
156 p->needInitState = False;
157 destPos += packSize;
158 p->srcPos += unpackSize;
159
160 if (outStream)
161 if (outStream->Write(outStream, outBuf, destPos) != destPos)
162 return SZ_ERROR_WRITE;
163 *packSizeRes = destPos;
164 return SZ_OK;
165 }
166 }
167
168 /* ---------- Lzma2 Props ---------- */
169
Lzma2EncProps_Init(CLzma2EncProps * p)170 void Lzma2EncProps_Init(CLzma2EncProps *p)
171 {
172 LzmaEncProps_Init(&p->lzmaProps);
173 p->numTotalThreads = -1;
174 p->numBlockThreads = -1;
175 p->blockSize = 0;
176 }
177
Lzma2EncProps_Normalize(CLzma2EncProps * p)178 void Lzma2EncProps_Normalize(CLzma2EncProps *p)
179 {
180 int t1, t1n, t2, t3;
181 {
182 CLzmaEncProps lzmaProps = p->lzmaProps;
183 LzmaEncProps_Normalize(&lzmaProps);
184 t1n = lzmaProps.numThreads;
185 }
186
187 t1 = p->lzmaProps.numThreads;
188 t2 = p->numBlockThreads;
189 t3 = p->numTotalThreads;
190
191 if (t2 > NUM_MT_CODER_THREADS_MAX)
192 t2 = NUM_MT_CODER_THREADS_MAX;
193
194 if (t3 <= 0)
195 {
196 if (t2 <= 0)
197 t2 = 1;
198 t3 = t1n * t2;
199 }
200 else if (t2 <= 0)
201 {
202 t2 = t3 / t1n;
203 if (t2 == 0)
204 {
205 t1 = 1;
206 t2 = t3;
207 }
208 if (t2 > NUM_MT_CODER_THREADS_MAX)
209 t2 = NUM_MT_CODER_THREADS_MAX;
210 }
211 else if (t1 <= 0)
212 {
213 t1 = t3 / t2;
214 if (t1 == 0)
215 t1 = 1;
216 }
217 else
218 t3 = t1n * t2;
219
220 p->lzmaProps.numThreads = t1;
221
222 LzmaEncProps_Normalize(&p->lzmaProps);
223
224 if (p->blockSize == 0)
225 {
226 UInt32 dictSize = p->lzmaProps.dictSize;
227 UInt64 blockSize = (UInt64)dictSize << 2;
228 const UInt32 kMinSize = (UInt32)1 << 20;
229 const UInt32 kMaxSize = (UInt32)1 << 28;
230 if (blockSize < kMinSize) blockSize = kMinSize;
231 if (blockSize > kMaxSize) blockSize = kMaxSize;
232 if (blockSize < dictSize) blockSize = dictSize;
233 p->blockSize = (size_t)blockSize;
234 }
235 if (t2 > 1)
236 {
237 UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1;
238 if (temp > p->lzmaProps.reduceSize)
239 {
240 UInt64 numBlocks = temp / p->blockSize;
241 if (numBlocks < t2)
242 {
243 t2 = (UInt32)numBlocks;
244 t3 = t1 * t2;
245 }
246 }
247 }
248 p->numBlockThreads = t2;
249 p->numTotalThreads = t3;
250 }
251
Progress(ICompressProgress * p,UInt64 inSize,UInt64 outSize)252 static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
253 {
254 return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
255 }
256
257 /* ---------- Lzma2 ---------- */
258
259 typedef struct
260 {
261 Byte propEncoded;
262 CLzma2EncProps props;
263
264 Byte *outBuf;
265
266 ISzAlloc *alloc;
267 ISzAlloc *allocBig;
268
269 CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];
270
271 #ifndef _7ZIP_ST
272 CMtCoder mtCoder;
273 #endif
274
275 } CLzma2Enc;
276
277
278 /* ---------- Lzma2EncThread ---------- */
279
Lzma2Enc_EncodeMt1(CLzma2EncInt * p,CLzma2Enc * mainEncoder,ISeqOutStream * outStream,ISeqInStream * inStream,ICompressProgress * progress)280 static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,
281 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
282 {
283 UInt64 packTotal = 0;
284 SRes res = SZ_OK;
285
286 if (mainEncoder->outBuf == 0)
287 {
288 mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
289 if (mainEncoder->outBuf == 0)
290 return SZ_ERROR_MEM;
291 }
292 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
293 RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,
294 mainEncoder->alloc, mainEncoder->allocBig));
295 for (;;)
296 {
297 size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
298 res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);
299 if (res != SZ_OK)
300 break;
301 packTotal += packSize;
302 res = Progress(progress, p->srcPos, packTotal);
303 if (res != SZ_OK)
304 break;
305 if (packSize == 0)
306 break;
307 }
308 LzmaEnc_Finish(p->enc);
309 if (res == SZ_OK)
310 {
311 Byte b = 0;
312 if (outStream->Write(outStream, &b, 1) != 1)
313 return SZ_ERROR_WRITE;
314 }
315 return res;
316 }
317
318 #ifndef _7ZIP_ST
319
320 typedef struct
321 {
322 IMtCoderCallback funcTable;
323 CLzma2Enc *lzma2Enc;
324 } CMtCallbackImp;
325
MtCallbackImp_Code(void * pp,unsigned index,Byte * dest,size_t * destSize,const Byte * src,size_t srcSize,int finished)326 static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize,
327 const Byte *src, size_t srcSize, int finished)
328 {
329 CMtCallbackImp *imp = (CMtCallbackImp *)pp;
330 CLzma2Enc *mainEncoder = imp->lzma2Enc;
331 CLzma2EncInt *p = &mainEncoder->coders[index];
332
333 SRes res = SZ_OK;
334 {
335 size_t destLim = *destSize;
336 *destSize = 0;
337
338 if (srcSize != 0)
339 {
340 RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
341
342 RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,
343 mainEncoder->alloc, mainEncoder->allocBig));
344
345 while (p->srcPos < srcSize)
346 {
347 size_t packSize = destLim - *destSize;
348 res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);
349 if (res != SZ_OK)
350 break;
351 *destSize += packSize;
352
353 if (packSize == 0)
354 {
355 res = SZ_ERROR_FAIL;
356 break;
357 }
358
359 if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)
360 {
361 res = SZ_ERROR_PROGRESS;
362 break;
363 }
364 }
365 LzmaEnc_Finish(p->enc);
366 if (res != SZ_OK)
367 return res;
368 }
369 if (finished)
370 {
371 if (*destSize == destLim)
372 return SZ_ERROR_OUTPUT_EOF;
373 dest[(*destSize)++] = 0;
374 }
375 }
376 return res;
377 }
378
379 #endif
380
381 /* ---------- Lzma2Enc ---------- */
382
Lzma2Enc_Create(ISzAlloc * alloc,ISzAlloc * allocBig)383 CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig)
384 {
385 CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc));
386 if (p == 0)
387 return NULL;
388 Lzma2EncProps_Init(&p->props);
389 Lzma2EncProps_Normalize(&p->props);
390 p->outBuf = 0;
391 p->alloc = alloc;
392 p->allocBig = allocBig;
393 {
394 unsigned i;
395 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
396 p->coders[i].enc = 0;
397 }
398 #ifndef _7ZIP_ST
399 MtCoder_Construct(&p->mtCoder);
400 #endif
401
402 return p;
403 }
404
Lzma2Enc_Destroy(CLzma2EncHandle pp)405 void Lzma2Enc_Destroy(CLzma2EncHandle pp)
406 {
407 CLzma2Enc *p = (CLzma2Enc *)pp;
408 unsigned i;
409 for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
410 {
411 CLzma2EncInt *t = &p->coders[i];
412 if (t->enc)
413 {
414 LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);
415 t->enc = 0;
416 }
417 }
418
419 #ifndef _7ZIP_ST
420 MtCoder_Destruct(&p->mtCoder);
421 #endif
422
423 IAlloc_Free(p->alloc, p->outBuf);
424 IAlloc_Free(p->alloc, pp);
425 }
426
Lzma2Enc_SetProps(CLzma2EncHandle pp,const CLzma2EncProps * props)427 SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
428 {
429 CLzma2Enc *p = (CLzma2Enc *)pp;
430 CLzmaEncProps lzmaProps = props->lzmaProps;
431 LzmaEncProps_Normalize(&lzmaProps);
432 if (lzmaProps.lc + lzmaProps.lp > LZMA2_LCLP_MAX)
433 return SZ_ERROR_PARAM;
434 p->props = *props;
435 Lzma2EncProps_Normalize(&p->props);
436 return SZ_OK;
437 }
438
Lzma2Enc_WriteProperties(CLzma2EncHandle pp)439 Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
440 {
441 CLzma2Enc *p = (CLzma2Enc *)pp;
442 unsigned i;
443 UInt32 dicSize = LzmaEncProps_GetDictSize(&p->props.lzmaProps);
444 for (i = 0; i < 40; i++)
445 if (dicSize <= LZMA2_DIC_SIZE_FROM_PROP(i))
446 break;
447 return (Byte)i;
448 }
449
Lzma2Enc_Encode(CLzma2EncHandle pp,ISeqOutStream * outStream,ISeqInStream * inStream,ICompressProgress * progress)450 SRes Lzma2Enc_Encode(CLzma2EncHandle pp,
451 ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
452 {
453 CLzma2Enc *p = (CLzma2Enc *)pp;
454 int i;
455
456 for (i = 0; i < p->props.numBlockThreads; i++)
457 {
458 CLzma2EncInt *t = &p->coders[i];
459 if (t->enc == NULL)
460 {
461 t->enc = LzmaEnc_Create(p->alloc);
462 if (t->enc == NULL)
463 return SZ_ERROR_MEM;
464 }
465 }
466
467 #ifndef _7ZIP_ST
468 if (p->props.numBlockThreads <= 1)
469 #endif
470 return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);
471
472 #ifndef _7ZIP_ST
473
474 {
475 CMtCallbackImp mtCallback;
476
477 mtCallback.funcTable.Code = MtCallbackImp_Code;
478 mtCallback.lzma2Enc = p;
479
480 p->mtCoder.progress = progress;
481 p->mtCoder.inStream = inStream;
482 p->mtCoder.outStream = outStream;
483 p->mtCoder.alloc = p->alloc;
484 p->mtCoder.mtCallback = &mtCallback.funcTable;
485
486 p->mtCoder.blockSize = p->props.blockSize;
487 p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;
488 p->mtCoder.numThreads = p->props.numBlockThreads;
489
490 return MtCoder_Code(&p->mtCoder);
491 }
492 #endif
493 }
494