1 // LzmaEncoder.cpp
2
3 #include "StdAfx.h"
4
5 #include "../../../C/Alloc.h"
6
7 #include "../Common/CWrappers.h"
8 #include "../Common/StreamUtils.h"
9
10 #include "LzmaEncoder.h"
11
12 // #define LOG_LZMA_THREADS
13
14 #ifdef LOG_LZMA_THREADS
15
16 #include <stdio.h>
17
18 #include "../../Common/IntToString.h"
19 #include "../../Windows/TimeUtils.h"
20
21 EXTERN_C_BEGIN
22 void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]);
23 EXTERN_C_END
24
25 #endif
26
27 namespace NCompress {
28 namespace NLzma {
29
CEncoder()30 CEncoder::CEncoder()
31 {
32 _encoder = NULL;
33 _encoder = LzmaEnc_Create(&g_AlignedAlloc);
34 if (!_encoder)
35 throw 1;
36 }
37
~CEncoder()38 CEncoder::~CEncoder()
39 {
40 if (_encoder)
41 LzmaEnc_Destroy(_encoder, &g_AlignedAlloc, &g_BigAlloc);
42 }
43
GetLowCharFast(wchar_t c)44 static inline wchar_t GetLowCharFast(wchar_t c)
45 {
46 return c |= 0x20;
47 }
48
ParseMatchFinder(const wchar_t * s,int * btMode,int * numHashBytes)49 static int ParseMatchFinder(const wchar_t *s, int *btMode, int *numHashBytes)
50 {
51 const wchar_t c = GetLowCharFast(*s++);
52 if (c == 'h')
53 {
54 if (GetLowCharFast(*s++) != 'c')
55 return 0;
56 const int num = (int)(*s++ - L'0');
57 if (num < 4 || num > 5)
58 return 0;
59 if (*s != 0)
60 return 0;
61 *btMode = 0;
62 *numHashBytes = num;
63 return 1;
64 }
65
66 if (c != 'b')
67 return 0;
68 {
69 if (GetLowCharFast(*s++) != 't')
70 return 0;
71 const int num = (int)(*s++ - L'0');
72 if (num < 2 || num > 5)
73 return 0;
74 if (*s != 0)
75 return 0;
76 *btMode = 1;
77 *numHashBytes = num;
78 return 1;
79 }
80 }
81
82 #define SET_PROP_32(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = (int)v; break;
83 #define SET_PROP_32U(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = v; break;
84
85 HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep);
SetLzmaProp(PROPID propID,const PROPVARIANT & prop,CLzmaEncProps & ep)86 HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep)
87 {
88 if (propID == NCoderPropID::kMatchFinder)
89 {
90 if (prop.vt != VT_BSTR)
91 return E_INVALIDARG;
92 return ParseMatchFinder(prop.bstrVal, &ep.btMode, &ep.numHashBytes) ? S_OK : E_INVALIDARG;
93 }
94
95 if (propID == NCoderPropID::kAffinity)
96 {
97 if (prop.vt == VT_UI8)
98 ep.affinity = prop.uhVal.QuadPart;
99 else
100 return E_INVALIDARG;
101 return S_OK;
102 }
103
104 if (propID == NCoderPropID::kHashBits)
105 {
106 if (prop.vt == VT_UI4)
107 ep.numHashOutBits = prop.ulVal;
108 else
109 return E_INVALIDARG;
110 return S_OK;
111 }
112
113 if (propID > NCoderPropID::kReduceSize)
114 return S_OK;
115
116 if (propID == NCoderPropID::kReduceSize)
117 {
118 if (prop.vt == VT_UI8)
119 ep.reduceSize = prop.uhVal.QuadPart;
120 else
121 return E_INVALIDARG;
122 return S_OK;
123 }
124
125 if (propID == NCoderPropID::kDictionarySize)
126 {
127 if (prop.vt == VT_UI8)
128 {
129 // 21.03 : we support 64-bit VT_UI8 for dictionary and (dict == 4 GiB)
130 const UInt64 v = prop.uhVal.QuadPart;
131 if (v > ((UInt64)1 << 32))
132 return E_INVALIDARG;
133 UInt32 dict;
134 if (v == ((UInt64)1 << 32))
135 dict = (UInt32)(Int32)-1;
136 else
137 dict = (UInt32)v;
138 ep.dictSize = dict;
139 return S_OK;
140 }
141 }
142
143 if (prop.vt != VT_UI4)
144 return E_INVALIDARG;
145 const UInt32 v = prop.ulVal;
146 switch (propID)
147 {
148 case NCoderPropID::kDefaultProp:
149 if (v > 32)
150 return E_INVALIDARG;
151 ep.dictSize = (v == 32) ? (UInt32)(Int32)-1 : (UInt32)1 << (unsigned)v;
152 break;
153 SET_PROP_32(kLevel, level)
154 SET_PROP_32(kNumFastBytes, fb)
155 SET_PROP_32U(kMatchFinderCycles, mc)
156 SET_PROP_32(kAlgorithm, algo)
157 SET_PROP_32U(kDictionarySize, dictSize)
158 SET_PROP_32(kPosStateBits, pb)
159 SET_PROP_32(kLitPosBits, lp)
160 SET_PROP_32(kLitContextBits, lc)
161 SET_PROP_32(kNumThreads, numThreads)
162 default: return E_INVALIDARG;
163 }
164 return S_OK;
165 }
166
Z7_COM7F_IMF(CEncoder::SetCoderProperties (const PROPID * propIDs,const PROPVARIANT * coderProps,UInt32 numProps))167 Z7_COM7F_IMF(CEncoder::SetCoderProperties(const PROPID *propIDs,
168 const PROPVARIANT *coderProps, UInt32 numProps))
169 {
170 CLzmaEncProps props;
171 LzmaEncProps_Init(&props);
172
173 for (UInt32 i = 0; i < numProps; i++)
174 {
175 const PROPVARIANT &prop = coderProps[i];
176 const PROPID propID = propIDs[i];
177 switch (propID)
178 {
179 case NCoderPropID::kEndMarker:
180 if (prop.vt != VT_BOOL)
181 return E_INVALIDARG;
182 props.writeEndMark = (prop.boolVal != VARIANT_FALSE);
183 break;
184 default:
185 RINOK(SetLzmaProp(propID, prop, props))
186 }
187 }
188 return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
189 }
190
191
Z7_COM7F_IMF(CEncoder::SetCoderPropertiesOpt (const PROPID * propIDs,const PROPVARIANT * coderProps,UInt32 numProps))192 Z7_COM7F_IMF(CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs,
193 const PROPVARIANT *coderProps, UInt32 numProps))
194 {
195 for (UInt32 i = 0; i < numProps; i++)
196 {
197 const PROPVARIANT &prop = coderProps[i];
198 const PROPID propID = propIDs[i];
199 if (propID == NCoderPropID::kExpectedDataSize)
200 if (prop.vt == VT_UI8)
201 LzmaEnc_SetDataSize(_encoder, prop.uhVal.QuadPart);
202 }
203 return S_OK;
204 }
205
206
Z7_COM7F_IMF(CEncoder::WriteCoderProperties (ISequentialOutStream * outStream))207 Z7_COM7F_IMF(CEncoder::WriteCoderProperties(ISequentialOutStream *outStream))
208 {
209 Byte props[LZMA_PROPS_SIZE];
210 SizeT size = LZMA_PROPS_SIZE;
211 RINOK(LzmaEnc_WriteProperties(_encoder, props, &size))
212 return WriteStream(outStream, props, size);
213 }
214
215
216 #define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \
217 if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes;
218
219
220
221 #ifdef LOG_LZMA_THREADS
222
GetTime64(const FILETIME & t)223 static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; }
224
PrintNum(UInt64 val,unsigned numDigits,char c=' ')225 static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ')
226 {
227 char temp[64];
228 char *p = temp + 32;
229 ConvertUInt64ToString(val, p);
230 unsigned len = (unsigned)strlen(p);
231 for (; len < numDigits; len++)
232 *--p = c;
233 printf("%s", p);
234 }
235
PrintTime(const char * s,UInt64 val,UInt64 total)236 static void PrintTime(const char *s, UInt64 val, UInt64 total)
237 {
238 printf(" %s :", s);
239 const UInt32 kFreq = 10000000;
240 UInt64 sec = val / kFreq;
241 PrintNum(sec, 6);
242 printf(" .");
243 UInt32 ms = (UInt32)(val - (sec * kFreq)) / (kFreq / 1000);
244 PrintNum(ms, 3, '0');
245
246 while (val > ((UInt64)1 << 56))
247 {
248 val >>= 1;
249 total >>= 1;
250 }
251
252 UInt64 percent = 0;
253 if (total != 0)
254 percent = val * 100 / total;
255 printf(" =");
256 PrintNum(percent, 4);
257 printf("%%");
258 }
259
260
261 struct CBaseStat
262 {
263 UInt64 kernelTime, userTime;
264
GetNCompress::NLzma::CBaseStat265 BOOL Get(HANDLE thread, const CBaseStat *prevStat)
266 {
267 FILETIME creationTimeFT, exitTimeFT, kernelTimeFT, userTimeFT;
268 BOOL res = GetThreadTimes(thread
269 , &creationTimeFT, &exitTimeFT, &kernelTimeFT, &userTimeFT);
270 if (res)
271 {
272 kernelTime = GetTime64(kernelTimeFT);
273 userTime = GetTime64(userTimeFT);
274 if (prevStat)
275 {
276 kernelTime -= prevStat->kernelTime;
277 userTime -= prevStat->userTime;
278 }
279 }
280 return res;
281 }
282 };
283
284
PrintStat(HANDLE thread,UInt64 totalTime,const CBaseStat * prevStat)285 static void PrintStat(HANDLE thread, UInt64 totalTime, const CBaseStat *prevStat)
286 {
287 CBaseStat newStat;
288 if (!newStat.Get(thread, prevStat))
289 return;
290
291 PrintTime("K", newStat.kernelTime, totalTime);
292
293 const UInt64 processTime = newStat.kernelTime + newStat.userTime;
294
295 PrintTime("U", newStat.userTime, totalTime);
296 PrintTime("S", processTime, totalTime);
297 printf("\n");
298 // PrintTime("G ", totalTime, totalTime);
299 }
300
301 #endif
302
303
304
Z7_COM7F_IMF(CEncoder::Code (ISequentialInStream * inStream,ISequentialOutStream * outStream,const UInt64 *,const UInt64 *,ICompressProgressInfo * progress))305 Z7_COM7F_IMF(CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
306 const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress))
307 {
308 CSeqInStreamWrap inWrap;
309 CSeqOutStreamWrap outWrap;
310 CCompressProgressWrap progressWrap;
311
312 inWrap.Init(inStream);
313 outWrap.Init(outStream);
314 progressWrap.Init(progress);
315
316 #ifdef LOG_LZMA_THREADS
317
318 FILETIME startTimeFT;
319 NWindows::NTime::GetCurUtcFileTime(startTimeFT);
320 UInt64 totalTime = GetTime64(startTimeFT);
321 CBaseStat oldStat;
322 if (!oldStat.Get(GetCurrentThread(), NULL))
323 return E_FAIL;
324
325 #endif
326
327
328 SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt,
329 progress ? &progressWrap.vt : NULL, &g_AlignedAlloc, &g_BigAlloc);
330
331 _inputProcessed = inWrap.Processed;
332
333 RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ)
334 RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE)
335 RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS)
336
337
338 #ifdef LOG_LZMA_THREADS
339
340 NWindows::NTime::GetCurUtcFileTime(startTimeFT);
341 totalTime = GetTime64(startTimeFT) - totalTime;
342 HANDLE lz_threads[2];
343 LzmaEnc_GetLzThreads(_encoder, lz_threads);
344 printf("\n");
345 printf("Main: "); PrintStat(GetCurrentThread(), totalTime, &oldStat);
346 printf("Hash: "); PrintStat(lz_threads[0], totalTime, NULL);
347 printf("BinT: "); PrintStat(lz_threads[1], totalTime, NULL);
348 // PrintTime("Total: ", totalTime, totalTime);
349 printf("\n");
350
351 #endif
352
353 return SResToHRESULT(res);
354 }
355
356 }}
357