1 // LzmaEncoder.cpp
2
3 #include "StdAfx.h"
4
5 #include "../../../C/Alloc.h"
6
7 #include "../Common/CWrappers.h"
8 #include "../Common/StreamUtils.h"
9
10 #include "LzmaEncoder.h"
11
12 // #define LOG_LZMA_THREADS
13
14 #ifdef LOG_LZMA_THREADS
15
16 #include <stdio.h>
17
18 #include "../../Common/IntToString.h"
19 #include "../../Windows/TimeUtils.h"
20
21 EXTERN_C_BEGIN
22 void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]);
23 EXTERN_C_END
24
25 #endif
26
27 namespace NCompress {
28 namespace NLzma {
29
CEncoder()30 CEncoder::CEncoder()
31 {
32 _encoder = NULL;
33 _encoder = LzmaEnc_Create(&g_AlignedAlloc);
34 if (!_encoder)
35 throw 1;
36 }
37
~CEncoder()38 CEncoder::~CEncoder()
39 {
40 if (_encoder)
41 LzmaEnc_Destroy(_encoder, &g_AlignedAlloc, &g_BigAlloc);
42 }
43
GetLowCharFast(wchar_t c)44 static inline wchar_t GetLowCharFast(wchar_t c)
45 {
46 return c |= 0x20;
47 }
48
ParseMatchFinder(const wchar_t * s,int * btMode,int * numHashBytes)49 static int ParseMatchFinder(const wchar_t *s, int *btMode, int *numHashBytes)
50 {
51 wchar_t c = GetLowCharFast(*s++);
52 if (c == 'h')
53 {
54 if (GetLowCharFast(*s++) != 'c')
55 return 0;
56 int num = (int)(*s++ - L'0');
57 if (num < 4 || num > 5)
58 return 0;
59 if (*s != 0)
60 return 0;
61 *btMode = 0;
62 *numHashBytes = num;
63 return 1;
64 }
65
66 if (c != 'b')
67 return 0;
68 {
69 if (GetLowCharFast(*s++) != 't')
70 return 0;
71 int num = (int)(*s++ - L'0');
72 if (num < 2 || num > 5)
73 return 0;
74 if (*s != 0)
75 return 0;
76 *btMode = 1;
77 *numHashBytes = num;
78 return 1;
79 }
80 }
81
82 #define SET_PROP_32(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = (int)v; break;
83 #define SET_PROP_32U(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = v; break;
84
85 HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep);
SetLzmaProp(PROPID propID,const PROPVARIANT & prop,CLzmaEncProps & ep)86 HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep)
87 {
88 if (propID == NCoderPropID::kMatchFinder)
89 {
90 if (prop.vt != VT_BSTR)
91 return E_INVALIDARG;
92 return ParseMatchFinder(prop.bstrVal, &ep.btMode, &ep.numHashBytes) ? S_OK : E_INVALIDARG;
93 }
94
95 if (propID == NCoderPropID::kAffinity)
96 {
97 if (prop.vt == VT_UI8)
98 ep.affinity = prop.uhVal.QuadPart;
99 else
100 return E_INVALIDARG;
101 return S_OK;
102 }
103
104 if (propID > NCoderPropID::kReduceSize)
105 return S_OK;
106
107 if (propID == NCoderPropID::kReduceSize)
108 {
109 if (prop.vt == VT_UI8)
110 ep.reduceSize = prop.uhVal.QuadPart;
111 else
112 return E_INVALIDARG;
113 return S_OK;
114 }
115
116 if (propID == NCoderPropID::kDictionarySize)
117 {
118 if (prop.vt == VT_UI8)
119 {
120 // 21.03 : we support 64-bit VT_UI8 for dictionary and (dict == 4 GiB)
121 const UInt64 v = prop.uhVal.QuadPart;
122 if (v > ((UInt64)1 << 32))
123 return E_INVALIDARG;
124 UInt32 dict;
125 if (v == ((UInt64)1 << 32))
126 dict = (UInt32)(Int32)-1;
127 else
128 dict = (UInt32)v;
129 ep.dictSize = dict;
130 return S_OK;
131 }
132 }
133
134 if (prop.vt != VT_UI4)
135 return E_INVALIDARG;
136 UInt32 v = prop.ulVal;
137 switch (propID)
138 {
139 case NCoderPropID::kDefaultProp:
140 if (v > 32)
141 return E_INVALIDARG;
142 ep.dictSize = (v == 32) ? (UInt32)(Int32)-1 : (UInt32)1 << (unsigned)v;
143 break;
144 SET_PROP_32(kLevel, level)
145 SET_PROP_32(kNumFastBytes, fb)
146 SET_PROP_32U(kMatchFinderCycles, mc)
147 SET_PROP_32(kAlgorithm, algo)
148 SET_PROP_32U(kDictionarySize, dictSize)
149 SET_PROP_32(kPosStateBits, pb)
150 SET_PROP_32(kLitPosBits, lp)
151 SET_PROP_32(kLitContextBits, lc)
152 SET_PROP_32(kNumThreads, numThreads)
153 default: return E_INVALIDARG;
154 }
155 return S_OK;
156 }
157
SetCoderProperties(const PROPID * propIDs,const PROPVARIANT * coderProps,UInt32 numProps)158 STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
159 const PROPVARIANT *coderProps, UInt32 numProps)
160 {
161 CLzmaEncProps props;
162 LzmaEncProps_Init(&props);
163
164 for (UInt32 i = 0; i < numProps; i++)
165 {
166 const PROPVARIANT &prop = coderProps[i];
167 PROPID propID = propIDs[i];
168 switch (propID)
169 {
170 case NCoderPropID::kEndMarker:
171 if (prop.vt != VT_BOOL)
172 return E_INVALIDARG;
173 props.writeEndMark = (prop.boolVal != VARIANT_FALSE);
174 break;
175 default:
176 RINOK(SetLzmaProp(propID, prop, props));
177 }
178 }
179 return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
180 }
181
182
SetCoderPropertiesOpt(const PROPID * propIDs,const PROPVARIANT * coderProps,UInt32 numProps)183 STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs,
184 const PROPVARIANT *coderProps, UInt32 numProps)
185 {
186 for (UInt32 i = 0; i < numProps; i++)
187 {
188 const PROPVARIANT &prop = coderProps[i];
189 PROPID propID = propIDs[i];
190 if (propID == NCoderPropID::kExpectedDataSize)
191 if (prop.vt == VT_UI8)
192 LzmaEnc_SetDataSize(_encoder, prop.uhVal.QuadPart);
193 }
194 return S_OK;
195 }
196
197
WriteCoderProperties(ISequentialOutStream * outStream)198 STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream)
199 {
200 Byte props[LZMA_PROPS_SIZE];
201 size_t size = LZMA_PROPS_SIZE;
202 RINOK(LzmaEnc_WriteProperties(_encoder, props, &size));
203 return WriteStream(outStream, props, size);
204 }
205
206
207 #define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \
208 if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes;
209
210
211
212 #ifdef LOG_LZMA_THREADS
213
GetTime64(const FILETIME & t)214 static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; }
215
PrintNum(UInt64 val,unsigned numDigits,char c=' ')216 static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ')
217 {
218 char temp[64];
219 char *p = temp + 32;
220 ConvertUInt64ToString(val, p);
221 unsigned len = (unsigned)strlen(p);
222 for (; len < numDigits; len++)
223 *--p = c;
224 printf("%s", p);
225 }
226
PrintTime(const char * s,UInt64 val,UInt64 total)227 static void PrintTime(const char *s, UInt64 val, UInt64 total)
228 {
229 printf(" %s :", s);
230 const UInt32 kFreq = 10000000;
231 UInt64 sec = val / kFreq;
232 PrintNum(sec, 6);
233 printf(" .");
234 UInt32 ms = (UInt32)(val - (sec * kFreq)) / (kFreq / 1000);
235 PrintNum(ms, 3, '0');
236
237 while (val > ((UInt64)1 << 56))
238 {
239 val >>= 1;
240 total >>= 1;
241 }
242
243 UInt64 percent = 0;
244 if (total != 0)
245 percent = val * 100 / total;
246 printf(" =");
247 PrintNum(percent, 4);
248 printf("%%");
249 }
250
251
252 struct CBaseStat
253 {
254 UInt64 kernelTime, userTime;
255
GetNCompress::NLzma::CBaseStat256 BOOL Get(HANDLE thread, const CBaseStat *prevStat)
257 {
258 FILETIME creationTimeFT, exitTimeFT, kernelTimeFT, userTimeFT;
259 BOOL res = GetThreadTimes(thread
260 , &creationTimeFT, &exitTimeFT, &kernelTimeFT, &userTimeFT);
261 if (res)
262 {
263 kernelTime = GetTime64(kernelTimeFT);
264 userTime = GetTime64(userTimeFT);
265 if (prevStat)
266 {
267 kernelTime -= prevStat->kernelTime;
268 userTime -= prevStat->userTime;
269 }
270 }
271 return res;
272 }
273 };
274
275
PrintStat(HANDLE thread,UInt64 totalTime,const CBaseStat * prevStat)276 static void PrintStat(HANDLE thread, UInt64 totalTime, const CBaseStat *prevStat)
277 {
278 CBaseStat newStat;
279 if (!newStat.Get(thread, prevStat))
280 return;
281
282 PrintTime("K", newStat.kernelTime, totalTime);
283
284 const UInt64 processTime = newStat.kernelTime + newStat.userTime;
285
286 PrintTime("U", newStat.userTime, totalTime);
287 PrintTime("S", processTime, totalTime);
288 printf("\n");
289 // PrintTime("G ", totalTime, totalTime);
290 }
291
292 #endif
293
294
295
Code(ISequentialInStream * inStream,ISequentialOutStream * outStream,const UInt64 *,const UInt64 *,ICompressProgressInfo * progress)296 STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
297 const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)
298 {
299 CSeqInStreamWrap inWrap;
300 CSeqOutStreamWrap outWrap;
301 CCompressProgressWrap progressWrap;
302
303 inWrap.Init(inStream);
304 outWrap.Init(outStream);
305 progressWrap.Init(progress);
306
307 #ifdef LOG_LZMA_THREADS
308
309 FILETIME startTimeFT;
310 NWindows::NTime::GetCurUtcFileTime(startTimeFT);
311 UInt64 totalTime = GetTime64(startTimeFT);
312 CBaseStat oldStat;
313 if (!oldStat.Get(GetCurrentThread(), NULL))
314 return E_FAIL;
315
316 #endif
317
318
319 SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt,
320 progress ? &progressWrap.vt : NULL, &g_AlignedAlloc, &g_BigAlloc);
321
322 _inputProcessed = inWrap.Processed;
323
324 RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ)
325 RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE)
326 RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS)
327
328
329 #ifdef LOG_LZMA_THREADS
330
331 NWindows::NTime::GetCurUtcFileTime(startTimeFT);
332 totalTime = GetTime64(startTimeFT) - totalTime;
333 HANDLE lz_threads[2];
334 LzmaEnc_GetLzThreads(_encoder, lz_threads);
335 printf("\n");
336 printf("Main: "); PrintStat(GetCurrentThread(), totalTime, &oldStat);
337 printf("Hash: "); PrintStat(lz_threads[0], totalTime, NULL);
338 printf("BinT: "); PrintStat(lz_threads[1], totalTime, NULL);
339 // PrintTime("Total: ", totalTime, totalTime);
340 printf("\n");
341
342 #endif
343
344 return SResToHRESULT(res);
345 }
346
347 }}
348