• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // LzmaEncoder.cpp
2 
3 #include "StdAfx.h"
4 
5 #include "../../../C/Alloc.h"
6 
7 #include "../Common/CWrappers.h"
8 #include "../Common/StreamUtils.h"
9 
10 #include "LzmaEncoder.h"
11 
12 // #define LOG_LZMA_THREADS
13 
14 #ifdef LOG_LZMA_THREADS
15 
16 #include <stdio.h>
17 
18 #include "../../Common/IntToString.h"
19 #include "../../Windows/TimeUtils.h"
20 
21 EXTERN_C_BEGIN
22 void LzmaEnc_GetLzThreads(CLzmaEncHandle pp, HANDLE lz_threads[2]);
23 EXTERN_C_END
24 
25 #endif
26 
27 namespace NCompress {
28 namespace NLzma {
29 
CEncoder()30 CEncoder::CEncoder()
31 {
32   _encoder = NULL;
33   _encoder = LzmaEnc_Create(&g_AlignedAlloc);
34   if (!_encoder)
35     throw 1;
36 }
37 
~CEncoder()38 CEncoder::~CEncoder()
39 {
40   if (_encoder)
41     LzmaEnc_Destroy(_encoder, &g_AlignedAlloc, &g_BigAlloc);
42 }
43 
GetLowCharFast(wchar_t c)44 static inline wchar_t GetLowCharFast(wchar_t c)
45 {
46   return c |= 0x20;
47 }
48 
ParseMatchFinder(const wchar_t * s,int * btMode,int * numHashBytes)49 static int ParseMatchFinder(const wchar_t *s, int *btMode, int *numHashBytes)
50 {
51   wchar_t c = GetLowCharFast(*s++);
52   if (c == 'h')
53   {
54     if (GetLowCharFast(*s++) != 'c')
55       return 0;
56     int num = (int)(*s++ - L'0');
57     if (num < 4 || num > 5)
58       return 0;
59     if (*s != 0)
60       return 0;
61     *btMode = 0;
62     *numHashBytes = num;
63     return 1;
64   }
65 
66   if (c != 'b')
67     return 0;
68   {
69     if (GetLowCharFast(*s++) != 't')
70       return 0;
71     int num = (int)(*s++ - L'0');
72     if (num < 2 || num > 5)
73       return 0;
74     if (*s != 0)
75       return 0;
76     *btMode = 1;
77     *numHashBytes = num;
78     return 1;
79   }
80 }
81 
82 #define SET_PROP_32(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = (int)v; break;
83 #define SET_PROP_32U(_id_, _dest_) case NCoderPropID::_id_: ep._dest_ = v; break;
84 
85 HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep);
SetLzmaProp(PROPID propID,const PROPVARIANT & prop,CLzmaEncProps & ep)86 HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep)
87 {
88   if (propID == NCoderPropID::kMatchFinder)
89   {
90     if (prop.vt != VT_BSTR)
91       return E_INVALIDARG;
92     return ParseMatchFinder(prop.bstrVal, &ep.btMode, &ep.numHashBytes) ? S_OK : E_INVALIDARG;
93   }
94 
95   if (propID == NCoderPropID::kAffinity)
96   {
97     if (prop.vt == VT_UI8)
98       ep.affinity = prop.uhVal.QuadPart;
99     else
100       return E_INVALIDARG;
101     return S_OK;
102   }
103 
104   if (propID > NCoderPropID::kReduceSize)
105     return S_OK;
106 
107   if (propID == NCoderPropID::kReduceSize)
108   {
109     if (prop.vt == VT_UI8)
110       ep.reduceSize = prop.uhVal.QuadPart;
111     else
112       return E_INVALIDARG;
113     return S_OK;
114   }
115 
116   if (propID == NCoderPropID::kDictionarySize)
117   {
118     if (prop.vt == VT_UI8)
119     {
120       // 21.03 : we support 64-bit VT_UI8 for dictionary and (dict == 4 GiB)
121       const UInt64 v = prop.uhVal.QuadPart;
122       if (v > ((UInt64)1 << 32))
123         return E_INVALIDARG;
124       UInt32 dict;
125       if (v == ((UInt64)1 << 32))
126         dict = (UInt32)(Int32)-1;
127       else
128         dict = (UInt32)v;
129       ep.dictSize = dict;
130       return S_OK;
131     }
132   }
133 
134   if (prop.vt != VT_UI4)
135     return E_INVALIDARG;
136   UInt32 v = prop.ulVal;
137   switch (propID)
138   {
139     case NCoderPropID::kDefaultProp:
140       if (v > 32)
141         return E_INVALIDARG;
142       ep.dictSize = (v == 32) ? (UInt32)(Int32)-1 : (UInt32)1 << (unsigned)v;
143       break;
144     SET_PROP_32(kLevel, level)
145     SET_PROP_32(kNumFastBytes, fb)
146     SET_PROP_32U(kMatchFinderCycles, mc)
147     SET_PROP_32(kAlgorithm, algo)
148     SET_PROP_32U(kDictionarySize, dictSize)
149     SET_PROP_32(kPosStateBits, pb)
150     SET_PROP_32(kLitPosBits, lp)
151     SET_PROP_32(kLitContextBits, lc)
152     SET_PROP_32(kNumThreads, numThreads)
153     default: return E_INVALIDARG;
154   }
155   return S_OK;
156 }
157 
SetCoderProperties(const PROPID * propIDs,const PROPVARIANT * coderProps,UInt32 numProps)158 STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
159     const PROPVARIANT *coderProps, UInt32 numProps)
160 {
161   CLzmaEncProps props;
162   LzmaEncProps_Init(&props);
163 
164   for (UInt32 i = 0; i < numProps; i++)
165   {
166     const PROPVARIANT &prop = coderProps[i];
167     PROPID propID = propIDs[i];
168     switch (propID)
169     {
170       case NCoderPropID::kEndMarker:
171         if (prop.vt != VT_BOOL)
172           return E_INVALIDARG;
173         props.writeEndMark = (prop.boolVal != VARIANT_FALSE);
174         break;
175       default:
176         RINOK(SetLzmaProp(propID, prop, props));
177     }
178   }
179   return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
180 }
181 
182 
SetCoderPropertiesOpt(const PROPID * propIDs,const PROPVARIANT * coderProps,UInt32 numProps)183 STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs,
184     const PROPVARIANT *coderProps, UInt32 numProps)
185 {
186   for (UInt32 i = 0; i < numProps; i++)
187   {
188     const PROPVARIANT &prop = coderProps[i];
189     PROPID propID = propIDs[i];
190     if (propID == NCoderPropID::kExpectedDataSize)
191       if (prop.vt == VT_UI8)
192         LzmaEnc_SetDataSize(_encoder, prop.uhVal.QuadPart);
193   }
194   return S_OK;
195 }
196 
197 
WriteCoderProperties(ISequentialOutStream * outStream)198 STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream)
199 {
200   Byte props[LZMA_PROPS_SIZE];
201   size_t size = LZMA_PROPS_SIZE;
202   RINOK(LzmaEnc_WriteProperties(_encoder, props, &size));
203   return WriteStream(outStream, props, size);
204 }
205 
206 
207 #define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \
208   if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes;
209 
210 
211 
212 #ifdef LOG_LZMA_THREADS
213 
GetTime64(const FILETIME & t)214 static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; }
215 
PrintNum(UInt64 val,unsigned numDigits,char c=' ')216 static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ')
217 {
218   char temp[64];
219   char *p = temp + 32;
220   ConvertUInt64ToString(val, p);
221   unsigned len = (unsigned)strlen(p);
222   for (; len < numDigits; len++)
223     *--p = c;
224   printf("%s", p);
225 }
226 
PrintTime(const char * s,UInt64 val,UInt64 total)227 static void PrintTime(const char *s, UInt64 val, UInt64 total)
228 {
229   printf("  %s :", s);
230   const UInt32 kFreq = 10000000;
231   UInt64 sec = val / kFreq;
232   PrintNum(sec, 6);
233   printf(" .");
234   UInt32 ms = (UInt32)(val - (sec * kFreq)) / (kFreq / 1000);
235   PrintNum(ms, 3, '0');
236 
237   while (val > ((UInt64)1 << 56))
238   {
239     val >>= 1;
240     total >>= 1;
241   }
242 
243   UInt64 percent = 0;
244   if (total != 0)
245     percent = val * 100 / total;
246   printf("  =");
247   PrintNum(percent, 4);
248   printf("%%");
249 }
250 
251 
252 struct CBaseStat
253 {
254   UInt64 kernelTime, userTime;
255 
GetNCompress::NLzma::CBaseStat256   BOOL Get(HANDLE thread, const CBaseStat *prevStat)
257   {
258     FILETIME creationTimeFT, exitTimeFT, kernelTimeFT, userTimeFT;
259     BOOL res = GetThreadTimes(thread
260       , &creationTimeFT, &exitTimeFT, &kernelTimeFT, &userTimeFT);
261     if (res)
262     {
263       kernelTime = GetTime64(kernelTimeFT);
264       userTime = GetTime64(userTimeFT);
265       if (prevStat)
266       {
267         kernelTime -= prevStat->kernelTime;
268         userTime -= prevStat->userTime;
269       }
270     }
271     return res;
272   }
273 };
274 
275 
PrintStat(HANDLE thread,UInt64 totalTime,const CBaseStat * prevStat)276 static void PrintStat(HANDLE thread, UInt64 totalTime, const CBaseStat *prevStat)
277 {
278   CBaseStat newStat;
279   if (!newStat.Get(thread, prevStat))
280     return;
281 
282   PrintTime("K", newStat.kernelTime, totalTime);
283 
284   const UInt64 processTime = newStat.kernelTime + newStat.userTime;
285 
286   PrintTime("U", newStat.userTime, totalTime);
287   PrintTime("S", processTime, totalTime);
288   printf("\n");
289   // PrintTime("G ", totalTime, totalTime);
290 }
291 
292 #endif
293 
294 
295 
Code(ISequentialInStream * inStream,ISequentialOutStream * outStream,const UInt64 *,const UInt64 *,ICompressProgressInfo * progress)296 STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
297     const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)
298 {
299   CSeqInStreamWrap inWrap;
300   CSeqOutStreamWrap outWrap;
301   CCompressProgressWrap progressWrap;
302 
303   inWrap.Init(inStream);
304   outWrap.Init(outStream);
305   progressWrap.Init(progress);
306 
307   #ifdef LOG_LZMA_THREADS
308 
309   FILETIME startTimeFT;
310   NWindows::NTime::GetCurUtcFileTime(startTimeFT);
311   UInt64 totalTime = GetTime64(startTimeFT);
312   CBaseStat oldStat;
313   if (!oldStat.Get(GetCurrentThread(), NULL))
314     return E_FAIL;
315 
316   #endif
317 
318 
319   SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt,
320       progress ? &progressWrap.vt : NULL, &g_AlignedAlloc, &g_BigAlloc);
321 
322   _inputProcessed = inWrap.Processed;
323 
324   RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ)
325   RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE)
326   RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS)
327 
328 
329   #ifdef LOG_LZMA_THREADS
330 
331   NWindows::NTime::GetCurUtcFileTime(startTimeFT);
332   totalTime = GetTime64(startTimeFT) - totalTime;
333   HANDLE lz_threads[2];
334   LzmaEnc_GetLzThreads(_encoder, lz_threads);
335   printf("\n");
336   printf("Main: ");  PrintStat(GetCurrentThread(), totalTime, &oldStat);
337   printf("Hash: ");  PrintStat(lz_threads[0], totalTime, NULL);
338   printf("BinT: ");  PrintStat(lz_threads[1], totalTime, NULL);
339   // PrintTime("Total: ", totalTime, totalTime);
340   printf("\n");
341 
342   #endif
343 
344   return SResToHRESULT(res);
345 }
346 
347 }}
348