1 // ZipUpdate.cpp
2
3 #include "StdAfx.h"
4
5 // #define DEBUG_CACHE
6
7 #ifdef DEBUG_CACHE
8 #include <stdio.h>
9 #define PRF(x) x
10 #else
11 #define PRF(x)
12 #endif
13
14 #include "../../../../C/Alloc.h"
15
16 #include "../../../Common/AutoPtr.h"
17 #include "../../../Common/Defs.h"
18 #include "../../../Common/StringConvert.h"
19
20 #include "../../../Windows/TimeUtils.h"
21 #include "../../../Windows/Thread.h"
22
23 #include "../../Common/CreateCoder.h"
24 #include "../../Common/LimitedStreams.h"
25 #include "../../Common/OutMemStream.h"
26 #include "../../Common/ProgressUtils.h"
27 #ifndef Z7_ST
28 #include "../../Common/ProgressMt.h"
29 #endif
30 #include "../../Common/StreamUtils.h"
31
32 #include "../../Compress/CopyCoder.h"
33 // #include "../../Compress/ZstdEncoderProps.h"
34
35 #include "ZipAddCommon.h"
36 #include "ZipOut.h"
37 #include "ZipUpdate.h"
38
39 using namespace NWindows;
40 using namespace NSynchronization;
41
42 namespace NArchive {
43 namespace NZip {
44
45 static const Byte kHostOS =
46 #ifdef _WIN32
47 NFileHeader::NHostOS::kFAT;
48 #else
49 NFileHeader::NHostOS::kUnix;
50 #endif
51
52 static const Byte kMadeByHostOS = kHostOS;
53
54 // 18.06: now we always write zero to high byte of ExtractVersion field.
55 // Previous versions of p7zip wrote (NFileHeader::NHostOS::kUnix) there, that is not correct
56 static const Byte kExtractHostOS = 0;
57
58 static const Byte kMethodForDirectory = NFileHeader::NCompressionMethod::kStore;
59
60
AddAesExtra(CItem & item,Byte aesKeyMode,UInt16 method)61 static void AddAesExtra(CItem &item, Byte aesKeyMode, UInt16 method)
62 {
63 CWzAesExtra wzAesField;
64 wzAesField.Strength = aesKeyMode;
65 wzAesField.Method = method;
66 item.Method = NFileHeader::NCompressionMethod::kWzAES;
67 item.Crc = 0;
68 CExtraSubBlock sb;
69 wzAesField.SetSubBlock(sb);
70 item.LocalExtra.SubBlocks.Add(sb);
71 item.CentralExtra.SubBlocks.Add(sb);
72 }
73
74
Copy_From_UpdateItem_To_ItemOut(const CUpdateItem & ui,CItemOut & item)75 static void Copy_From_UpdateItem_To_ItemOut(const CUpdateItem &ui, CItemOut &item)
76 {
77 item.Name = ui.Name;
78 item.Name_Utf = ui.Name_Utf;
79 item.Comment = ui.Comment;
80 item.SetUtf8(ui.IsUtf8);
81 // item.SetFlag_AltStream(ui.IsAltStream);
82 // item.ExternalAttrib = ui.Attrib;
83 item.Time = ui.Time;
84 item.Ntfs_MTime = ui.Ntfs_MTime;
85 item.Ntfs_ATime = ui.Ntfs_ATime;
86 item.Ntfs_CTime = ui.Ntfs_CTime;
87
88 item.Write_UnixTime = ui.Write_UnixTime;
89 item.Write_NtfsTime = ui.Write_NtfsTime;
90 }
91
SetFileHeader(const CCompressionMethodMode & options,const CUpdateItem & ui,bool useDescriptor,CItemOut & item)92 static void SetFileHeader(
93 const CCompressionMethodMode &options,
94 const CUpdateItem &ui,
95 bool useDescriptor,
96 CItemOut &item)
97 {
98 item.Size = ui.Size;
99 const bool isDir = ui.IsDir;
100
101 item.ClearFlags();
102
103 if (ui.NewProps)
104 {
105 Copy_From_UpdateItem_To_ItemOut(ui, item);
106 // item.SetFlag_AltStream(ui.IsAltStream);
107 item.ExternalAttrib = ui.Attrib;
108 }
109 /*
110 else
111 isDir = item.IsDir();
112 */
113
114 item.MadeByVersion.HostOS = kMadeByHostOS;
115 item.MadeByVersion.Version = NFileHeader::NCompressionMethod::kMadeByProgramVersion;
116
117 item.ExtractVersion.HostOS = kExtractHostOS;
118
119 item.InternalAttrib = 0; // test it
120 item.SetEncrypted(!isDir && options.Password_Defined);
121 item.SetDescriptorMode(useDescriptor);
122
123 if (isDir)
124 {
125 item.ExtractVersion.Version = NFileHeader::NCompressionMethod::kExtractVersion_Dir;
126 item.Method = kMethodForDirectory;
127 item.PackSize = 0;
128 item.Size = 0;
129 item.Crc = 0;
130 }
131
132 item.LocalExtra.Clear();
133 item.CentralExtra.Clear();
134
135 if (isDir)
136 {
137 item.ExtractVersion.Version = NFileHeader::NCompressionMethod::kExtractVersion_Dir;
138 item.Method = kMethodForDirectory;
139 item.PackSize = 0;
140 item.Size = 0;
141 item.Crc = 0;
142 }
143 else if (options.IsRealAesMode())
144 AddAesExtra(item, options.AesKeyMode, (Byte)(options.MethodSequence.IsEmpty() ? 8 : options.MethodSequence[0]));
145 }
146
147
148 // we call SetItemInfoFromCompressingResult() after SetFileHeader()
149
SetItemInfoFromCompressingResult(const CCompressingResult & compressingResult,bool isAesMode,Byte aesKeyMode,CItem & item)150 static void SetItemInfoFromCompressingResult(const CCompressingResult &compressingResult,
151 bool isAesMode, Byte aesKeyMode, CItem &item)
152 {
153 item.ExtractVersion.Version = compressingResult.ExtractVersion;
154 item.Method = compressingResult.Method;
155 if (compressingResult.Method == NFileHeader::NCompressionMethod::kLZMA && compressingResult.LzmaEos)
156 item.Flags |= NFileHeader::NFlags::kLzmaEOS;
157 item.Crc = compressingResult.CRC;
158 item.Size = compressingResult.UnpackSize;
159 item.PackSize = compressingResult.PackSize;
160
161 item.LocalExtra.Clear();
162 item.CentralExtra.Clear();
163
164 if (isAesMode)
165 AddAesExtra(item, aesKeyMode, compressingResult.Method);
166 }
167
168
169 #ifndef Z7_ST
170
171 struct CMtSem
172 {
173 NWindows::NSynchronization::CSemaphore Semaphore;
174 NWindows::NSynchronization::CCriticalSection CS;
175 CIntVector Indexes;
176 int Head;
177
ReleaseItemNArchive::NZip::CMtSem178 void ReleaseItem(unsigned index)
179 {
180 {
181 CCriticalSectionLock lock(CS);
182 Indexes[index] = Head;
183 Head = (int)index;
184 }
185 Semaphore.Release();
186 }
187
GetFreeItemNArchive::NZip::CMtSem188 int GetFreeItem()
189 {
190 int i;
191 {
192 CCriticalSectionLock lock(CS);
193 i = Head;
194 Head = Indexes[(unsigned)i];
195 }
196 return i;
197 }
198 };
199
200 static THREAD_FUNC_DECL CoderThread(void *threadCoderInfo);
201
202 struct CThreadInfo
203 {
204 DECL_EXTERNAL_CODECS_LOC_VARS_DECL
205
206 NWindows::CThread Thread;
207 NWindows::NSynchronization::CAutoResetEvent CompressEvent;
208 CMtSem *MtSem;
209 unsigned ThreadIndex;
210
211 bool ExitThread;
212
213 CMtCompressProgress *ProgressSpec;
214 CMyComPtr<ICompressProgressInfo> Progress;
215
216 COutMemStream *OutStreamSpec;
217 CMyComPtr<IOutStream> OutStream;
218 CMyComPtr<ISequentialInStream> InStream;
219
220 CAddCommon Coder;
221 HRESULT Result;
222 CCompressingResult CompressingResult;
223
224 bool IsFree;
225 bool InSeqMode;
226 bool OutSeqMode;
227 bool ExpectedDataSize_IsConfirmed;
228
229 UInt32 UpdateIndex;
230 UInt32 FileTime;
231 UInt64 ExpectedDataSize;
232
CThreadInfoNArchive::NZip::CThreadInfo233 CThreadInfo():
234 MtSem(NULL),
235 ExitThread(false),
236 ProgressSpec(NULL),
237 OutStreamSpec(NULL),
238 IsFree(true),
239 InSeqMode(false),
240 OutSeqMode(false),
241 ExpectedDataSize_IsConfirmed(false),
242 FileTime(0),
243 ExpectedDataSize((UInt64)(Int64)-1)
244 {}
245
SetOptionsNArchive::NZip::CThreadInfo246 void SetOptions(const CCompressionMethodMode &options)
247 {
248 Coder.SetOptions(options);
249 }
250
CreateEventsNArchive::NZip::CThreadInfo251 HRESULT CreateEvents()
252 {
253 WRes wres = CompressEvent.CreateIfNotCreated_Reset();
254 return HRESULT_FROM_WIN32(wres);
255 }
256
CreateThreadNArchive::NZip::CThreadInfo257 HRESULT CreateThread()
258 {
259 WRes wres = Thread.Create(CoderThread, this);
260 return HRESULT_FROM_WIN32(wres);
261 }
262
263 void WaitAndCode();
264
StopWait_CloseNArchive::NZip::CThreadInfo265 void StopWait_Close()
266 {
267 ExitThread = true;
268 if (OutStreamSpec)
269 OutStreamSpec->StopWriting(E_ABORT);
270 if (CompressEvent.IsCreated())
271 CompressEvent.Set();
272 Thread.Wait_Close();
273 }
274 };
275
WaitAndCode()276 void CThreadInfo::WaitAndCode()
277 {
278 for (;;)
279 {
280 CompressEvent.Lock();
281 if (ExitThread)
282 return;
283
284 Result = Coder.Compress(
285 EXTERNAL_CODECS_LOC_VARS
286 InStream, OutStream,
287 InSeqMode, OutSeqMode, FileTime, ExpectedDataSize,
288 ExpectedDataSize_IsConfirmed,
289 Progress, CompressingResult);
290
291 if (Result == S_OK && Progress)
292 Result = Progress->SetRatioInfo(&CompressingResult.UnpackSize, &CompressingResult.PackSize);
293
294 MtSem->ReleaseItem(ThreadIndex);
295 }
296 }
297
CoderThread(void * threadCoderInfo)298 static THREAD_FUNC_DECL CoderThread(void *threadCoderInfo)
299 {
300 ((CThreadInfo *)threadCoderInfo)->WaitAndCode();
301 return 0;
302 }
303
304 class CThreads
305 {
306 public:
307 CObjectVector<CThreadInfo> Threads;
~CThreads()308 ~CThreads()
309 {
310 FOR_VECTOR (i, Threads)
311 Threads[i].StopWait_Close();
312 }
313 };
314
315 struct CMemBlocks2: public CMemLockBlocks
316 {
317 bool Skip;
318 bool InSeqMode;
319 bool PreDescriptorMode;
320 bool Finished;
321 CCompressingResult CompressingResult;
322
CMemBlocks2NArchive::NZip::CMemBlocks2323 CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false),
324 CompressingResult() {}
325 };
326
327 class CMemRefs
328 {
329 public:
330 CMemBlockManagerMt *Manager;
331 CObjectVector<CMemBlocks2> Refs;
CMemRefs(CMemBlockManagerMt * manager)332 CMemRefs(CMemBlockManagerMt *manager): Manager(manager) {}
~CMemRefs()333 ~CMemRefs()
334 {
335 FOR_VECTOR (i, Refs)
336 Refs[i].FreeOpt(Manager);
337 }
338 };
339
340
341 Z7_CLASS_IMP_NOQIB_1(
342 CMtProgressMixer2
343 , ICompressProgressInfo
344 )
345 UInt64 ProgressOffset;
346 UInt64 InSizes[2];
347 UInt64 OutSizes[2];
348 CMyComPtr<IProgress> Progress;
349 CMyComPtr<ICompressProgressInfo> RatioProgress;
350 bool _inSizeIsMain;
351 public:
352 NWindows::NSynchronization::CCriticalSection CriticalSection;
353 void Create(IProgress *progress, bool inSizeIsMain);
354 void SetProgressOffset(UInt64 progressOffset);
355 void SetProgressOffset_NoLock(UInt64 progressOffset);
356 HRESULT SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize);
357 };
358
359 void CMtProgressMixer2::Create(IProgress *progress, bool inSizeIsMain)
360 {
361 Progress = progress;
362 Progress.QueryInterface(IID_ICompressProgressInfo, &RatioProgress);
363 _inSizeIsMain = inSizeIsMain;
364 ProgressOffset = InSizes[0] = InSizes[1] = OutSizes[0] = OutSizes[1] = 0;
365 }
366
367 void CMtProgressMixer2::SetProgressOffset_NoLock(UInt64 progressOffset)
368 {
369 InSizes[1] = OutSizes[1] = 0;
370 ProgressOffset = progressOffset;
371 }
372
373 void CMtProgressMixer2::SetProgressOffset(UInt64 progressOffset)
374 {
375 CriticalSection.Enter();
376 SetProgressOffset_NoLock(progressOffset);
377 CriticalSection.Leave();
378 }
379
380 HRESULT CMtProgressMixer2::SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize)
381 {
382 NWindows::NSynchronization::CCriticalSectionLock lock(CriticalSection);
383 if (index == 0 && RatioProgress)
384 {
385 RINOK(RatioProgress->SetRatioInfo(inSize, outSize))
386 }
387 if (inSize)
388 InSizes[index] = *inSize;
389 if (outSize)
390 OutSizes[index] = *outSize;
391 UInt64 v = ProgressOffset + (_inSizeIsMain ?
392 (InSizes[0] + InSizes[1]) :
393 (OutSizes[0] + OutSizes[1]));
394 return Progress->SetCompleted(&v);
395 }
396
397 Z7_COM7F_IMF(CMtProgressMixer2::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize))
398 {
399 return SetRatioInfo(0, inSize, outSize);
400 }
401
402
403 Z7_CLASS_IMP_NOQIB_1(
404 CMtProgressMixer
405 , ICompressProgressInfo
406 )
407 public:
408 CMtProgressMixer2 *Mixer2;
409 CMyComPtr<ICompressProgressInfo> RatioProgress;
410 void Create(IProgress *progress, bool inSizeIsMain);
411 };
412
413 void CMtProgressMixer::Create(IProgress *progress, bool inSizeIsMain)
414 {
415 Mixer2 = new CMtProgressMixer2;
416 RatioProgress = Mixer2;
417 Mixer2->Create(progress, inSizeIsMain);
418 }
419
420 Z7_COM7F_IMF(CMtProgressMixer::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize))
421 {
422 return Mixer2->SetRatioInfo(1, inSize, outSize);
423 }
424
425
426 #endif
427
428 static HRESULT UpdateItemOldData(
429 COutArchive &archive,
430 CInArchive *inArchive,
431 const CItemEx &itemEx,
432 const CUpdateItem &ui,
433 CItemOut &item,
434 /* bool izZip64, */
435 ICompressProgressInfo *progress,
436 IArchiveUpdateCallbackFile *opCallback,
437 UInt64 &complexity)
438 {
439 if (opCallback)
440 {
441 RINOK(opCallback->ReportOperation(
442 NEventIndexType::kInArcIndex, (UInt32)ui.IndexInArc,
443 NUpdateNotifyOp::kReplicate))
444 }
445
446 UInt64 rangeSize;
447
448 RINOK(archive.ClearRestriction())
449
450 if (ui.NewProps)
451 {
452 if (item.HasDescriptor())
453 return E_NOTIMPL;
454
455 // we keep ExternalAttrib and some another properties from old archive
456 // item.ExternalAttrib = ui.Attrib;
457 // if we don't change Comment, we keep Comment from OldProperties
458 Copy_From_UpdateItem_To_ItemOut(ui, item);
459 // item.SetFlag_AltStream(ui.IsAltStream);
460
461 item.CentralExtra.RemoveUnknownSubBlocks();
462 item.LocalExtra.RemoveUnknownSubBlocks();
463
464 archive.WriteLocalHeader(item);
465 rangeSize = item.GetPackSizeWithDescriptor();
466 }
467 else
468 {
469 item.LocalHeaderPos = archive.GetCurPos();
470 rangeSize = itemEx.GetLocalFullSize();
471 }
472
473 CMyComPtr<ISequentialInStream> packStream;
474
475 RINOK(inArchive->GetItemStream(itemEx, ui.NewProps, packStream))
476 if (!packStream)
477 return E_NOTIMPL;
478
479 complexity += rangeSize;
480
481 CMyComPtr<ISequentialOutStream> outStream;
482 archive.CreateStreamForCopying(outStream);
483 HRESULT res = NCompress::CopyStream_ExactSize(packStream, outStream, rangeSize, progress);
484 archive.MoveCurPos(rangeSize);
485 return res;
486 }
487
488
489 static HRESULT WriteDirHeader(COutArchive &archive, const CCompressionMethodMode *options,
490 const CUpdateItem &ui, CItemOut &item)
491 {
492 SetFileHeader(*options, ui, false, item);
493 RINOK(archive.ClearRestriction())
494 archive.WriteLocalHeader(item);
495 return S_OK;
496 }
497
498
499 static void UpdatePropsFromStream(
500 const CUpdateOptions &options,
501 CUpdateItem &item, ISequentialInStream *fileInStream,
502 IArchiveUpdateCallback *updateCallback, UInt64 &totalComplexity)
503 {
504 CMyComPtr<IStreamGetProps> getProps;
505 fileInStream->QueryInterface(IID_IStreamGetProps, (void **)&getProps);
506 UInt64 size = (UInt64)(Int64)-1;
507 bool size_WasSet = false;
508
509 if (getProps)
510 {
511 FILETIME cTime, aTime, mTime;
512 UInt32 attrib;
513 if (getProps->GetProps(&size, &cTime, &aTime, &mTime, &attrib) == S_OK)
514 {
515 if (options.Write_MTime)
516 if (!FILETIME_IsZero(mTime))
517 {
518 item.Ntfs_MTime = mTime;
519 NTime::UtcFileTime_To_LocalDosTime(mTime, item.Time);
520 }
521
522 if (options.Write_CTime) if (!FILETIME_IsZero(cTime)) item.Ntfs_CTime = cTime;
523 if (options.Write_ATime) if (!FILETIME_IsZero(aTime)) item.Ntfs_ATime = aTime;
524
525 item.Attrib = attrib;
526 size_WasSet = true;
527 }
528 }
529
530 if (!size_WasSet)
531 {
532 CMyComPtr<IStreamGetSize> streamGetSize;
533 fileInStream->QueryInterface(IID_IStreamGetSize, (void **)&streamGetSize);
534 if (streamGetSize)
535 {
536 if (streamGetSize->GetSize(&size) == S_OK)
537 size_WasSet = true;
538 }
539 }
540
541 if (size_WasSet && size != (UInt64)(Int64)-1)
542 {
543 item.Size_WasSetFromStream = true;
544 if (size != item.Size)
545 {
546 const Int64 newComplexity = (Int64)totalComplexity + ((Int64)size - (Int64)item.Size);
547 if (newComplexity > 0)
548 {
549 totalComplexity = (UInt64)newComplexity;
550 updateCallback->SetTotal(totalComplexity);
551 }
552 item.Size = size;
553 }
554 }
555 }
556
557
558 /*
559 static HRESULT ReportProps(
560 IArchiveUpdateCallbackArcProp *reportArcProp,
561 UInt32 index,
562 const CItemOut &item,
563 bool isAesMode)
564 {
565 PROPVARIANT prop;
566 prop.vt = VT_EMPTY;
567 prop.wReserved1 = 0;
568
569 NCOM::PropVarEm_Set_UInt64(&prop, item.Size);
570 RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidSize, &prop));
571
572 NCOM::PropVarEm_Set_UInt64(&prop, item.PackSize);
573 RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidPackSize, &prop));
574
575 if (!isAesMode)
576 {
577 NCOM::PropVarEm_Set_UInt32(&prop, item.Crc);
578 RINOK(reportArcProp->ReportProp(NEventIndexType::kOutArcIndex, index, kpidCRC, &prop));
579 }
580
581 RINOK(reportArcProp->ReportFinished(NEventIndexType::kOutArcIndex, index, NUpdate::NOperationResult::kOK));
582
583 // if (opCallback) RINOK(opCallback->ReportOperation(NEventIndexType::kOutArcIndex, index, NUpdateNotifyOp::kOpFinished))
584
585 return S_OK;
586 }
587 */
588
589 /*
590 struct CTotalStats
591 {
592 UInt64 Size;
593 UInt64 PackSize;
594
595 void UpdateWithItem(const CItemOut &item)
596 {
597 Size += item.Size;
598 PackSize += item.PackSize;
599 }
600 };
601
602 static HRESULT ReportArcProps(IArchiveUpdateCallbackArcProp *reportArcProp,
603 CTotalStats &st)
604 {
605 PROPVARIANT prop;
606 prop.vt = VT_EMPTY;
607 prop.wReserved1 = 0;
608 {
609 NWindows::NCOM::PropVarEm_Set_UInt64(&prop, st.Size);
610 RINOK(reportArcProp->ReportProp(
611 NEventIndexType::kArcProp, 0, kpidSize, &prop));
612 }
613 {
614 NWindows::NCOM::PropVarEm_Set_UInt64(&prop, st.PackSize);
615 RINOK(reportArcProp->ReportProp(
616 NEventIndexType::kArcProp, 0, kpidPackSize, &prop));
617 }
618 return S_OK;
619 }
620 */
621
622
623 static HRESULT Update2St(
624 DECL_EXTERNAL_CODECS_LOC_VARS
625 COutArchive &archive,
626 CInArchive *inArchive,
627 const CObjectVector<CItemEx> &inputItems,
628 CObjectVector<CUpdateItem> &updateItems,
629 const CUpdateOptions &updateOptions,
630 const CCompressionMethodMode *options, bool outSeqMode,
631 const CByteBuffer *comment,
632 IArchiveUpdateCallback *updateCallback,
633 UInt64 &totalComplexity,
634 IArchiveUpdateCallbackFile *opCallback
635 // , IArchiveUpdateCallbackArcProp *reportArcProp
636 )
637 {
638 CLocalProgress *lps = new CLocalProgress;
639 CMyComPtr<ICompressProgressInfo> progress = lps;
640 lps->Init(updateCallback, true);
641
642 CAddCommon compressor;
643 compressor.SetOptions(*options);
644
645 CObjectVector<CItemOut> items;
646 UInt64 unpackSizeTotal = 0, packSizeTotal = 0;
647
648 FOR_VECTOR (itemIndex, updateItems)
649 {
650 lps->InSize = unpackSizeTotal;
651 lps->OutSize = packSizeTotal;
652 RINOK(lps->SetCur())
653 CUpdateItem &ui = updateItems[itemIndex];
654 CItemEx itemEx;
655 CItemOut item;
656
657 if (!ui.NewProps || !ui.NewData)
658 {
659 // Note: for (ui.NewProps && !ui.NewData) it copies Props from old archive,
660 // But we will rewrite all important properties later. But we can keep some properties like Comment
661 itemEx = inputItems[(unsigned)ui.IndexInArc];
662 if (inArchive->Read_LocalItem_After_CdItem_Full(itemEx) != S_OK)
663 return E_NOTIMPL;
664 (CItem &)item = itemEx;
665 }
666
667 if (ui.NewData)
668 {
669 // bool isDir = ((ui.NewProps) ? ui.IsDir : item.IsDir());
670 bool isDir = ui.IsDir;
671 if (isDir)
672 {
673 RINOK(WriteDirHeader(archive, options, ui, item))
674 }
675 else
676 {
677 CMyComPtr<ISequentialInStream> fileInStream;
678 {
679 HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream);
680 if (res == S_FALSE)
681 {
682 lps->ProgressOffset += ui.Size;
683 RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK))
684 continue;
685 }
686 RINOK(res)
687 if (!fileInStream)
688 return E_INVALIDARG;
689
690 bool inSeqMode = false;
691 if (!inSeqMode)
692 {
693 CMyComPtr<IInStream> inStream2;
694 fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2);
695 inSeqMode = (inStream2 == NULL);
696 }
697 // seqMode = true; // to test seqMode
698
699 UpdatePropsFromStream(updateOptions, ui, fileInStream, updateCallback, totalComplexity);
700
701 CCompressingResult compressingResult;
702
703 RINOK(compressor.Set_Pre_CompressionResult(
704 inSeqMode, outSeqMode,
705 ui.Size,
706 compressingResult))
707
708 SetFileHeader(*options, ui, compressingResult.DescriptorMode, item);
709
710 // file Size can be 64-bit !!!
711
712 SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item);
713
714 RINOK(archive.SetRestrictionFromCurrent())
715 archive.WriteLocalHeader(item);
716
717 CMyComPtr<IOutStream> outStream;
718 archive.CreateStreamForCompressing(outStream);
719
720 RINOK(compressor.Compress(
721 EXTERNAL_CODECS_LOC_VARS
722 fileInStream, outStream,
723 inSeqMode, outSeqMode,
724 ui.Time,
725 ui.Size, ui.Size_WasSetFromStream,
726 progress, compressingResult))
727
728 if (item.HasDescriptor() != compressingResult.DescriptorMode)
729 return E_FAIL;
730
731 SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item);
732
733 archive.WriteLocalHeader_Replace(item);
734 }
735 // if (reportArcProp) RINOK(ReportProps(reportArcProp, ui.IndexInClient, item, options->IsRealAesMode()))
736 RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK))
737 unpackSizeTotal += item.Size;
738 packSizeTotal += item.PackSize;
739 }
740 }
741 else
742 {
743 UInt64 complexity = 0;
744 lps->SendRatio = false;
745
746 RINOK(UpdateItemOldData(archive, inArchive, itemEx, ui, item, progress, opCallback, complexity))
747
748 lps->SendRatio = true;
749 lps->ProgressOffset += complexity;
750 }
751
752 items.Add(item);
753 lps->ProgressOffset += kLocalHeaderSize;
754 }
755
756 lps->InSize = unpackSizeTotal;
757 lps->OutSize = packSizeTotal;
758 RINOK(lps->SetCur())
759
760 RINOK(archive.WriteCentralDir(items, comment))
761
762 /*
763 CTotalStats stat;
764 stat.Size = unpackSizeTotal;
765 stat.PackSize = packSizeTotal;
766 if (reportArcProp)
767 RINOK(ReportArcProps(reportArcProp, stat))
768 */
769
770 lps->ProgressOffset += kCentralHeaderSize * updateItems.Size() + 1;
771 return lps->SetCur();
772 }
773
774 #ifndef Z7_ST
775
776
777 static const size_t kBlockSize = 1 << 16;
778 // kMemPerThread must be >= kBlockSize
779 //
780 static const size_t kMemPerThread = (size_t)sizeof(size_t) << 23;
781 // static const size_t kMemPerThread = (size_t)sizeof(size_t) << 16; // for debug
782 // static const size_t kMemPerThread = (size_t)1 << 16; // for debug
783
784 /*
785 in:
786 nt_Zip >= 1: the starting maximum number of ZIP threads for search
787 out:
788 nt_Zip: calculated number of ZIP threads
789 returns: calculated number of ZSTD threads
790 */
791 /*
792 static UInt32 CalcThreads_for_ZipZstd(CZstdEncProps *zstdProps,
793 UInt64 memLimit, UInt32 totalThreads,
794 UInt32 &nt_Zip)
795 {
796 for (; nt_Zip > 1; nt_Zip--)
797 {
798 UInt64 mem1 = memLimit / nt_Zip;
799 if (mem1 <= kMemPerThread)
800 continue;
801 mem1 -= kMemPerThread;
802 UInt32 n_ZSTD = ZstdEncProps_GetNumThreads_for_MemUsageLimit(
803 zstdProps, mem1, totalThreads / nt_Zip);
804 // we don't allow (nbWorkers == 1) here
805 if (n_ZSTD <= 1)
806 n_ZSTD = 0;
807 zstdProps->nbWorkers = n_ZSTD;
808 mem1 = ZstdEncProps_GetMemUsage(zstdProps);
809 if ((mem1 + kMemPerThread) * nt_Zip <= memLimit)
810 return n_ZSTD;
811 }
812 return ZstdEncProps_GetNumThreads_for_MemUsageLimit(
813 zstdProps, memLimit, totalThreads);
814 }
815
816
817 static UInt32 SetZstdThreads(
818 const CCompressionMethodMode &options,
819 COneMethodInfo *oneMethodMain,
820 UInt32 numThreads,
821 UInt32 numZipThreads_limit,
822 UInt64 numFilesToCompress,
823 UInt64 numBytesToCompress)
824 {
825 NCompress::NZstd::CEncoderProps encoderProps;
826 RINOK(encoderProps.SetFromMethodProps(*oneMethodMain));
827 CZstdEncProps &zstdProps = encoderProps.EncProps;
828 ZstdEncProps_NormalizeFull(&zstdProps);
829 if (oneMethodMain->FindProp(NCoderPropID::kNumThreads) >= 0)
830 {
831 // threads for ZSTD are fixed
832 if (zstdProps.nbWorkers > 1)
833 numThreads /= zstdProps.nbWorkers;
834 if (numThreads > numZipThreads_limit)
835 numThreads = numZipThreads_limit;
836 if (options._memUsage_WasSet
837 && !options._numThreads_WasForced)
838 {
839 const UInt64 mem1 = ZstdEncProps_GetMemUsage(&zstdProps);
840 const UInt64 numZipThreads = options._memUsage_Compress / (mem1 + kMemPerThread);
841 if (numThreads > numZipThreads)
842 numThreads = (UInt32)numZipThreads;
843 }
844 return numThreads;
845 }
846 {
847 // threads for ZSTD are not fixed
848
849 // calculate estimated required number of ZST threads per file size statistics
850 UInt32 t = MY_ZSTDMT_NBWORKERS_MAX;
851 {
852 UInt64 averageNumberOfBlocks = 0;
853 const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
854 const UInt64 jobSize = zstdProps.jobSize;
855 if (jobSize != 0)
856 averageNumberOfBlocks = averageSize / jobSize + 0;
857 if (t > averageNumberOfBlocks)
858 t = (UInt32)averageNumberOfBlocks;
859 }
860 if (t > numThreads)
861 t = numThreads;
862
863 // calculate the nuber of zip threads
864 UInt32 numZipThreads = numThreads;
865 if (t > 1)
866 numZipThreads = numThreads / t;
867 if (numZipThreads > numZipThreads_limit)
868 numZipThreads = numZipThreads_limit;
869 if (numZipThreads < 1)
870 numZipThreads = 1;
871 {
872 // recalculate the number of ZSTD threads via the number of ZIP threads
873 const UInt32 t2 = numThreads / numZipThreads;
874 if (t < t2)
875 t = t2;
876 }
877
878 if (options._memUsage_WasSet
879 && !options._numThreads_WasForced)
880 {
881 t = CalcThreads_for_ZipZstd(&zstdProps,
882 options._memUsage_Compress, numThreads, numZipThreads);
883 numThreads = numZipThreads;
884 }
885 // we don't use (nbWorkers = 1) here
886 if (t <= 1)
887 t = 0;
888 oneMethodMain->AddProp_NumThreads(t);
889 return numThreads;
890 }
891 }
892 */
893
894 #endif
895
896
897
898
899 static HRESULT Update2(
900 DECL_EXTERNAL_CODECS_LOC_VARS
901 COutArchive &archive,
902 CInArchive *inArchive,
903 const CObjectVector<CItemEx> &inputItems,
904 CObjectVector<CUpdateItem> &updateItems,
905 const CUpdateOptions &updateOptions,
906 const CCompressionMethodMode &options, bool outSeqMode,
907 const CByteBuffer *comment,
908 IArchiveUpdateCallback *updateCallback)
909 {
910 CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
911 updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
912
913 /*
914 CMyComPtr<IArchiveUpdateCallbackArcProp> reportArcProp;
915 updateCallback->QueryInterface(IID_IArchiveUpdateCallbackArcProp, (void **)&reportArcProp);
916 */
917
918 bool unknownComplexity = false;
919 UInt64 complexity = 0;
920 #ifndef Z7_ST
921 UInt64 numFilesToCompress = 0;
922 UInt64 numBytesToCompress = 0;
923 #endif
924
925 unsigned i;
926
927 for (i = 0; i < updateItems.Size(); i++)
928 {
929 const CUpdateItem &ui = updateItems[i];
930 if (ui.NewData)
931 {
932 if (ui.Size == (UInt64)(Int64)-1)
933 unknownComplexity = true;
934 else
935 complexity += ui.Size;
936 #ifndef Z7_ST
937 numBytesToCompress += ui.Size;
938 numFilesToCompress++;
939 #endif
940 /*
941 if (ui.Commented)
942 complexity += ui.CommentRange.Size;
943 */
944 }
945 else
946 {
947 CItemEx inputItem = inputItems[(unsigned)ui.IndexInArc];
948 if (inArchive->Read_LocalItem_After_CdItem_Full(inputItem) != S_OK)
949 return E_NOTIMPL;
950 complexity += inputItem.GetLocalFullSize();
951 // complexity += inputItem.GetCentralExtraPlusCommentSize();
952 }
953 complexity += kLocalHeaderSize;
954 complexity += kCentralHeaderSize;
955 }
956
957 if (comment)
958 complexity += comment->Size();
959 complexity++; // end of central
960
961 if (!unknownComplexity)
962 updateCallback->SetTotal(complexity);
963
964 UInt64 totalComplexity = complexity;
965
966 CCompressionMethodMode options2 = options;
967
968 if (options2._methods.IsEmpty())
969 {
970 // we need method item, if default method was used
971 options2._methods.AddNew();
972 }
973
974 CAddCommon compressor;
975 compressor.SetOptions(options2);
976
977 complexity = 0;
978
979 const Byte method = options.MethodSequence.FrontItem();
980
981 COneMethodInfo *oneMethodMain = NULL;
982 if (!options2._methods.IsEmpty())
983 oneMethodMain = &options2._methods[0];
984
985 {
986 FOR_VECTOR (mi, options2._methods)
987 {
988 options2.SetGlobalLevelTo(options2._methods[mi]);
989 }
990 }
991
992 if (oneMethodMain)
993 {
994 // appnote recommends to use EOS marker for LZMA.
995 if (method == NFileHeader::NCompressionMethod::kLZMA)
996 oneMethodMain->AddProp_EndMarker_if_NotFound(true);
997 }
998
999
1000 #ifndef Z7_ST
1001
1002 UInt32 numThreads = options._numThreads;
1003
1004 UInt32 numZipThreads_limit = numThreads;
1005 if (numZipThreads_limit > numFilesToCompress)
1006 numZipThreads_limit = (UInt32)numFilesToCompress;
1007
1008 if (numZipThreads_limit > 1)
1009 {
1010 const unsigned numFiles_OPEN_MAX = NSystem::Get_File_OPEN_MAX_Reduced_for_3_tasks();
1011 // printf("\nzip:numFiles_OPEN_MAX =%d\n", (unsigned)numFiles_OPEN_MAX);
1012 if (numZipThreads_limit > numFiles_OPEN_MAX)
1013 numZipThreads_limit = (UInt32)numFiles_OPEN_MAX;
1014 }
1015
1016 {
1017 const UInt32 kNumMaxThreads =
1018 #ifdef _WIN32
1019 64; // _WIN32 supports only 64 threads in one group. So no need for more threads here
1020 #else
1021 128;
1022 #endif
1023 if (numThreads > kNumMaxThreads)
1024 numThreads = kNumMaxThreads;
1025 }
1026 /*
1027 if (numThreads > MAXIMUM_WAIT_OBJECTS) // is 64 in Windows
1028 numThreads = MAXIMUM_WAIT_OBJECTS;
1029 */
1030
1031
1032 /*
1033 // zstd supports (numThreads == 0);
1034 if (numThreads < 1)
1035 numThreads = 1;
1036 */
1037
1038 bool mtMode = (numThreads > 1);
1039
1040 if (numFilesToCompress <= 1)
1041 mtMode = false;
1042
1043 // mtMode = true; // debug: to test mtMode
1044
1045 if (!mtMode)
1046 {
1047 // if (oneMethodMain) {
1048 /*
1049 if (method == NFileHeader::NCompressionMethod::kZstdWz)
1050 {
1051 if (oneMethodMain->FindProp(NCoderPropID::kNumThreads) < 0)
1052 {
1053 // numZstdThreads was not forced in oneMethodMain
1054 if (numThreads >= 1
1055 && options._memUsage_WasSet
1056 && !options._numThreads_WasForced)
1057 {
1058 NCompress::NZstd::CEncoderProps encoderProps;
1059 RINOK(encoderProps.SetFromMethodProps(*oneMethodMain))
1060 CZstdEncProps &zstdProps = encoderProps.EncProps;
1061 ZstdEncProps_NormalizeFull(&zstdProps);
1062 numThreads = ZstdEncProps_GetNumThreads_for_MemUsageLimit(
1063 &zstdProps, options._memUsage_Compress, numThreads);
1064 // we allow (nbWorkers = 1) here.
1065 }
1066 oneMethodMain->AddProp_NumThreads(numThreads);
1067 }
1068 } // kZstdWz
1069 */
1070 // } // oneMethodMain
1071
1072 FOR_VECTOR (mi, options2._methods)
1073 {
1074 COneMethodInfo &onem = options2._methods[mi];
1075
1076 if (onem.FindProp(NCoderPropID::kNumThreads) < 0)
1077 {
1078 // fixme: we should check the number of threads for xz method also
1079 // fixed for 9.31. bzip2 default is just one thread.
1080 onem.AddProp_NumThreads(numThreads);
1081 }
1082 }
1083 }
1084 else // mtMode
1085 {
1086 if (method == NFileHeader::NCompressionMethod::kStore && !options.Password_Defined)
1087 numThreads = 1;
1088
1089 if (oneMethodMain)
1090 {
1091
1092 if (method == NFileHeader::NCompressionMethod::kBZip2)
1093 {
1094 bool fixedNumber;
1095 UInt32 numBZip2Threads = oneMethodMain->Get_BZip2_NumThreads(fixedNumber);
1096 if (!fixedNumber)
1097 {
1098 const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
1099 const UInt32 blockSize = oneMethodMain->Get_BZip2_BlockSize();
1100 const UInt64 averageNumberOfBlocks = averageSize / blockSize + 1;
1101 numBZip2Threads = 64;
1102 if (numBZip2Threads > averageNumberOfBlocks)
1103 numBZip2Threads = (UInt32)averageNumberOfBlocks;
1104 if (numBZip2Threads > numThreads)
1105 numBZip2Threads = numThreads;
1106 oneMethodMain->AddProp_NumThreads(numBZip2Threads);
1107 }
1108 numThreads /= numBZip2Threads;
1109 }
1110 else if (method == NFileHeader::NCompressionMethod::kXz)
1111 {
1112 UInt32 numLzmaThreads = 1;
1113 int numXzThreads = oneMethodMain->Get_Xz_NumThreads(numLzmaThreads);
1114 if (numXzThreads < 0)
1115 {
1116 // numXzThreads is unknown
1117 const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
1118 const UInt64 blockSize = oneMethodMain->Get_Xz_BlockSize();
1119 UInt64 averageNumberOfBlocks = 1;
1120 if (blockSize != (UInt64)(Int64)-1)
1121 averageNumberOfBlocks = averageSize / blockSize + 1;
1122 UInt32 t = 256;
1123 if (t > averageNumberOfBlocks)
1124 t = (UInt32)averageNumberOfBlocks;
1125 t *= numLzmaThreads;
1126 if (t > numThreads)
1127 t = numThreads;
1128 oneMethodMain->AddProp_NumThreads(t);
1129 numXzThreads = (int)t;
1130 }
1131 numThreads /= (unsigned)numXzThreads;
1132 }
1133 /*
1134 else if (method == NFileHeader::NCompressionMethod::kZstdWz)
1135 {
1136 numThreads = SetZstdThreads(options,
1137 oneMethodMain, numThreads,
1138 numZipThreads_limit,
1139 numFilesToCompress, numBytesToCompress);
1140 }
1141 */
1142 else if (
1143 method == NFileHeader::NCompressionMethod::kDeflate
1144 || method == NFileHeader::NCompressionMethod::kDeflate64
1145 || method == NFileHeader::NCompressionMethod::kPPMd)
1146 {
1147 if (numThreads > 1
1148 && options._memUsage_WasSet
1149 && !options._numThreads_WasForced)
1150 {
1151 UInt64 methodMemUsage;
1152 if (method == NFileHeader::NCompressionMethod::kPPMd)
1153 methodMemUsage = oneMethodMain->Get_Ppmd_MemSize();
1154 else
1155 methodMemUsage = (4 << 20); // for deflate
1156 const UInt64 threadMemUsage = kMemPerThread + methodMemUsage;
1157 const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage;
1158 if (numThreads64 < numThreads)
1159 numThreads = (UInt32)numThreads64;
1160 }
1161 }
1162 else if (method == NFileHeader::NCompressionMethod::kLZMA)
1163 {
1164 // we suppose that default LZMA is 2 thread. So we don't change it
1165 const UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads();
1166 numThreads /= numLZMAThreads;
1167
1168 if (numThreads > 1
1169 && options._memUsage_WasSet
1170 && !options._numThreads_WasForced)
1171 {
1172 const UInt64 methodMemUsage = oneMethodMain->Get_Lzma_MemUsage(true);
1173 const UInt64 threadMemUsage = kMemPerThread + methodMemUsage;
1174 const UInt64 numThreads64 = options._memUsage_Compress / threadMemUsage;
1175 if (numThreads64 < numThreads)
1176 numThreads = (UInt32)numThreads64;
1177 }
1178 }
1179 } // (oneMethodMain)
1180
1181 if (numThreads > numZipThreads_limit)
1182 numThreads = numZipThreads_limit;
1183 if (numThreads <= 1)
1184 {
1185 mtMode = false;
1186 numThreads = 1;
1187 }
1188 }
1189
1190 // mtMode = true; // to test mtMode for seqMode
1191
1192 if (!mtMode)
1193 #endif
1194 return Update2St(
1195 EXTERNAL_CODECS_LOC_VARS
1196 archive, inArchive,
1197 inputItems, updateItems,
1198 updateOptions,
1199 &options2, outSeqMode,
1200 comment, updateCallback, totalComplexity,
1201 opCallback
1202 // , reportArcProp
1203 );
1204
1205
1206 #ifndef Z7_ST
1207
1208 /*
1209 CTotalStats stat;
1210 stat.Size = 0;
1211 stat.PackSize = 0;
1212 */
1213 if (numThreads < 1)
1214 numThreads = 1;
1215
1216 CObjectVector<CItemOut> items;
1217
1218 CMtProgressMixer *mtProgressMixerSpec = new CMtProgressMixer;
1219 CMyComPtr<ICompressProgressInfo> progress = mtProgressMixerSpec;
1220 mtProgressMixerSpec->Create(updateCallback, true);
1221
1222 CMtCompressProgressMixer mtCompressProgressMixer;
1223 mtCompressProgressMixer.Init(numThreads, mtProgressMixerSpec->RatioProgress);
1224
1225 CMemBlockManagerMt memManager(kBlockSize);
1226 CMemRefs refs(&memManager);
1227
1228 CMtSem mtSem;
1229 CThreads threads;
1230 mtSem.Head = -1;
1231 mtSem.Indexes.ClearAndSetSize(numThreads);
1232 {
1233 WRes wres = mtSem.Semaphore.Create(0, numThreads);
1234 if (wres != 0)
1235 return HRESULT_FROM_WIN32(wres);
1236 }
1237
1238 CUIntVector threadIndices; // list threads in order of updateItems
1239
1240 {
1241 RINOK(memManager.AllocateSpaceAlways((size_t)numThreads * (kMemPerThread / kBlockSize)))
1242 for (i = 0; i < updateItems.Size(); i++)
1243 refs.Refs.Add(CMemBlocks2());
1244
1245 for (i = 0; i < numThreads; i++)
1246 {
1247 threads.Threads.AddNew();
1248 // mtSem.Indexes[i] = -1; // actually we don't use these values
1249 }
1250
1251 for (i = 0; i < numThreads; i++)
1252 {
1253 CThreadInfo &threadInfo = threads.Threads[i];
1254 threadInfo.ThreadIndex = i;
1255 threadInfo.SetOptions(options2);
1256 #ifdef Z7_EXTERNAL_CODECS
1257 threadInfo._externalCodecs = _externalCodecs;
1258 #endif
1259 RINOK(threadInfo.CreateEvents())
1260 threadInfo.OutStreamSpec = new COutMemStream(&memManager);
1261 RINOK(threadInfo.OutStreamSpec->CreateEvents(SYNC_WFMO(&memManager.Synchro)))
1262 threadInfo.OutStream = threadInfo.OutStreamSpec;
1263 threadInfo.ProgressSpec = new CMtCompressProgress();
1264 threadInfo.Progress = threadInfo.ProgressSpec;
1265 threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, i);
1266 threadInfo.MtSem = &mtSem;
1267 RINOK(threadInfo.CreateThread())
1268 }
1269 }
1270
1271 unsigned mtItemIndex = 0;
1272 unsigned itemIndex = 0;
1273 int lastRealStreamItemIndex = -1;
1274
1275
1276 while (itemIndex < updateItems.Size())
1277 {
1278 if (threadIndices.Size() < numThreads && mtItemIndex < updateItems.Size())
1279 {
1280 // we start ahead the threads for compressing
1281 // also we set refs.Refs[itemIndex].SeqMode that is used later
1282 // don't move that code block
1283
1284 CUpdateItem &ui = updateItems[mtItemIndex++];
1285 if (!ui.NewData)
1286 continue;
1287 CItemEx itemEx;
1288 CItemOut item;
1289
1290 if (ui.NewProps)
1291 {
1292 if (ui.IsDir)
1293 continue;
1294 }
1295 else
1296 {
1297 itemEx = inputItems[(unsigned)ui.IndexInArc];
1298 if (inArchive->Read_LocalItem_After_CdItem_Full(itemEx) != S_OK)
1299 return E_NOTIMPL;
1300 (CItem &)item = itemEx;
1301 if (item.IsDir() != ui.IsDir)
1302 return E_NOTIMPL;
1303 if (ui.IsDir)
1304 continue;
1305 }
1306
1307 CMyComPtr<ISequentialInStream> fileInStream;
1308
1309 CMemBlocks2 &memRef2 = refs.Refs[mtItemIndex - 1];
1310
1311 {
1312 NWindows::NSynchronization::CCriticalSectionLock lock(mtProgressMixerSpec->Mixer2->CriticalSection);
1313 const HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream);
1314 if (res == S_FALSE)
1315 {
1316 complexity += ui.Size;
1317 complexity += kLocalHeaderSize;
1318 mtProgressMixerSpec->Mixer2->SetProgressOffset_NoLock(complexity);
1319 RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK))
1320 memRef2.Skip = true;
1321 continue;
1322 }
1323 RINOK(res)
1324 if (!fileInStream)
1325 return E_INVALIDARG;
1326 UpdatePropsFromStream(updateOptions, ui, fileInStream, updateCallback, totalComplexity);
1327 RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK))
1328 }
1329
1330 UInt32 k;
1331 for (k = 0; k < numThreads; k++)
1332 if (threads.Threads[k].IsFree)
1333 break;
1334
1335 if (k == numThreads)
1336 return E_FAIL;
1337 {
1338 {
1339 CThreadInfo &threadInfo = threads.Threads[k];
1340 threadInfo.IsFree = false;
1341 threadInfo.InStream = fileInStream;
1342
1343 bool inSeqMode = false;
1344
1345 if (!inSeqMode)
1346 {
1347 CMyComPtr<IInStream> inStream2;
1348 fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2);
1349 inSeqMode = (inStream2 == NULL);
1350 }
1351 memRef2.InSeqMode = inSeqMode;
1352
1353 // !!!!! we must release ref before sending event
1354 // BUG was here in v4.43 and v4.44. It could change ref counter in two threads in same time
1355 fileInStream.Release();
1356
1357 threadInfo.OutStreamSpec->Init();
1358 threadInfo.ProgressSpec->Reinit();
1359
1360 threadInfo.UpdateIndex = mtItemIndex - 1;
1361 threadInfo.InSeqMode = inSeqMode;
1362 threadInfo.OutSeqMode = outSeqMode;
1363 threadInfo.FileTime = ui.Time; // FileTime is used for ZipCrypto only in seqMode
1364 threadInfo.ExpectedDataSize = ui.Size;
1365 threadInfo.ExpectedDataSize_IsConfirmed = ui.Size_WasSetFromStream;
1366
1367 threadInfo.CompressEvent.Set();
1368
1369 threadIndices.Add(k);
1370 }
1371 }
1372
1373 continue;
1374 }
1375
1376 if (refs.Refs[itemIndex].Skip)
1377 {
1378 itemIndex++;
1379 continue;
1380 }
1381
1382 const CUpdateItem &ui = updateItems[itemIndex];
1383
1384 CItemEx itemEx;
1385 CItemOut item;
1386
1387 if (!ui.NewProps || !ui.NewData)
1388 {
1389 itemEx = inputItems[(unsigned)ui.IndexInArc];
1390 if (inArchive->Read_LocalItem_After_CdItem_Full(itemEx) != S_OK)
1391 return E_NOTIMPL;
1392 (CItem &)item = itemEx;
1393 }
1394
1395 if (ui.NewData)
1396 {
1397 // bool isDir = ((ui.NewProps) ? ui.IsDir : item.IsDir());
1398 const bool isDir = ui.IsDir;
1399
1400 if (isDir)
1401 {
1402 RINOK(WriteDirHeader(archive, &options, ui, item))
1403 }
1404 else
1405 {
1406 CMemBlocks2 &memRef = refs.Refs[itemIndex];
1407
1408 if (memRef.Finished)
1409 {
1410 if (lastRealStreamItemIndex < (int)itemIndex)
1411 lastRealStreamItemIndex = (int)itemIndex;
1412
1413 SetFileHeader(options, ui, memRef.CompressingResult.DescriptorMode, item);
1414
1415 // the BUG was fixed in 9.26:
1416 // SetItemInfoFromCompressingResult must be after SetFileHeader
1417 // to write correct Size.
1418
1419 SetItemInfoFromCompressingResult(memRef.CompressingResult,
1420 options.IsRealAesMode(), options.AesKeyMode, item);
1421 RINOK(archive.ClearRestriction())
1422 archive.WriteLocalHeader(item);
1423 // RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK));
1424 CMyComPtr<ISequentialOutStream> outStream;
1425 archive.CreateStreamForCopying(outStream);
1426 memRef.WriteToStream(memManager.GetBlockSize(), outStream);
1427 // v23: we fixed the bug: we need to write descriptor also
1428 if (item.HasDescriptor())
1429 {
1430 /* that function doesn't rewrite local header, if item.HasDescriptor().
1431 it just writes descriptor */
1432 archive.WriteLocalHeader_Replace(item);
1433 }
1434 else
1435 archive.MoveCurPos(item.PackSize);
1436 memRef.FreeOpt(&memManager);
1437 /*
1438 if (reportArcProp)
1439 {
1440 stat.UpdateWithItem(item);
1441 RINOK(ReportProps(reportArcProp, ui.IndexInClient, item, options.IsRealAesMode()));
1442 }
1443 */
1444 }
1445 else
1446 {
1447 // current file was not finished
1448
1449 if (lastRealStreamItemIndex < (int)itemIndex)
1450 {
1451 // LocalHeader was not written for current itemIndex still
1452
1453 lastRealStreamItemIndex = (int)itemIndex;
1454
1455 // thread was started before for that item already, and memRef.SeqMode was set
1456
1457 CCompressingResult compressingResult;
1458 RINOK(compressor.Set_Pre_CompressionResult(
1459 memRef.InSeqMode, outSeqMode,
1460 ui.Size,
1461 compressingResult))
1462
1463 memRef.PreDescriptorMode = compressingResult.DescriptorMode;
1464 SetFileHeader(options, ui, compressingResult.DescriptorMode, item);
1465
1466 SetItemInfoFromCompressingResult(compressingResult, options.IsRealAesMode(), options.AesKeyMode, item);
1467
1468 // file Size can be 64-bit !!!
1469 RINOK(archive.SetRestrictionFromCurrent())
1470 archive.WriteLocalHeader(item);
1471 }
1472
1473 {
1474 CThreadInfo &thread = threads.Threads[threadIndices.FrontItem()];
1475 if (!thread.OutStreamSpec->WasUnlockEventSent())
1476 {
1477 CMyComPtr<IOutStream> outStream;
1478 archive.CreateStreamForCompressing(outStream);
1479 thread.OutStreamSpec->SetOutStream(outStream);
1480 thread.OutStreamSpec->SetRealStreamMode();
1481 }
1482 }
1483
1484 const WRes wres = mtSem.Semaphore.Lock();
1485 if (wres != 0)
1486 return HRESULT_FROM_WIN32(wres);
1487
1488 const int ti = mtSem.GetFreeItem();
1489 if (ti < 0)
1490 return E_FAIL;
1491
1492 CThreadInfo &threadInfo = threads.Threads[(unsigned)ti];
1493 threadInfo.InStream.Release();
1494 threadInfo.IsFree = true;
1495 RINOK(threadInfo.Result)
1496
1497 unsigned t = 0;
1498
1499 for (;;)
1500 {
1501 if (t == threadIndices.Size())
1502 return E_FAIL;
1503 if (threadIndices[t] == (unsigned)ti)
1504 break;
1505 t++;
1506 }
1507 threadIndices.Delete(t);
1508
1509 if (t == 0)
1510 {
1511 // if thread for current file was finished.
1512 if (threadInfo.UpdateIndex != itemIndex)
1513 return E_FAIL;
1514
1515 if (memRef.PreDescriptorMode != threadInfo.CompressingResult.DescriptorMode)
1516 return E_FAIL;
1517
1518 RINOK(threadInfo.OutStreamSpec->WriteToRealStream())
1519 threadInfo.OutStreamSpec->ReleaseOutStream();
1520 SetFileHeader(options, ui, threadInfo.CompressingResult.DescriptorMode, item);
1521 SetItemInfoFromCompressingResult(threadInfo.CompressingResult,
1522 options.IsRealAesMode(), options.AesKeyMode, item);
1523
1524 archive.WriteLocalHeader_Replace(item);
1525
1526 /*
1527 if (reportArcProp)
1528 {
1529 stat.UpdateWithItem(item);
1530 RINOK(ReportProps(reportArcProp, ui.IndexInClient, item, options.IsRealAesMode()));
1531 }
1532 */
1533 }
1534 else
1535 {
1536 // it's not current file. So we must store information in array
1537 CMemBlocks2 &memRef2 = refs.Refs[threadInfo.UpdateIndex];
1538 threadInfo.OutStreamSpec->DetachData(memRef2);
1539 memRef2.CompressingResult = threadInfo.CompressingResult;
1540 // memRef2.SeqMode = threadInfo.SeqMode; // it was set before
1541 memRef2.Finished = true;
1542 continue;
1543 }
1544 }
1545 }
1546 }
1547 else
1548 {
1549 RINOK(UpdateItemOldData(archive, inArchive, itemEx, ui, item, progress, opCallback, complexity))
1550 }
1551
1552 items.Add(item);
1553 complexity += kLocalHeaderSize;
1554 mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity);
1555 itemIndex++;
1556 }
1557
1558 RINOK(mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL))
1559
1560 RINOK(archive.WriteCentralDir(items, comment))
1561
1562 /*
1563 if (reportArcProp)
1564 {
1565 RINOK(ReportArcProps(reportArcProp, stat));
1566 }
1567 */
1568
1569 complexity += kCentralHeaderSize * updateItems.Size() + 1;
1570 mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity);
1571 return mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL);
1572
1573 #endif
1574 }
1575
1576 /*
1577 // we need CSeekOutStream, if we need Seek(0, STREAM_SEEK_CUR) for seqential stream
1578 Z7_CLASS_IMP_COM_1(
1579 CSeekOutStream
1580 , IOutStream
1581 )
1582 Z7_IFACE_COM7_IMP(ISequentialOutStream)
1583
1584 CMyComPtr<ISequentialOutStream> _seqStream;
1585 UInt64 _size;
1586 public:
1587 void Init(ISequentialOutStream *seqStream)
1588 {
1589 _size = 0;
1590 _seqStream = seqStream;
1591 }
1592 };
1593
1594 Z7_COM7F_IMF(CSeekOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize))
1595 {
1596 UInt32 realProcessedSize;
1597 const HRESULT result = _seqStream->Write(data, size, &realProcessedSize);
1598 _size += realProcessedSize;
1599 if (processedSize)
1600 *processedSize = realProcessedSize;
1601 return result;
1602 }
1603
1604 Z7_COM7F_IMF(CSeekOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition))
1605 {
1606 if (seekOrigin != STREAM_SEEK_CUR || offset != 0)
1607 return E_NOTIMPL;
1608 if (newPosition)
1609 *newPosition = (UInt64)_size;
1610 return S_OK;
1611 }
1612
1613 Z7_COM7F_IMF(CSeekOutStream::SetSize(UInt64 newSize))
1614 {
1615 UNUSED_VAR(newSize)
1616 return E_NOTIMPL;
1617 }
1618 */
1619
1620 static const size_t kCacheBlockSize = 1 << 20;
1621 static const size_t kCacheSize = kCacheBlockSize << 2;
1622 static const size_t kCacheMask = kCacheSize - 1;
1623
1624 Z7_CLASS_IMP_NOQIB_2(
1625 CCacheOutStream
1626 , IOutStream
1627 , IStreamSetRestriction
1628 )
1629 Z7_IFACE_COM7_IMP(ISequentialOutStream)
1630
1631 HRESULT _hres;
1632 CMyComPtr<ISequentialOutStream> _seqStream;
1633 CMyComPtr<IOutStream> _stream;
1634 CMyComPtr<IStreamSetRestriction> _setRestriction;
1635 Byte *_cache;
1636 size_t _cachedSize;
1637 UInt64 _cachedPos;
1638 UInt64 _virtPos;
1639 UInt64 _virtSize;
1640 UInt64 _phyPos;
1641 UInt64 _phySize;
1642 UInt64 _restrict_begin;
1643 UInt64 _restrict_end;
1644
1645 HRESULT FlushFromCache(size_t size);
1646 HRESULT FlushNonRestrictedBlocks();
1647 HRESULT FlushCache();
1648 HRESULT SetRestriction_ForWrite(size_t writeSize) const;
1649
1650 HRESULT SeekPhy(UInt64 pos)
1651 {
1652 if (pos == _phyPos)
1653 return S_OK;
1654 if (!_stream)
1655 return E_NOTIMPL;
1656 _hres = _stream->Seek((Int64)pos, STREAM_SEEK_SET, &_phyPos);
1657 if (_hres == S_OK && _phyPos != pos)
1658 _hres = E_FAIL;
1659 return _hres;
1660 }
1661
1662 public:
1663 CCacheOutStream(): _cache(NULL) {}
1664 ~CCacheOutStream();
1665 bool Allocate()
1666 {
1667 if (!_cache)
1668 _cache = (Byte *)::MidAlloc(kCacheSize);
1669 return _cache != NULL;
1670 }
1671 HRESULT Init(ISequentialOutStream *seqStream, IOutStream *stream, IStreamSetRestriction *setRestriction);
1672 HRESULT FinalFlush();
1673 };
1674
1675 CCacheOutStream::~CCacheOutStream()
1676 {
1677 ::MidFree(_cache);
1678 }
1679
1680
1681 HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *stream, IStreamSetRestriction *setRestriction)
1682 {
1683 _hres = S_OK;
1684 _cachedSize = 0;
1685 _cachedPos = 0;
1686 _virtPos = 0;
1687 _virtSize = 0;
1688 // by default we have no restriction
1689 _restrict_begin = 0;
1690 _restrict_end = 0;
1691 _seqStream = seqStream;
1692 _stream = stream;
1693 _setRestriction = setRestriction;
1694 if (_stream)
1695 {
1696 RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos))
1697 RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize))
1698 RINOK(_stream->Seek((Int64)_virtPos, STREAM_SEEK_SET, &_virtPos))
1699 }
1700 _phyPos = _virtPos;
1701 _phySize = _virtSize;
1702 return S_OK;
1703 }
1704
1705
1706 /* we call SetRestriction_ForWrite() just before Write() from cache.
1707 (_phyPos == _cachedPos)
1708 (writeSize != 0)
1709 */
1710 HRESULT CCacheOutStream::SetRestriction_ForWrite(size_t writeSize) const
1711 {
1712 if (!_setRestriction)
1713 return S_OK;
1714 PRF(printf("\n-- CCacheOutStream::SetRestriction_ForWrite _cachedPos = 0x%x, writeSize = %d\n", (unsigned)_cachedPos, (unsigned)writeSize));
1715 UInt64 begin = _restrict_begin;
1716 UInt64 end = _restrict_end;
1717 const UInt64 phyPos = _phyPos;
1718 if (phyPos != _cachedPos) return E_FAIL;
1719 if (phyPos == _phySize)
1720 {
1721 // The writing will be to the end of phy stream.
1722 // So we will try to use non-restricted write, if possible.
1723 if (begin == end)
1724 begin = _virtPos; // _virtSize; // it's supposed that (_virtSize == _virtPos)
1725 if (phyPos + writeSize <= begin)
1726 {
1727 // the write is not restricted
1728 PRF(printf("\n+++ write is not restricted \n"));
1729 begin = 0;
1730 end = 0;
1731 }
1732 else
1733 {
1734 if (begin > phyPos)
1735 begin = phyPos;
1736 end = (UInt64)(Int64)-1;
1737 }
1738 }
1739 else
1740 {
1741 // (phyPos != _phySize)
1742 if (begin == end || begin > phyPos)
1743 begin = phyPos;
1744 end = (UInt64)(Int64)-1;
1745 }
1746 return _setRestriction->SetRestriction(begin, end);
1747 }
1748
1749
1750 /* it writes up to (size) bytes from cache.
1751 (size > _cachedSize) is allowed
1752 */
1753 HRESULT CCacheOutStream::FlushFromCache(size_t size)
1754 {
1755 PRF(printf("\n-- CCacheOutStream::FlushFromCache %u\n", (unsigned)size));
1756 if (_hres != S_OK)
1757 return _hres;
1758 if (size == 0 || _cachedSize == 0)
1759 return S_OK;
1760 RINOK(SeekPhy(_cachedPos))
1761 for (;;)
1762 {
1763 // (_phyPos == _cachedPos)
1764 const size_t pos = (size_t)_cachedPos & kCacheMask;
1765 size_t cur = kCacheSize - pos;
1766 cur = MyMin(cur, _cachedSize);
1767 cur = MyMin(cur, size);
1768 _hres = SetRestriction_ForWrite(cur);
1769 RINOK(_hres)
1770 PRF(printf("\n-- CCacheOutStream::WriteFromCache _phyPos = 0x%x, size = %d\n", (unsigned)_phyPos, (unsigned)cur));
1771 _hres = WriteStream(_seqStream, _cache + pos, cur);
1772 RINOK(_hres)
1773 _phyPos += cur;
1774 if (_phySize < _phyPos)
1775 _phySize = _phyPos;
1776 _cachedPos += cur;
1777 _cachedSize -= cur;
1778 size -= cur;
1779 if (size == 0 || _cachedSize == 0)
1780 return S_OK;
1781 }
1782 }
1783
1784
1785 HRESULT CCacheOutStream::FlushNonRestrictedBlocks()
1786 {
1787 for (;;)
1788 {
1789 const size_t size = kCacheBlockSize - ((size_t)_cachedPos & (kCacheBlockSize - 1));
1790 if (_cachedSize < size)
1791 break;
1792 UInt64 begin = _restrict_begin;
1793 if (begin == _restrict_end)
1794 begin = _virtPos;
1795 // we don't flush the data to restricted area
1796 if (_cachedPos + size > begin)
1797 break;
1798 RINOK(FlushFromCache(size))
1799 }
1800 return S_OK;
1801 }
1802
1803
1804 HRESULT CCacheOutStream::FlushCache()
1805 {
1806 return FlushFromCache(_cachedSize);
1807 }
1808
1809 HRESULT CCacheOutStream::FinalFlush()
1810 {
1811 _restrict_begin = 0;
1812 _restrict_end = 0;
1813 RINOK(FlushCache())
1814 if (_stream && _hres == S_OK)
1815 {
1816 if (_virtSize != _phySize)
1817 {
1818 // it's unexpected
1819 RINOK(_stream->SetSize(_virtSize))
1820 _phySize = _virtSize;
1821 }
1822 _hres = SeekPhy(_virtPos);
1823 }
1824 return _hres;
1825 }
1826
1827
1828 Z7_COM7F_IMF(CCacheOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize))
1829 {
1830 PRF(printf("\n==== CCacheOutStream::Write virtPos=0x%x, %u\n", (unsigned)_virtPos, (unsigned)size));
1831
1832 if (processedSize)
1833 *processedSize = 0;
1834 if (size == 0)
1835 return S_OK;
1836 if (_hres != S_OK)
1837 return _hres;
1838
1839 if (_cachedSize != 0)
1840 if (_virtPos < _cachedPos ||
1841 _virtPos > _cachedPos + _cachedSize)
1842 {
1843 RINOK(FlushCache())
1844 }
1845
1846 if (_cachedSize == 0)
1847 _cachedPos = _virtPos;
1848
1849 const size_t pos = (size_t)_virtPos & kCacheMask;
1850 {
1851 const size_t blockRem = kCacheBlockSize - ((size_t)_virtPos & (kCacheBlockSize - 1));
1852 if (size > blockRem)
1853 size = (UInt32)blockRem;
1854 }
1855 // _cachedPos <= _virtPos <= _cachedPos + _cachedSize
1856 const UInt64 cachedRem = _cachedPos + _cachedSize - _virtPos;
1857 if (cachedRem)
1858 {
1859 // _virtPos < _cachedPos + _cachedSize
1860 // we rewrite only existing data in cache. So _cachedSize will be not changed
1861 if (size > cachedRem)
1862 size = (UInt32)cachedRem;
1863 }
1864 else
1865 {
1866 // _virtPos == _cachedPos + _cachedSize
1867 // so we need to add new data to the end of cache
1868 if (_cachedSize == kCacheSize)
1869 {
1870 // cache is full. So we need to flush some part of cache.
1871 // we flush only one block, but we are allowed to flush any size here
1872 RINOK(FlushFromCache(kCacheBlockSize - ((size_t)_cachedPos & (kCacheBlockSize - 1))))
1873 }
1874 // _cachedSize != kCacheSize
1875 // so we have some space for new data in cache
1876 if (_cachedSize == 0)
1877 {
1878 /* this code is optional (for optimization):
1879 we write data directly without cache,
1880 if there is no restriction and we have full block. */
1881 if (_restrict_begin == _restrict_end
1882 && size == kCacheBlockSize)
1883 {
1884 RINOK(SeekPhy(_virtPos))
1885 if (_setRestriction)
1886 {
1887 _hres = _setRestriction->SetRestriction(_restrict_begin, _restrict_end);
1888 RINOK(_hres)
1889 }
1890 PRF(printf("\n-- CCacheOutStream::WriteDirectly _phyPos = 0x%x, size = %d\n", (unsigned)_phyPos, (unsigned)size));
1891 _hres = WriteStream(_seqStream, data, size);
1892 RINOK(_hres)
1893 if (processedSize)
1894 *processedSize = size;
1895 _virtPos += size;
1896 if (_virtSize < _virtPos)
1897 _virtSize = _virtPos;
1898 _phyPos += size;
1899 if (_phySize < _phyPos)
1900 _phySize = _phyPos;
1901 return S_OK;
1902 }
1903 }
1904 else // (_cachedSize != 0)
1905 {
1906 const size_t startPos = (size_t)_cachedPos & kCacheMask;
1907 // we don't allow new data to overwrite old start data in cache.
1908 // (startPos == pos) here means that cache is empty.
1909 // (startPos == pos) is not possible here.
1910 if (startPos > pos)
1911 size = (UInt32)MyMin((size_t)size, (size_t)(startPos - pos));
1912 }
1913 // _virtPos == (_cachedPos + _cachedSize) still
1914 _cachedSize += size;
1915 }
1916
1917 memcpy(_cache + pos, data, size);
1918 if (processedSize)
1919 *processedSize = size;
1920 _virtPos += size;
1921 if (_virtSize < _virtPos)
1922 _virtSize = _virtPos;
1923 return FlushNonRestrictedBlocks();
1924 }
1925
1926
1927 Z7_COM7F_IMF(CCacheOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition))
1928 {
1929 PRF(printf("\n==== CCacheOutStream::Seek seekOrigin=%d Seek =%u\n", seekOrigin, (unsigned)offset));
1930 switch (seekOrigin)
1931 {
1932 case STREAM_SEEK_SET: break;
1933 case STREAM_SEEK_CUR: offset += _virtPos; break;
1934 case STREAM_SEEK_END: offset += _virtSize; break;
1935 default: return STG_E_INVALIDFUNCTION;
1936 }
1937 if (offset < 0)
1938 return HRESULT_WIN32_ERROR_NEGATIVE_SEEK;
1939 _virtPos = (UInt64)offset;
1940 if (newPosition)
1941 *newPosition = (UInt64)offset;
1942 return S_OK;
1943 }
1944
1945
1946 Z7_COM7F_IMF(CCacheOutStream::SetSize(UInt64 newSize))
1947 {
1948 if (_hres != S_OK)
1949 return _hres;
1950
1951 if (newSize <= _cachedPos || _cachedSize == 0)
1952 {
1953 _cachedSize = 0;
1954 _cachedPos = newSize;
1955 }
1956 else
1957 {
1958 // _cachedSize != 0
1959 // newSize > _cachedPos
1960 const UInt64 offset = newSize - _cachedPos;
1961 if (offset <= _cachedSize)
1962 {
1963 // newSize is inside cached block or is touching cached block.
1964 // so we reduce cache
1965 _cachedSize = (size_t)offset;
1966 if (_phySize <= newSize)
1967 return S_OK; // _phySize will be restored later after cache flush
1968 // (_phySize > newSize)
1969 // so we must reduce phyStream size to (newSize) or to (_cachedPos)
1970 // newPhySize = _cachedPos; // optional reduce to _cachedPos
1971 }
1972 else
1973 {
1974 // newSize > _cachedPos + _cachedSize
1975 /* It's possible that we need to write zeros,
1976 if new size is larger than old size.
1977 We don't optimize for possible cases here.
1978 So we just flush the cache. */
1979 _hres = FlushCache();
1980 }
1981 }
1982
1983 _virtSize = newSize;
1984
1985 if (_hres != S_OK)
1986 return _hres;
1987
1988 if (newSize != _phySize)
1989 {
1990 if (!_stream)
1991 return E_NOTIMPL;
1992 // if (_phyPos > newSize)
1993 RINOK(SeekPhy(newSize))
1994 if (_setRestriction)
1995 {
1996 UInt64 begin = _restrict_begin;
1997 UInt64 end = _restrict_end;
1998 if (_cachedSize != 0)
1999 {
2000 if (begin > _cachedPos)
2001 begin = _cachedPos;
2002 end = (UInt64)(Int64)-1;
2003 }
2004 _hres = _setRestriction->SetRestriction(begin, end);
2005 RINOK(_hres)
2006 }
2007 _hres = _stream->SetSize(newSize);
2008 RINOK(_hres)
2009 _phySize = newSize;
2010 }
2011 return S_OK;
2012 }
2013
2014
2015 Z7_COM7F_IMF(CCacheOutStream::SetRestriction(UInt64 begin, UInt64 end))
2016 {
2017 PRF(printf("\n============ CCacheOutStream::SetRestriction 0x%x, %u\n", (unsigned)begin, (unsigned)end));
2018 _restrict_begin = begin;
2019 _restrict_end = end;
2020 return FlushNonRestrictedBlocks();
2021 }
2022
2023
2024
2025 HRESULT Update(
2026 DECL_EXTERNAL_CODECS_LOC_VARS
2027 const CObjectVector<CItemEx> &inputItems,
2028 CObjectVector<CUpdateItem> &updateItems,
2029 ISequentialOutStream *seqOutStream,
2030 CInArchive *inArchive, bool removeSfx,
2031 const CUpdateOptions &updateOptions,
2032 const CCompressionMethodMode &compressionMethodMode,
2033 IArchiveUpdateCallback *updateCallback)
2034 {
2035 /*
2036 // it was tested before
2037 if (inArchive)
2038 {
2039 if (!inArchive->CanUpdate())
2040 return E_NOTIMPL;
2041 }
2042 */
2043
2044 CMyComPtr<IStreamSetRestriction> setRestriction;
2045 seqOutStream->QueryInterface(IID_IStreamSetRestriction, (void **)&setRestriction);
2046 if (setRestriction)
2047 {
2048 RINOK(setRestriction->SetRestriction(0, 0))
2049 }
2050
2051 CMyComPtr<IOutStream> outStream;
2052 CCacheOutStream *cacheStream;
2053 bool outSeqMode;
2054
2055 {
2056 CMyComPtr<IOutStream> outStreamReal;
2057
2058 if (!compressionMethodMode.Force_SeqOutMode)
2059 {
2060 seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStreamReal);
2061 /*
2062 if (!outStreamReal)
2063 return E_NOTIMPL;
2064 */
2065 }
2066
2067 if (inArchive)
2068 {
2069 if (!inArchive->IsMultiVol && inArchive->ArcInfo.Base > 0 && !removeSfx)
2070 {
2071 IInStream *baseStream = inArchive->GetBaseStream();
2072 RINOK(InStream_SeekToBegin(baseStream))
2073 RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, (UInt64)inArchive->ArcInfo.Base, NULL))
2074 }
2075 }
2076
2077 outSeqMode = (outStreamReal == NULL);
2078 if (outSeqMode)
2079 setRestriction.Release();
2080 /* CCacheOutStream works as non-restricted by default.
2081 So we use (setRestriction == NULL) for outSeqMode */
2082 // bool use_cacheStream = true;
2083 // if (use_cacheStream)
2084 {
2085 cacheStream = new CCacheOutStream();
2086 outStream = cacheStream;
2087 if (!cacheStream->Allocate())
2088 return E_OUTOFMEMORY;
2089 RINOK(cacheStream->Init(seqOutStream, outStreamReal, setRestriction))
2090 setRestriction.Release();
2091 if (!outSeqMode)
2092 setRestriction = cacheStream;
2093 }
2094 /*
2095 else if (!outStreamReal)
2096 {
2097 CSeekOutStream *seekOutStream = new CSeekOutStream();
2098 outStream = seekOutStream;
2099 seekOutStream->Init(seqOutStream);
2100 }
2101 else
2102 outStream = outStreamReal;
2103 */
2104 }
2105
2106 COutArchive outArchive;
2107 outArchive.SetRestriction = setRestriction;
2108
2109 RINOK(outArchive.Create(outStream))
2110
2111 if (inArchive)
2112 {
2113 if (!inArchive->IsMultiVol && (Int64)inArchive->ArcInfo.MarkerPos2 > inArchive->ArcInfo.Base)
2114 {
2115 IInStream *baseStream = inArchive->GetBaseStream();
2116 RINOK(InStream_SeekSet(baseStream, (UInt64)inArchive->ArcInfo.Base))
2117 const UInt64 embStubSize = (UInt64)((Int64)inArchive->ArcInfo.MarkerPos2 - inArchive->ArcInfo.Base);
2118 RINOK(NCompress::CopyStream_ExactSize(baseStream, outStream, embStubSize, NULL))
2119 outArchive.MoveCurPos(embStubSize);
2120 }
2121 }
2122
2123 RINOK (Update2(
2124 EXTERNAL_CODECS_LOC_VARS
2125 outArchive, inArchive,
2126 inputItems, updateItems,
2127 updateOptions,
2128 compressionMethodMode, outSeqMode,
2129 inArchive ? &inArchive->ArcInfo.Comment : NULL,
2130 updateCallback))
2131
2132 return cacheStream->FinalFlush();
2133 }
2134
2135 }}
2136