1 // 7zUpdate.cpp
2
3 #include "StdAfx.h"
4
5 #include "../../../../C/CpuArch.h"
6
7 #include "../../../Common/MyLinux.h"
8 #include "../../../Common/StringToInt.h"
9 #include "../../../Common/Wildcard.h"
10
11 #include "../../Common/CreateCoder.h"
12 #include "../../Common/LimitedStreams.h"
13 #include "../../Common/ProgressUtils.h"
14
15 #include "../../Compress/CopyCoder.h"
16
17 #include "../Common/ItemNameUtils.h"
18
19 #include "7zDecode.h"
20 #include "7zEncode.h"
21 #include "7zFolderInStream.h"
22 #include "7zHandler.h"
23 #include "7zOut.h"
24 #include "7zUpdate.h"
25
26 namespace NArchive {
27 namespace N7z {
28
29 #define k_X86 k_BCJ
30
31 struct CFilterMode
32 {
33 UInt32 Id;
34 UInt32 Delta; // required File Size alignment, if Id is not k_Delta.
35 // (Delta == 0) means unknown alignment
36 UInt32 Offset; // for k_ARM64 / k_RISCV
37 // UInt32 AlignSizeOpt; // for k_ARM64
38
CFilterModeNArchive::N7z::CFilterMode39 CFilterMode():
40 Id(0),
41 Delta(0),
42 Offset(0)
43 // , AlignSizeOpt(0)
44 {}
45
ClearFilterModeNArchive::N7z::CFilterMode46 void ClearFilterMode()
47 {
48 Id = 0;
49 Delta = 0;
50 Offset = 0;
51 // AlignSizeOpt = 0;
52 }
53
54 // it sets Delta as Align value, if Id is exe filter
55 // in another cases it sets Delta = 0, that
SetDeltaNArchive::N7z::CFilterMode56 void SetDelta()
57 {
58 if (Id == k_IA64)
59 Delta = 16;
60 else if (Id == k_ARM64 || Id == k_ARM || Id == k_PPC || Id == k_SPARC)
61 Delta = 4;
62 else if (Id == k_ARMT || Id == k_RISCV)
63 Delta = 2;
64 else if (Id == k_BCJ || Id == k_BCJ2)
65 Delta = 1; // do we need it?
66 else
67 Delta = 0;
68 }
69 };
70
71
72 /* ---------- PE ---------- */
73
74 #define MZ_SIG 0x5A4D
75
76 #define PE_SIG 0x00004550
77 #define PE_OptHeader_Magic_32 0x10B
78 #define PE_OptHeader_Magic_64 0x20B
79 // #define PE_SectHeaderSize 40
80 // #define PE_SECT_EXECUTE 0x20000000
81
Parse_EXE(const Byte * buf,size_t size,CFilterMode * filterMode)82 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode)
83 {
84 if (size < 512 || GetUi16(buf) != MZ_SIG)
85 return 0;
86
87 const Byte *p;
88 UInt32 peOffset, optHeaderSize, filterId;
89
90 peOffset = GetUi32(buf + 0x3C);
91 if (peOffset >= 0x1000 || peOffset + 512 > size || (peOffset & 7) != 0)
92 return 0;
93 p = buf + peOffset;
94 if (GetUi32(p) != PE_SIG)
95 return 0;
96 p += 4;
97
98 const unsigned machine = GetUi16(p);
99
100 switch (machine)
101 {
102 case 0x014C:
103 case 0x8664: filterId = k_X86; break;
104 case 0xAA64: filterId = k_ARM64; break;
105
106 /*
107 IMAGE_FILE_MACHINE_ARM 0x01C0 // ARM LE
108 IMAGE_FILE_MACHINE_THUMB 0x01C2 // ARM Thumb / Thumb-2 LE
109 IMAGE_FILE_MACHINE_ARMNT 0x01C4 // ARM Thumb-2, LE
110 Note: We use ARM filter for 0x01C2. (WinCE 5 - 0x01C2) files mostly contain ARM code (not Thumb/Thumb-2).
111 */
112
113 case 0x01C0: // WinCE old
114 case 0x01C2: filterId = k_ARM; break; // WinCE new
115 case 0x01C4: filterId = k_ARMT; break; // WinRT
116
117 case 0x5032: // RISCV32
118 case 0x5064: // RISCV64
119 // case 0x5128: // RISCV128
120 filterId = k_RISCV; break;
121
122 case 0x0200: filterId = k_IA64; break;
123 default: return 0;
124 }
125
126 const UInt32 numSections = GetUi16(p + 2);
127 optHeaderSize = GetUi16(p + 16);
128 if (optHeaderSize > (1 << 10))
129 return 0;
130
131 p += 20; /* headerSize */
132
133 switch (GetUi16(p))
134 {
135 case PE_OptHeader_Magic_32:
136 case PE_OptHeader_Magic_64:
137 break;
138 default:
139 return 0;
140 }
141
142 // Windows exe file sizes are not aligned for 4 KiB.
143 // So we can't use (CFilterMode::Offset != 0) in solid archives.
144 // So we just don't set Offset here.
145 #define NUM_SCAN_SECTIONS_MAX (1 << 6)
146 // #define EXE_SECTION_OFFSET_MAX (1 << 27)
147 // #define EXE_SECTION_SIZE_MIN (1 << 8)
148 // #define EXE_SECTION_SIZE_MAX (1 << 27)
149 #define PE_SectHeaderSize 40
150 // #define PE_SECT_EXECUTE 0x20000000
151
152 /*
153 if (numSections > NUM_SCAN_SECTIONS_MAX)
154 return 0;
155 */
156
157 if ((size_t)(p - buf) + optHeaderSize <= size)
158 {
159 p += optHeaderSize;
160 /*
161 // UInt32 numExeSections = 0;
162 // bool execute_finded = false;
163 // UInt32 sect_va = 0;
164 // UInt32 sect_size = 0;
165 // UInt32 sect_offset = 0;
166 */
167 if (numSections <= NUM_SCAN_SECTIONS_MAX)
168 if (machine == 0x8664)
169 for (UInt32 i = 0; i < numSections
170 ; i++, p += PE_SectHeaderSize)
171 {
172 // UInt32 characts, rawSize, offset;
173 if ((UInt32)(p - buf) + PE_SectHeaderSize > size)
174 {
175 // return 0;
176 break;
177 }
178 if (memcmp(p, ".a64xrm", 8) == 0)
179 {
180 // ARM64EC
181 filterId = k_ARM64;
182 break;
183 }
184 /*
185 rawSize = GetUi32(p + 16);
186 offset = GetUi32(p + 20);
187 characts = GetUi32(p + 36);
188 if (rawSize >= EXE_SECTION_SIZE_MIN &&
189 rawSize <= EXE_SECTION_SIZE_MAX &&
190 offset <= EXE_SECTION_OFFSET_MAX &&
191 // offset < limit &&
192 offset > 0)
193 {
194 if ((characts & PE_SECT_EXECUTE) != 0)
195 {
196 // execute_finded = true;
197 // sect_va = GetUi32(p + 12);
198 // sect_size = rawSize;
199 // sect_offset = offset;
200 break;
201 }
202 }
203 */
204 }
205 }
206
207 /*
208 filterMode->Offset = 0;
209 if (filterId == k_ARM64)
210 {
211 // filterMode->AlignSizeOpt = (1 << 12);
212 // const UInt32 offs = (sect_va - sect_offset) & 0xFFF;
213 // if (offs != 0)
214 // filterMode->Offset = offs; // change it
215 }
216 */
217 filterMode->Id = filterId;
218 return 1;
219 }
220
221
222 /* ---------- ELF ---------- */
223
224 #define ELF_SIG 0x464C457F
225
226 #define ELF_CLASS_32 1
227 #define ELF_CLASS_64 2
228
229 #define ELF_DATA_2LSB 1
230 #define ELF_DATA_2MSB 2
231
Get16(const Byte * p,BoolInt be)232 static UInt16 Get16(const Byte *p, BoolInt be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); }
Get32(const Byte * p,BoolInt be)233 static UInt32 Get32(const Byte *p, BoolInt be) { if (be) return GetBe32(p); return GetUi32(p); }
234 // static UInt64 Get64(const Byte *p, BoolInt be) { if (be) return GetBe64(p); return GetUi64(p); }
235
Parse_ELF(const Byte * buf,size_t size,CFilterMode * filterMode)236 static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode)
237 {
238 BoolInt /* is32, */ be;
239 UInt32 filterId;
240
241 if (size < 512 || buf[6] != 1) /* ver */
242 return 0;
243
244 if (GetUi32(buf) != ELF_SIG)
245 return 0;
246
247 switch (buf[4])
248 {
249 case ELF_CLASS_32: /* is32 = True; */ break;
250 case ELF_CLASS_64: /* is32 = False; */ break;
251 default: return 0;
252 }
253
254 switch (buf[5])
255 {
256 case ELF_DATA_2LSB: be = False; break;
257 case ELF_DATA_2MSB: be = True; break;
258 default: return 0;
259 }
260
261 switch (Get16(buf + 0x12, be))
262 {
263 case 3:
264 case 6:
265 case 62: filterId = k_X86; break;
266 case 2:
267 case 18:
268 case 43: filterId = k_SPARC; break;
269 case 20:
270 case 21: if (!be) return 0; filterId = k_PPC; break;
271 case 40: if (be) return 0; filterId = k_ARM; break;
272 case 183: if (be) return 0; filterId = k_ARM64; break;
273 case 243: if (be) return 0; filterId = k_RISCV; break;
274
275 /* Some IA-64 ELF executables have size that is not aligned for 16 bytes.
276 So we don't use IA-64 filter for IA-64 ELF */
277 // case 50: if ( be) return 0; filterId = k_IA64; break;
278
279 default: return 0;
280 }
281
282 filterMode->Id = filterId;
283 return 1;
284 }
285
286
287
288 /* ---------- Mach-O ---------- */
289
290 #define MACH_SIG_BE_32 0xCEFAEDFE
291 #define MACH_SIG_BE_64 0xCFFAEDFE
292 #define MACH_SIG_LE_32 0xFEEDFACE
293 #define MACH_SIG_LE_64 0xFEEDFACF
294
295 #define MACH_ARCH_ABI64 (1 << 24)
296 #define MACH_MACHINE_386 7
297 #define MACH_MACHINE_ARM 12
298 #define MACH_MACHINE_SPARC 14
299 #define MACH_MACHINE_PPC 18
300 #define MACH_MACHINE_PPC64 (MACH_ARCH_ABI64 | MACH_MACHINE_PPC)
301 #define MACH_MACHINE_AMD64 (MACH_ARCH_ABI64 | MACH_MACHINE_386)
302 #define MACH_MACHINE_ARM64 (MACH_ARCH_ABI64 | MACH_MACHINE_ARM)
303
Parse_MACH(const Byte * buf,size_t size,CFilterMode * filterMode)304 static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode)
305 {
306 UInt32 filterId, numCommands, commandsSize;
307
308 if (size < 512)
309 return 0;
310
311 BoolInt /* mode64, */ be;
312 switch (GetUi32(buf))
313 {
314 case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break;
315 case MACH_SIG_BE_64: /* mode64 = True; */ be = True; break;
316 case MACH_SIG_LE_32: /* mode64 = False; */ be = False; break;
317 case MACH_SIG_LE_64: /* mode64 = True; */ be = False; break;
318 default: return 0;
319 }
320
321 switch (Get32(buf + 4, be))
322 {
323 case MACH_MACHINE_386:
324 case MACH_MACHINE_AMD64: filterId = k_X86; break;
325 case MACH_MACHINE_ARM: if ( be) return 0; filterId = k_ARM; break;
326 case MACH_MACHINE_SPARC: if (!be) return 0; filterId = k_SPARC; break;
327 case MACH_MACHINE_PPC:
328 case MACH_MACHINE_PPC64: if (!be) return 0; filterId = k_PPC; break;
329 case MACH_MACHINE_ARM64: if ( be) return 0; filterId = k_ARM64; break;
330 default: return 0;
331 }
332
333 numCommands = Get32(buf + 0x10, be);
334 commandsSize = Get32(buf + 0x14, be);
335
336 if (commandsSize > (1 << 24) || numCommands > (1 << 18))
337 return 0;
338
339 filterMode->Id = filterId;
340 return 1;
341 }
342
343
344 /* ---------- WAV ---------- */
345
346 #define WAV_SUBCHUNK_fmt 0x20746D66
347 #define WAV_SUBCHUNK_data 0x61746164
348
349 #define RIFF_SIG 0x46464952
350
Parse_WAV(const Byte * buf,size_t size,CFilterMode * filterMode)351 static BoolInt Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode)
352 {
353 UInt32 subChunkSize, pos;
354 if (size < 0x2C)
355 return False;
356
357 if (GetUi32(buf + 0) != RIFF_SIG ||
358 GetUi32(buf + 8) != 0x45564157 || // WAVE
359 GetUi32(buf + 0xC) != WAV_SUBCHUNK_fmt)
360 return False;
361 subChunkSize = GetUi32(buf + 0x10);
362 /* [0x14 = format] = 1 (PCM) */
363 if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1)
364 return False;
365
366 const unsigned numChannels = GetUi16(buf + 0x16);
367 const unsigned bitsPerSample = GetUi16(buf + 0x22);
368 if ((bitsPerSample & 0x7) != 0)
369 return False;
370 const UInt32 delta = (UInt32)numChannels * (bitsPerSample >> 3);
371 if (delta == 0 || delta > 256)
372 return False;
373
374 pos = 0x14 + subChunkSize;
375
376 const int kNumSubChunksTests = 10;
377 // Do we need to scan more than 3 sub-chunks?
378 for (int i = 0; i < kNumSubChunksTests; i++)
379 {
380 if (pos + 8 > size)
381 return False;
382 subChunkSize = GetUi32(buf + pos + 4);
383 if (GetUi32(buf + pos) == WAV_SUBCHUNK_data)
384 {
385 filterMode->Id = k_Delta;
386 filterMode->Delta = delta;
387 return True;
388 }
389 if (subChunkSize > (1 << 16))
390 return False;
391 pos += subChunkSize + 8;
392 }
393 return False;
394 }
395
396
397 /*
398 filterMode->Delta will be set as:
399 = delta value : [1, 256] : for k_Delta
400 = 0 for another filters (branch filters)
401 */
ParseFile(const Byte * buf,size_t size,CFilterMode * filterMode)402 static BoolInt ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode)
403 {
404 filterMode->ClearFilterMode();
405
406 if (Parse_EXE(buf, size, filterMode)) return True;
407 if (Parse_ELF(buf, size, filterMode)) return True;
408 if (Parse_MACH(buf, size, filterMode)) return True;
409 return Parse_WAV(buf, size, filterMode);
410 }
411
412
413
414
415 struct CFilterMode2: public CFilterMode
416 {
417 bool Encrypted;
418 unsigned GroupIndex;
419
CFilterMode2NArchive::N7z::CFilterMode2420 CFilterMode2(): Encrypted(false) {}
421
CompareNArchive::N7z::CFilterMode2422 int Compare(const CFilterMode2 &m) const
423 {
424 if (!Encrypted)
425 {
426 if (m.Encrypted)
427 return -1;
428 }
429 else if (!m.Encrypted)
430 return 1;
431
432 const UInt32 id1 = Id;
433 const UInt32 id2 = m.Id;
434 /*
435 // we can change the order to place k_ARM64 files close to another exe files
436 if (id1 <= k_SPARC &&
437 id2 <= k_SPARC)
438 {
439 #define k_ARM64_FOR_SORT 0x3030901
440 if (id1 == k_ARM64) id1 = k_ARM64_FOR_SORT;
441 if (id2 == k_ARM64) id2 = k_ARM64_FOR_SORT;
442 }
443 */
444 if (id1 < id2) return -1;
445 if (id1 > id2) return 1;
446
447 if (Delta < m.Delta) return -1;
448 if (Delta > m.Delta) return 1;
449
450 if (Offset < m.Offset) return -1;
451 if (Offset > m.Offset) return 1;
452
453 /* we don't go here, because GetGroup()
454 and operator ==(const CFilterMode2 &m)
455 add only unique CFilterMode2:: { Id, Delta, Offset, Encrypted } items.
456 */
457 /*
458 if (GroupIndex < m.GroupIndex) return -1;
459 if (GroupIndex > m.GroupIndex) return 1;
460 */
461 return 0;
462 }
463
operator ==NArchive::N7z::CFilterMode2464 bool operator ==(const CFilterMode2 &m) const
465 {
466 return Id == m.Id
467 && Delta == m.Delta
468 && Offset == m.Offset
469 && Encrypted == m.Encrypted;
470 }
471 };
472
GetGroup(CRecordVector<CFilterMode2> & filters,const CFilterMode2 & m)473 static unsigned GetGroup(CRecordVector<CFilterMode2> &filters, const CFilterMode2 &m)
474 {
475 unsigned i;
476 for (i = 0; i < filters.Size(); i++)
477 {
478 const CFilterMode2 &m2 = filters[i];
479 if (m == m2)
480 return i;
481 /*
482 if (m.Encrypted != m2.Encrypted)
483 {
484 if (!m.Encrypted)
485 break;
486 continue;
487 }
488
489 if (m.Id < m2.Id) break;
490 if (m.Id != m2.Id) continue;
491
492 if (m.Delta < m2.Delta) break;
493 if (m.Delta != m2.Delta) continue;
494 */
495 }
496 // filters.Insert(i, m);
497 // return i;
498 return filters.Add(m);
499 }
500
Is86Filter(CMethodId m)501 static inline bool Is86Filter(CMethodId m)
502 {
503 return (m == k_BCJ || m == k_BCJ2);
504 }
505
IsExeFilter(CMethodId m)506 static inline bool IsExeFilter(CMethodId m)
507 {
508 switch (m)
509 {
510 case k_ARM64:
511 case k_RISCV:
512 case k_BCJ:
513 case k_BCJ2:
514 case k_ARM:
515 case k_ARMT:
516 case k_PPC:
517 case k_SPARC:
518 case k_IA64:
519 return true;
520 default: break;
521 }
522 return false;
523 }
524
Get_FilterGroup_for_Folder(CRecordVector<CFilterMode2> & filters,const CFolderEx & f,bool extractFilter)525 static unsigned Get_FilterGroup_for_Folder(
526 CRecordVector<CFilterMode2> &filters, const CFolderEx &f, bool extractFilter)
527 {
528 CFilterMode2 m;
529 // m.Id = 0;
530 // m.Delta = 0;
531 // m.Offset = 0;
532 m.Encrypted = f.IsEncrypted();
533
534 if (extractFilter)
535 {
536 const CCoderInfo &coder = f.Coders[f.UnpackCoder];
537
538 if (coder.MethodID == k_Delta)
539 {
540 if (coder.Props.Size() == 1)
541 {
542 m.Delta = (unsigned)coder.Props[0] + 1;
543 m.Id = k_Delta;
544 }
545 }
546 else if (IsExeFilter(coder.MethodID))
547 {
548 m.Id = (UInt32)coder.MethodID;
549 if (m.Id == k_BCJ2)
550 m.Id = k_BCJ;
551 m.SetDelta();
552 if (m.Id == k_ARM64 ||
553 m.Id == k_RISCV)
554 if (coder.Props.Size() == 4)
555 m.Offset = GetUi32(coder.Props);
556 }
557 }
558
559 return GetGroup(filters, m);
560 }
561
562
563
564
WriteRange(IInStream * inStream,ISequentialOutStream * outStream,UInt64 position,UInt64 size,ICompressProgressInfo * progress)565 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
566 UInt64 position, UInt64 size, ICompressProgressInfo *progress)
567 {
568 RINOK(InStream_SeekSet(inStream, position))
569 CMyComPtr2_Create<ISequentialInStream, CLimitedSequentialInStream> streamSpec;
570 streamSpec->SetStream(inStream);
571 streamSpec->Init(size);
572 CMyComPtr2_Create<ICompressCoder, NCompress::CCopyCoder> copyCoder;
573 RINOK(copyCoder.Interface()->Code(streamSpec, outStream, NULL, NULL, progress))
574 return (copyCoder->TotalSize == size ? S_OK : E_FAIL);
575 }
576
577 /*
578 unsigned CUpdateItem::GetExtensionPos() const
579 {
580 int slashPos = Name.ReverseFind_PathSepar();
581 int dotPos = Name.ReverseFind_Dot();
582 if (dotPos <= slashPos)
583 return Name.Len();
584 return dotPos + 1;
585 }
586
587 UString CUpdateItem::GetExtension() const
588 {
589 return Name.Ptr(GetExtensionPos());
590 }
591 */
592
593 #define RINOZ(x) { const int _t_ = (x); if (_t_ != 0) return _t_; }
594
595 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
596
597 /*
598 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
599 {
600 size_t c1 = a1.GetCapacity();
601 size_t c2 = a2.GetCapacity();
602 RINOZ_COMP(c1, c2);
603 for (size_t i = 0; i < c1; i++)
604 RINOZ_COMP(a1[i], a2[i]);
605 return 0;
606 }
607
608 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
609 {
610 RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
611 RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
612 RINOZ_COMP(c1.MethodID, c2.MethodID);
613 return CompareBuffers(c1.Props, c2.Props);
614 }
615
616 static int CompareBonds(const CBond &b1, const CBond &b2)
617 {
618 RINOZ_COMP(b1.InIndex, b2.InIndex);
619 return MyCompare(b1.OutIndex, b2.OutIndex);
620 }
621
622 static int CompareFolders(const CFolder &f1, const CFolder &f2)
623 {
624 int s1 = f1.Coders.Size();
625 int s2 = f2.Coders.Size();
626 RINOZ_COMP(s1, s2);
627 int i;
628 for (i = 0; i < s1; i++)
629 RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
630 s1 = f1.Bonds.Size();
631 s2 = f2.Bonds.Size();
632 RINOZ_COMP(s1, s2);
633 for (i = 0; i < s1; i++)
634 RINOZ(CompareBonds(f1.Bonds[i], f2.Bonds[i]));
635 return 0;
636 }
637 */
638
639 /*
640 static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
641 {
642 return CompareFileNames(f1.Name, f2.Name);
643 }
644 */
645
646 struct CFolderRepack
647 {
648 unsigned FolderIndex;
649 CNum NumCopyFiles;
650 };
651
652 /*
653 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *)
654 {
655 int i1 = p1->FolderIndex;
656 int i2 = p2->FolderIndex;
657 // In that version we don't want to parse folders here, so we don't compare folders
658 // probably it must be improved in future
659 // const CDbEx &db = *(const CDbEx *)param;
660 // RINOZ(CompareFolders(
661 // db.Folders[i1],
662 // db.Folders[i2]));
663
664 return MyCompare(i1, i2);
665
666 // RINOZ_COMP(
667 // db.NumUnpackStreamsVector[i1],
668 // db.NumUnpackStreamsVector[i2]);
669 // if (db.NumUnpackStreamsVector[i1] == 0)
670 // return 0;
671 // return CompareFiles(
672 // db.Files[db.FolderStartFileIndex[i1]],
673 // db.Files[db.FolderStartFileIndex[i2]]);
674 }
675 */
676
677 /*
678 we sort empty files and dirs in such order:
679 - Dir.NonAnti (name sorted)
680 - File.NonAnti (name sorted)
681 - File.Anti (name sorted)
682 - Dir.Anti (reverse name sorted)
683 */
684
CompareEmptyItems(const unsigned * p1,const unsigned * p2,void * param)685 static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param)
686 {
687 const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
688 const CUpdateItem &u1 = updateItems[*p1];
689 const CUpdateItem &u2 = updateItems[*p2];
690 // NonAnti < Anti
691 if (u1.IsAnti != u2.IsAnti)
692 return (u1.IsAnti ? 1 : -1);
693 if (u1.IsDir != u2.IsDir)
694 {
695 // Dir.NonAnti < File < Dir.Anti
696 if (u1.IsDir)
697 return (u1.IsAnti ? 1 : -1);
698 return (u2.IsAnti ? -1 : 1);
699 }
700 int n = CompareFileNames(u1.Name, u2.Name);
701 return (u1.IsDir && u1.IsAnti) ? -n : n;
702 }
703
704 static const char *g_Exts =
705 " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
706 " zip jar ear war msi"
707 " 3gp avi mov mpeg mpg mpe wmv"
708 " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
709 " swf"
710 " chm hxi hxs"
711 " gif jpeg jpg jp2 png tiff bmp ico psd psp"
712 " awg ps eps cgm dxf svg vrml wmf emf ai md"
713 " cad dwg pps key sxi"
714 " max 3ds"
715 " iso bin nrg mdf img pdi tar cpio xpi"
716 " vfd vhd vud vmc vsv"
717 " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
718 " inl inc idl acf asa"
719 " h hpp hxx c cpp cxx m mm go swift"
720 " rc java cs rs pas bas vb cls ctl frm dlg def"
721 " f77 f f90 f95"
722 " asm s"
723 " sql manifest dep"
724 " mak clw csproj vcproj sln dsp dsw"
725 " class"
726 " bat cmd bash sh"
727 " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
728 " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs"
729 " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
730 " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
731 " abw afp cwk lwp wpd wps wpt wrf wri"
732 " abf afm bdf fon mgf otf pcf pfa snf ttf"
733 " dbf mdb nsf ntf wdb db fdb gdb"
734 " exe dll ocx vbx sfx sys tlb awx com obj lib out o so"
735 " pdb pch idb ncb opt";
736
GetExtIndex(const char * ext)737 static unsigned GetExtIndex(const char *ext)
738 {
739 unsigned extIndex = 1;
740 const char *p = g_Exts;
741 for (;;)
742 {
743 char c = *p++;
744 if (c == 0)
745 return extIndex;
746 if (c == ' ')
747 continue;
748 unsigned pos = 0;
749 for (;;)
750 {
751 char c2 = ext[pos++];
752 if (c2 == 0 && (c == 0 || c == ' '))
753 return extIndex;
754 if (c != c2)
755 break;
756 c = *p++;
757 }
758 extIndex++;
759 for (;;)
760 {
761 if (c == 0)
762 return extIndex;
763 if (c == ' ')
764 break;
765 c = *p++;
766 }
767 }
768 }
769
770 struct CRefItem
771 {
772 const CUpdateItem *UpdateItem;
773 UInt32 Index;
774 unsigned ExtensionPos;
775 unsigned NamePos;
776 unsigned ExtensionIndex;
777
CRefItemNArchive::N7z::CRefItem778 CRefItem() {}
CRefItemNArchive::N7z::CRefItem779 CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
780 UpdateItem(&ui),
781 Index(index),
782 ExtensionPos(0),
783 NamePos(0),
784 ExtensionIndex(0)
785 {
786 if (sortByType)
787 {
788 const int slashPos = ui.Name.ReverseFind_PathSepar();
789 NamePos = (unsigned)(slashPos + 1);
790 const int dotPos = ui.Name.ReverseFind_Dot();
791 if (dotPos <= slashPos)
792 ExtensionPos = ui.Name.Len();
793 else
794 {
795 ExtensionPos = (unsigned)(dotPos + 1);
796 if (ExtensionPos != ui.Name.Len())
797 {
798 AString s;
799 for (unsigned pos = ExtensionPos;; pos++)
800 {
801 const wchar_t c = ui.Name[pos];
802 if (c >= 0x80)
803 break;
804 if (c == 0)
805 {
806 ExtensionIndex = GetExtIndex(s);
807 break;
808 }
809 s.Add_Char((char)MyCharLower_Ascii((char)c));
810 }
811 }
812 }
813 }
814 }
815 };
816
817 struct CSortParam
818 {
819 // const CObjectVector<CTreeFolder> *TreeFolders;
820 bool SortByType;
821 };
822
823 /*
824 we sort files in such order:
825 - Dir.NonAnti (name sorted)
826 - alt streams
827 - Dirs
828 - Dir.Anti (reverse name sorted)
829 */
830
831
CompareUpdateItems(const CRefItem * p1,const CRefItem * p2,void * param)832 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
833 {
834 const CRefItem &a1 = *p1;
835 const CRefItem &a2 = *p2;
836 const CUpdateItem &u1 = *a1.UpdateItem;
837 const CUpdateItem &u2 = *a2.UpdateItem;
838
839 /*
840 if (u1.IsAltStream != u2.IsAltStream)
841 return u1.IsAltStream ? 1 : -1;
842 */
843
844 // Actually there are no dirs that time. They were stored in other steps
845 // So that code is unused?
846 if (u1.IsDir != u2.IsDir)
847 return u1.IsDir ? 1 : -1;
848 if (u1.IsDir)
849 {
850 if (u1.IsAnti != u2.IsAnti)
851 return (u1.IsAnti ? 1 : -1);
852 int n = CompareFileNames(u1.Name, u2.Name);
853 return -n;
854 }
855
856 // bool sortByType = *(bool *)param;
857 const CSortParam *sortParam = (const CSortParam *)param;
858 const bool sortByType = sortParam->SortByType;
859 if (sortByType)
860 {
861 RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex)
862 RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)))
863 RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)))
864 if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
865 if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
866 if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime)
867 RINOZ_COMP(u1.Size, u2.Size)
868 }
869 /*
870 int par1 = a1.UpdateItem->ParentFolderIndex;
871 int par2 = a2.UpdateItem->ParentFolderIndex;
872 const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
873 const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
874
875 int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
876 int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
877 if (b1 < b2)
878 {
879 if (e1 <= b2)
880 return -1;
881 // p2 in p1
882 int par = par2;
883 for (;;)
884 {
885 const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
886 par = tf.Parent;
887 if (par == par1)
888 {
889 RINOZ(CompareFileNames(u1.Name, tf.Name));
890 break;
891 }
892 }
893 }
894 else if (b2 < b1)
895 {
896 if (e2 <= b1)
897 return 1;
898 // p1 in p2
899 int par = par1;
900 for (;;)
901 {
902 const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
903 par = tf.Parent;
904 if (par == par2)
905 {
906 RINOZ(CompareFileNames(tf.Name, u2.Name));
907 break;
908 }
909 }
910 }
911 */
912 // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
913 RINOK(CompareFileNames(u1.Name, u2.Name))
914 RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient)
915 RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive)
916 return 0;
917 }
918
919 struct CSolidGroup
920 {
921 CRecordVector<UInt32> Indices;
922
923 CRecordVector<CFolderRepack> folderRefs;
924 };
925
926 static const char * const g_Exe_Exts[] =
927 {
928 "dll"
929 , "exe"
930 , "ocx"
931 , "sfx"
932 , "sys"
933 };
934
935 static const char * const g_ExeUnix_Exts[] =
936 {
937 "so"
938 , "dylib"
939 };
940
IsExt_Exe(const wchar_t * ext)941 static bool IsExt_Exe(const wchar_t *ext)
942 {
943 for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_Exe_Exts); i++)
944 if (StringsAreEqualNoCase_Ascii(ext, g_Exe_Exts[i]))
945 return true;
946 return false;
947 }
948
949 /*
950 static bool IsExt_ExeUnix(const wchar_t *ext)
951 {
952 for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_ExeUnix_Exts); i++)
953 if (StringsAreEqualNoCase_Ascii(ext, g_ExeUnix_Exts[i]))
954 return true;
955 return false;
956 }
957 */
958
959 // we try to find "so" extension in such name: libstdc++.so.6.0.29
IsExt_ExeUnix_NumericAllowed(const UString & path)960 static bool IsExt_ExeUnix_NumericAllowed(const UString &path)
961 {
962 unsigned pos = path.Len();
963 unsigned dotPos = pos;
964 for (;;)
965 {
966 if (pos == 0)
967 return false;
968 const wchar_t c = path[--pos];
969 if (IS_PATH_SEPAR(c))
970 return false;
971 if (c == '.')
972 {
973 const unsigned num = (dotPos - pos) - 1;
974 if (num < 1)
975 return false;
976 const wchar_t *cur = path.Ptr(pos + 1);
977 for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_ExeUnix_Exts); i++)
978 {
979 const char *ext = g_ExeUnix_Exts[i];
980 if (num == MyStringLen(ext))
981 if (IsString1PrefixedByString2_NoCase_Ascii(cur, ext))
982 return true;
983 }
984 const wchar_t *end;
985 ConvertStringToUInt32(cur, &end);
986 if ((size_t)(end - cur) != num)
987 return false;
988 dotPos = pos;
989 }
990 }
991 }
992
993
994 struct CAnalysis
995 {
996 CMyComPtr<IArchiveUpdateCallbackFile> Callback;
997 CByteBuffer Buffer;
998
999 bool ParseWav;
1000 bool ParseExe;
1001 bool ParseExeUnix;
1002 bool ParseNoExt;
1003 bool ParseAll;
1004
1005 /*
1006 bool Need_ATime;
1007 bool ATime_Defined;
1008 FILETIME ATime;
1009 */
1010
CAnalysisNArchive::N7z::CAnalysis1011 CAnalysis():
1012 ParseWav(false),
1013 ParseExe(false),
1014 ParseExeUnix(false),
1015 ParseNoExt(false),
1016 ParseAll(false)
1017 /*
1018 , Need_ATime(false)
1019 , ATime_Defined(false)
1020 */
1021 {}
1022
1023 HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode);
1024 };
1025
1026 static const size_t kAnalysisBufSize = 1 << 14;
1027
GetFilterGroup(UInt32 index,const CUpdateItem & ui,CFilterMode & filterMode)1028 HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode)
1029 {
1030 filterMode.Id = 0;
1031 filterMode.Delta = 0;
1032 filterMode.Offset = 0;
1033
1034 CFilterMode filterModeTemp = filterMode;
1035
1036 const int slashPos = ui.Name.ReverseFind_PathSepar();
1037 const int dotPos = ui.Name.ReverseFind_Dot();
1038
1039 // if (dotPos > slashPos)
1040 {
1041 bool needReadFile = ParseAll;
1042 /* if (Callback) is not supported by client,
1043 we still try to use file name extension to detect executable file */
1044 bool probablyIsSameIsa = false;
1045
1046 if (!needReadFile || !Callback)
1047 {
1048 const wchar_t *ext = NULL;
1049 if (dotPos > slashPos)
1050 ext = ui.Name.Ptr((unsigned)(dotPos + 1));
1051 // 7-zip stores posix attributes in high 16 bits and sets (0x8000) flag
1052 if (ui.Attrib & 0x8000)
1053 {
1054 const unsigned st_mode = ui.Attrib >> 16;
1055 /* note: executable ".so" can be without execute permission,
1056 and symbolic link to such ".so" file is possible */
1057 // st_mode = 00111; // for debug
1058 /* in Linux we expect such permissions:
1059 0755 : for most executables
1060 0644 : for some ".so" files
1061 0777 : in WSL for all files.
1062 We can try to exclude some such 0777 cases from analysis,
1063 if there is non-executable extension.
1064 */
1065
1066 if ((st_mode & (
1067 MY_LIN_S_IXUSR |
1068 MY_LIN_S_IXGRP |
1069 MY_LIN_S_IXOTH)) != 0
1070 && MY_LIN_S_ISREG(st_mode)
1071 && (ui.Size >= (1u << 11)))
1072 {
1073 #ifndef _WIN32
1074 probablyIsSameIsa = true;
1075 #endif
1076 needReadFile = true;
1077 }
1078 }
1079
1080 if (!needReadFile)
1081 {
1082 if (!ext)
1083 needReadFile = ParseNoExt;
1084 else
1085 {
1086 bool isUnixExt = false;
1087 if (ParseExeUnix)
1088 isUnixExt = IsExt_ExeUnix_NumericAllowed(ui.Name);
1089 if (isUnixExt)
1090 {
1091 needReadFile = true;
1092 #ifndef _WIN32
1093 probablyIsSameIsa = true;
1094 #endif
1095 }
1096 else if (IsExt_Exe(ext))
1097 {
1098 needReadFile = ParseExe;
1099 #ifdef _WIN32
1100 probablyIsSameIsa = true;
1101 #endif
1102 }
1103 else if (StringsAreEqualNoCase_Ascii(ext, "wav"))
1104 {
1105 if (!needReadFile)
1106 needReadFile = ParseWav;
1107 }
1108 }
1109 }
1110 }
1111
1112 if (needReadFile)
1113 {
1114 BoolInt parseRes = false;
1115 if (Callback)
1116 {
1117 if (Buffer.Size() != kAnalysisBufSize)
1118 Buffer.Alloc(kAnalysisBufSize);
1119 CMyComPtr<ISequentialInStream> stream;
1120 HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze);
1121 if (result == S_OK && stream)
1122 {
1123 /*
1124 if (Need_ATime)
1125 {
1126 // access time could be changed in analysis pass
1127 CMyComPtr<IStreamGetProps> getProps;
1128 stream.QueryInterface(IID_IStreamGetProps, (void **)&getProps);
1129 if (getProps)
1130 if (getProps->GetProps(NULL, NULL, &ATime, NULL, NULL) == S_OK)
1131 ATime_Defined = true;
1132 }
1133 */
1134 size_t size = kAnalysisBufSize;
1135 result = ReadStream(stream, Buffer, &size);
1136 stream.Release();
1137 // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK));
1138 if (result == S_OK)
1139 {
1140 parseRes = ParseFile(Buffer, size, &filterModeTemp);
1141 }
1142 }
1143 } // Callback
1144 else if (probablyIsSameIsa)
1145 {
1146 #ifdef MY_CPU_X86_OR_AMD64
1147 filterModeTemp.Id = k_X86;
1148 #endif
1149 #ifdef MY_CPU_ARM64
1150 filterModeTemp.Id = k_ARM64;
1151 #endif
1152 #ifdef MY_CPU_RISCV
1153 filterModeTemp.Id = k_RISCV;
1154 #endif
1155 #ifdef MY_CPU_SPARC
1156 filterModeTemp.Id = k_SPARC;
1157 #endif
1158 parseRes = true;
1159 }
1160
1161 if (parseRes
1162 && filterModeTemp.Id != k_Delta
1163 && filterModeTemp.Delta == 0)
1164 {
1165 /* ParseFile() sets (filterModeTemp.Delta == 0) for all
1166 methods except of k_Delta. */
1167 // it's not k_Delta
1168 // So we call SetDelta() to set Delta
1169 filterModeTemp.SetDelta();
1170 if (filterModeTemp.Delta > 1)
1171 {
1172 /* If file Size is not aligned, then branch filter
1173 will not work for next file in solid block.
1174 Maybe we should allow filter for non-aligned-size file in non-solid archives ?
1175 */
1176 if (ui.Size % filterModeTemp.Delta != 0)
1177 parseRes = false;
1178 // windows exe files are not aligned for 4 KiB.
1179 /*
1180 else if (filterModeTemp.Id == k_ARM64 && filterModeTemp.Offset != 0)
1181 {
1182 if (ui.Size % (1 << 12) != 0)
1183 {
1184 // If Size is not aligned for 4 KiB, then Offset will not work for next file in solid block.
1185 // so we place such file in group with (Offset==0).
1186 filterModeTemp.Offset = 0;
1187 }
1188 }
1189 */
1190 }
1191 }
1192 if (!parseRes)
1193 filterModeTemp.ClearFilterMode();
1194 }
1195 }
1196
1197 filterMode = filterModeTemp;
1198 return S_OK;
1199 }
1200
GetMethodFull(UInt64 methodID,UInt32 numStreams,CMethodFull & m)1201 static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull &m)
1202 {
1203 m.Id = methodID;
1204 m.NumStreams = numStreams;
1205 }
1206
1207
1208 // we add bond for mode.Methods[0] that is filter
AddBondForFilter(CCompressionMethodMode & mode)1209 static HRESULT AddBondForFilter(CCompressionMethodMode &mode)
1210 {
1211 for (unsigned c = 1; c < mode.Methods.Size(); c++)
1212 {
1213 if (!mode.IsThereBond_to_Coder(c))
1214 {
1215 CBond2 bond;
1216 bond.OutCoder = 0;
1217 bond.OutStream = 0;
1218 bond.InCoder = c;
1219 mode.Bonds.Add(bond);
1220 return S_OK;
1221 }
1222 }
1223 return E_INVALIDARG;
1224 }
1225
1226 /*
1227 static HRESULT AddBondForFilter_if_ThereAreBonds(CCompressionMethodMode &mode)
1228 {
1229 if (!mode.Bonds.IsEmpty())
1230 return AddBondForFilter(mode);
1231 return S_OK;
1232 }
1233 */
1234
AddBcj2Methods(CCompressionMethodMode & mode)1235 static HRESULT AddBcj2Methods(CCompressionMethodMode &mode)
1236 {
1237 // mode.Methods[0] must be k_BCJ2 method !
1238 // mode.Methods[1] : we expect that there is at least one method after BCJ2
1239
1240 CMethodFull m;
1241 GetMethodFull(k_LZMA, 1, m);
1242
1243 m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
1244 m.AddProp32(NCoderPropID::kNumFastBytes, 128);
1245 m.AddProp32(NCoderPropID::kNumThreads, 1);
1246 m.AddProp32(NCoderPropID::kLitPosBits, 2);
1247 m.AddProp32(NCoderPropID::kLitContextBits, 0);
1248 // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2");
1249
1250 const unsigned methodIndex = mode.Methods.Size();
1251
1252 if (mode.Bonds.IsEmpty())
1253 {
1254 for (unsigned i = 1; i + 1 < mode.Methods.Size(); i++)
1255 {
1256 CBond2 bond;
1257 bond.OutCoder = i;
1258 bond.OutStream = 0;
1259 bond.InCoder = i + 1;
1260 mode.Bonds.Add(bond);
1261 }
1262 }
1263
1264 mode.Methods.Add(m);
1265 mode.Methods.Add(m);
1266
1267 RINOK(AddBondForFilter(mode))
1268 CBond2 bond;
1269 bond.OutCoder = 0; // index of BCJ2 coder
1270 bond.InCoder = methodIndex; bond.OutStream = 1; mode.Bonds.Add(bond);
1271 bond.InCoder = methodIndex + 1; bond.OutStream = 2; mode.Bonds.Add(bond);
1272 return S_OK;
1273 }
1274
1275
MakeExeMethod(CCompressionMethodMode & mode,const CFilterMode & filterMode,const bool bcj2_IsAllowed,const CUIntVector & disabledFilterIDs)1276 static HRESULT MakeExeMethod(CCompressionMethodMode &mode,
1277 const CFilterMode &filterMode,
1278 const bool bcj2_IsAllowed,
1279 const CUIntVector &disabledFilterIDs)
1280 {
1281 if (mode.Filter_was_Inserted)
1282 {
1283 // filter was inserted, but bond for that filter was not added still.
1284 const CMethodFull &m = mode.Methods[0];
1285 if (m.Id == k_BCJ2)
1286 return AddBcj2Methods(mode);
1287 if (!m.IsSimpleCoder())
1288 return E_NOTIMPL;
1289 if (mode.Bonds.IsEmpty())
1290 return S_OK;
1291 return AddBondForFilter(mode);
1292 }
1293
1294 if (filterMode.Id == 0)
1295 return S_OK;
1296
1297 unsigned nextCoder;
1298
1299 const bool useBcj2 = bcj2_IsAllowed
1300 && Is86Filter(filterMode.Id)
1301 && disabledFilterIDs.FindInSorted(k_BCJ2) < 0;
1302
1303 if (!useBcj2 && disabledFilterIDs.FindInSorted(filterMode.Id) >= 0)
1304 {
1305 // required filter is disabled,
1306 // but we still can use information about data alignment.
1307 #if 0 // 1 for debug
1308 // we can return here, if we want default lzma properties
1309 return S_OK;
1310 #else
1311 // we will try to change lzma/lzma2 properties
1312 nextCoder = 0;
1313 if (!mode.Bonds.IsEmpty())
1314 for (unsigned c = 0;; c++)
1315 {
1316 if (c == mode.Methods.Size())
1317 return S_OK;
1318 if (!mode.IsThereBond_to_Coder(c))
1319 {
1320 nextCoder = c;
1321 break;
1322 }
1323 }
1324 #endif
1325 }
1326 else
1327 {
1328 // we insert new filter method:
1329 CMethodFull &m = mode.Methods.InsertNew(0); // 0 == index of new inserted item
1330 {
1331 // we move all coder indexes in bonds up for 1 position:
1332 FOR_VECTOR (k, mode.Bonds)
1333 {
1334 CBond2 &bond = mode.Bonds[k];
1335 bond.InCoder++;
1336 bond.OutCoder++;
1337 }
1338 }
1339 if (useBcj2)
1340 {
1341 GetMethodFull(k_BCJ2, 4, m);
1342 return AddBcj2Methods(mode);
1343 }
1344
1345 GetMethodFull(filterMode.Id, 1, m);
1346
1347 if (filterMode.Id == k_Delta)
1348 m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta);
1349 else if (filterMode.Id == k_ARM64
1350 || filterMode.Id == k_RISCV)
1351 {
1352 // if (filterMode.Offset != 0)
1353 m.AddProp32(
1354 NCoderPropID::kDefaultProp,
1355 // NCoderPropID::kBranchOffset,
1356 filterMode.Offset);
1357 }
1358
1359 nextCoder = 1;
1360 if (!mode.Bonds.IsEmpty())
1361 {
1362 RINOK(AddBondForFilter(mode))
1363 nextCoder = mode.Bonds.Back().InCoder;
1364 }
1365 }
1366
1367 if (nextCoder >= mode.Methods.Size())
1368 {
1369 // we don't expect that case, if there was non-filter method.
1370 // but we return S_OK to support filter-only case.
1371 return S_OK;
1372 }
1373
1374 int alignBits = -1;
1375 {
1376 const UInt32 delta = filterMode.Delta;
1377 if (delta == 0 || delta > 16)
1378 {
1379 // if (delta == 0) alignBits = GetAlignForFilterMethod(filterMode.Id);
1380 }
1381 else if ((delta & ((1 << 4) - 1)) == 0) alignBits = 4;
1382 else if ((delta & ((1 << 3) - 1)) == 0) alignBits = 3;
1383 else if ((delta & ((1 << 2) - 1)) == 0) alignBits = 2;
1384 else if ((delta & ((1 << 1) - 1)) == 0) alignBits = 1;
1385 // else alignBits = 0;
1386 /* alignBits=0 is default mode for lzma/lzma2.
1387 So we don't set alignBits=0 here. */
1388 }
1389 if (alignBits <= 0)
1390 return S_OK;
1391 // (alignBits > 0)
1392 CMethodFull &nextMethod = mode.Methods[nextCoder];
1393 if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2)
1394 if (!nextMethod.Are_Lzma_Model_Props_Defined())
1395 {
1396 if (alignBits > 2 || filterMode.Id == k_Delta)
1397 nextMethod.AddProp32(NCoderPropID::kPosStateBits, (unsigned)alignBits);
1398 const unsigned lc = (alignBits < 3) ? (unsigned)(3 - alignBits) : 0u;
1399 nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc);
1400 nextMethod.AddProp32(NCoderPropID::kLitPosBits, (unsigned)alignBits);
1401 }
1402 return S_OK;
1403 }
1404
1405
UpdateItem_To_FileItem2(const CUpdateItem & ui,CFileItem2 & file2)1406 static void UpdateItem_To_FileItem2(const CUpdateItem &ui, CFileItem2 &file2)
1407 {
1408 file2.Attrib = ui.Attrib; file2.AttribDefined = ui.AttribDefined;
1409 file2.CTime = ui.CTime; file2.CTimeDefined = ui.CTimeDefined;
1410 file2.ATime = ui.ATime; file2.ATimeDefined = ui.ATimeDefined;
1411 file2.MTime = ui.MTime; file2.MTimeDefined = ui.MTimeDefined;
1412 file2.IsAnti = ui.IsAnti;
1413 // file2.IsAux = false;
1414 file2.StartPosDefined = false;
1415 // file2.StartPos = 0;
1416 }
1417
1418
UpdateItem_To_FileItem(const CUpdateItem & ui,CFileItem & file,CFileItem2 & file2)1419 static void UpdateItem_To_FileItem(const CUpdateItem &ui,
1420 CFileItem &file, CFileItem2 &file2)
1421 {
1422 UpdateItem_To_FileItem2(ui, file2);
1423
1424 file.Size = ui.Size;
1425 file.IsDir = ui.IsDir;
1426 file.HasStream = ui.HasStream();
1427 // file.IsAltStream = ui.IsAltStream;
1428 }
1429
1430
1431
1432 Z7_CLASS_IMP_COM_2(
1433 CRepackInStreamWithSizes
1434 , ISequentialInStream
1435 , ICompressGetSubStreamSize
1436 )
1437 CMyComPtr<ISequentialInStream> _stream;
1438 UInt64 _size;
1439 const CBoolVector *_extractStatuses;
1440 UInt32 _startIndex;
1441 public:
1442 const CDbEx *_db;
1443
1444 void Init(ISequentialInStream *stream, UInt32 startIndex, const CBoolVector *extractStatuses)
1445 {
1446 _startIndex = startIndex;
1447 _extractStatuses = extractStatuses;
1448 _size = 0;
1449 _stream = stream;
1450 }
1451 UInt64 GetSize() const { return _size; }
1452 };
1453
1454 Z7_COM7F_IMF(CRepackInStreamWithSizes::Read(void *data, UInt32 size, UInt32 *processedSize))
1455 {
1456 UInt32 realProcessedSize;
1457 const HRESULT result = _stream->Read(data, size, &realProcessedSize);
1458 _size += realProcessedSize;
1459 if (processedSize)
1460 *processedSize = realProcessedSize;
1461 return result;
1462 }
1463
1464 Z7_COM7F_IMF(CRepackInStreamWithSizes::GetSubStreamSize(UInt64 subStream, UInt64 *value))
1465 {
1466 *value = 0;
1467 if (subStream >= _extractStatuses->Size())
1468 return S_FALSE; // E_FAIL;
1469 const unsigned index = (unsigned)subStream;
1470 if ((*_extractStatuses)[index])
1471 {
1472 const CFileItem &fi = _db->Files[_startIndex + index];
1473 if (fi.HasStream)
1474 *value = fi.Size;
1475 }
1476 return S_OK;
1477 }
1478
1479
1480 class CRepackStreamBase
1481 {
1482 protected:
1483 bool _needWrite;
1484 bool _fileIsOpen;
1485 bool _calcCrc;
1486 UInt32 _crc;
1487 UInt64 _rem;
1488
1489 const CBoolVector *_extractStatuses;
1490 UInt32 _startIndex;
1491 unsigned _currentIndex;
1492
1493 HRESULT OpenFile();
1494 HRESULT CloseFile();
1495 HRESULT ProcessEmptyFiles();
1496
1497 public:
1498 const CDbEx *_db;
1499 CMyComPtr<IArchiveUpdateCallbackFile> _opCallback;
1500 CMyComPtr<IArchiveExtractCallbackMessage2> _extractCallback;
1501
1502 HRESULT Init(UInt32 startIndex, const CBoolVector *extractStatuses);
1503 HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
1504 };
1505
1506 HRESULT CRepackStreamBase::Init(UInt32 startIndex, const CBoolVector *extractStatuses)
1507 {
1508 _startIndex = startIndex;
1509 _extractStatuses = extractStatuses;
1510
1511 _currentIndex = 0;
1512 _fileIsOpen = false;
1513
1514 return ProcessEmptyFiles();
1515 }
1516
1517 HRESULT CRepackStreamBase::OpenFile()
1518 {
1519 UInt32 arcIndex = _startIndex + _currentIndex;
1520 const CFileItem &fi = _db->Files[arcIndex];
1521
1522 _needWrite = (*_extractStatuses)[_currentIndex];
1523 if (_opCallback)
1524 {
1525 RINOK(_opCallback->ReportOperation(
1526 NEventIndexType::kInArcIndex, arcIndex,
1527 _needWrite ?
1528 NUpdateNotifyOp::kRepack :
1529 NUpdateNotifyOp::kSkip))
1530 }
1531
1532 _crc = CRC_INIT_VAL;
1533 _calcCrc = (fi.CrcDefined && !fi.IsDir);
1534
1535 _fileIsOpen = true;
1536 _rem = fi.Size;
1537 return S_OK;
1538 }
1539
1540 const HRESULT k_My_HRESULT_CRC_ERROR = 0x20000002;
1541
1542 HRESULT CRepackStreamBase::CloseFile()
1543 {
1544 UInt32 arcIndex = _startIndex + _currentIndex;
1545 const CFileItem &fi = _db->Files[arcIndex];
1546 _fileIsOpen = false;
1547 _currentIndex++;
1548 if (!_calcCrc || fi.Crc == CRC_GET_DIGEST(_crc))
1549 return S_OK;
1550
1551 if (_extractCallback)
1552 {
1553 RINOK(_extractCallback->ReportExtractResult(
1554 NEventIndexType::kInArcIndex, arcIndex,
1555 NExtract::NOperationResult::kCRCError))
1556 }
1557 // return S_FALSE;
1558 return k_My_HRESULT_CRC_ERROR;
1559 }
1560
1561 HRESULT CRepackStreamBase::ProcessEmptyFiles()
1562 {
1563 while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
1564 {
1565 RINOK(OpenFile())
1566 RINOK(CloseFile())
1567 }
1568 return S_OK;
1569 }
1570
1571
1572
1573 #ifndef Z7_ST
1574
1575 class CFolderOutStream2 Z7_final:
1576 public CRepackStreamBase,
1577 public ISequentialOutStream,
1578 public CMyUnknownImp
1579 {
1580 Z7_COM_UNKNOWN_IMP_0
1581 Z7_IFACE_COM7_IMP(ISequentialOutStream)
1582 public:
1583 CMyComPtr<ISequentialOutStream> _stream;
1584 };
1585
1586 Z7_COM7F_IMF(CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize))
1587 {
1588 if (processedSize)
1589 *processedSize = 0;
1590
1591 while (size != 0)
1592 {
1593 if (_fileIsOpen)
1594 {
1595 UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1596 HRESULT result = S_OK;
1597 if (_needWrite)
1598 result = _stream->Write(data, cur, &cur);
1599 if (_calcCrc)
1600 _crc = CrcUpdate(_crc, data, cur);
1601 if (processedSize)
1602 *processedSize += cur;
1603 data = (const Byte *)data + cur;
1604 size -= cur;
1605 _rem -= cur;
1606 if (_rem == 0)
1607 {
1608 RINOK(CloseFile())
1609 RINOK(ProcessEmptyFiles())
1610 }
1611 RINOK(result)
1612 if (cur == 0)
1613 break;
1614 continue;
1615 }
1616
1617 RINOK(ProcessEmptyFiles())
1618 if (_currentIndex == _extractStatuses->Size())
1619 {
1620 // we don't support write cut here
1621 return E_FAIL;
1622 }
1623 RINOK(OpenFile())
1624 }
1625
1626 return S_OK;
1627 }
1628
1629 #endif
1630
1631
1632
1633 static const UInt32 kTempBufSize = 1 << 16;
1634
1635 class CFolderInStream2 Z7_final:
1636 public CRepackStreamBase,
1637 public ISequentialInStream,
1638 public CMyUnknownImp
1639 {
1640 Z7_COM_UNKNOWN_IMP_0
1641 Z7_IFACE_COM7_IMP(ISequentialInStream)
1642
1643 Byte *_buf;
1644 public:
1645 CMyComPtr<ISequentialInStream> _inStream;
1646 HRESULT Result;
1647
1648 CFolderInStream2():
1649 Result(S_OK)
1650 {
1651 _buf = new Byte[kTempBufSize];
1652 }
1653
1654 ~CFolderInStream2()
1655 {
1656 delete []_buf;
1657 }
1658
1659 void Init() { Result = S_OK; }
1660 };
1661
1662 Z7_COM7F_IMF(CFolderInStream2::Read(void *data, UInt32 size, UInt32 *processedSize))
1663 {
1664 if (processedSize)
1665 *processedSize = 0;
1666
1667 while (size != 0)
1668 {
1669 if (_fileIsOpen)
1670 {
1671 UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1672
1673 void *buf;
1674 if (_needWrite)
1675 buf = data;
1676 else
1677 {
1678 buf = _buf;
1679 if (cur > kTempBufSize)
1680 cur = kTempBufSize;
1681 }
1682
1683 const HRESULT result = _inStream->Read(buf, cur, &cur);
1684 _crc = CrcUpdate(_crc, buf, cur);
1685 _rem -= cur;
1686
1687 if (_needWrite)
1688 {
1689 data = (Byte *)data + cur;
1690 size -= cur;
1691 if (processedSize)
1692 *processedSize += cur;
1693 }
1694
1695 if (result != S_OK)
1696 Result = result;
1697
1698 if (_rem == 0)
1699 {
1700 RINOK(CloseFile())
1701 RINOK(ProcessEmptyFiles())
1702 }
1703
1704 RINOK(result)
1705
1706 if (cur == 0)
1707 return E_FAIL;
1708
1709 continue;
1710 }
1711
1712 RINOK(ProcessEmptyFiles())
1713 if (_currentIndex == _extractStatuses->Size())
1714 {
1715 return S_OK;
1716 }
1717 RINOK(OpenFile())
1718 }
1719
1720 return S_OK;
1721 }
1722
1723
1724 class CThreadDecoder Z7_final
1725 #ifndef Z7_ST
1726 : public CVirtThread
1727 #endif
1728 {
1729 public:
1730 CDecoder Decoder;
1731
1732 CThreadDecoder(bool multiThreadMixer):
1733 Decoder(multiThreadMixer)
1734 {
1735 #ifndef Z7_ST
1736 if (multiThreadMixer)
1737 {
1738 MtMode = false;
1739 NumThreads = 1;
1740 FosSpec = new CFolderOutStream2;
1741 Fos = FosSpec;
1742 Result = E_FAIL;
1743 }
1744 #endif
1745 // UnpackSize = 0;
1746 // send_UnpackSize = false;
1747 }
1748
1749 #ifndef Z7_ST
1750
1751 bool dataAfterEnd_Error;
1752 HRESULT Result;
1753 CMyComPtr<IInStream> InStream;
1754
1755 CFolderOutStream2 *FosSpec;
1756 CMyComPtr<ISequentialOutStream> Fos;
1757
1758 UInt64 StartPos;
1759 const CFolders *Folders;
1760 unsigned FolderIndex;
1761
1762 // bool send_UnpackSize;
1763 // UInt64 UnpackSize;
1764
1765 #ifndef Z7_NO_CRYPTO
1766 CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1767 #endif
1768
1769 DECL_EXTERNAL_CODECS_LOC_VARS_DECL
1770
1771 #ifndef Z7_ST
1772 bool MtMode;
1773 UInt32 NumThreads;
1774 #endif
1775
1776
1777 ~CThreadDecoder() Z7_DESTRUCTOR_override
1778 {
1779 /* WaitThreadFinish() will be called in ~CVirtThread().
1780 But we need WaitThreadFinish() call before
1781 destructors of this class members.
1782 */
1783 CVirtThread::WaitThreadFinish();
1784 }
1785 private:
1786 virtual void Execute() Z7_override;
1787
1788 #endif
1789 };
1790
1791 #ifndef Z7_ST
1792
1793 void CThreadDecoder::Execute()
1794 {
1795 try
1796 {
1797 #ifndef Z7_NO_CRYPTO
1798 bool isEncrypted = false;
1799 bool passwordIsDefined = false;
1800 UString password;
1801 #endif
1802
1803 dataAfterEnd_Error = false;
1804
1805 Result = Decoder.Decode(
1806 EXTERNAL_CODECS_LOC_VARS
1807 InStream,
1808 StartPos,
1809 *Folders, FolderIndex,
1810
1811 // send_UnpackSize ? &UnpackSize : NULL,
1812 NULL, // unpackSize : FULL unpack
1813
1814 Fos,
1815 NULL, // compressProgress
1816
1817 NULL // *inStreamMainRes
1818 , dataAfterEnd_Error
1819
1820 Z7_7Z_DECODER_CRYPRO_VARS
1821 #ifndef Z7_ST
1822 , MtMode, NumThreads,
1823 0 // MemUsage
1824 #endif
1825
1826 );
1827 }
1828 catch(...)
1829 {
1830 Result = E_FAIL;
1831 }
1832
1833 /*
1834 if (Result == S_OK)
1835 Result = FosSpec->CheckFinishedState();
1836 */
1837 FosSpec->_stream.Release();
1838 }
1839
1840 #endif
1841
1842 #ifndef Z7_NO_CRYPTO
1843
1844 Z7_CLASS_IMP_NOQIB_1(
1845 CCryptoGetTextPassword
1846 , ICryptoGetTextPassword
1847 )
1848 public:
1849 UString Password;
1850 };
1851
1852 Z7_COM7F_IMF(CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password))
1853 {
1854 return StringToBstr(Password, password);
1855 }
1856
1857 #endif
1858
1859
1860 static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFileItem2 &file2)
1861 {
1862 file = inDb.Files[index];
1863 file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
1864 file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
1865 file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
1866 file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
1867 file2.AttribDefined = inDb.Attrib.GetItem(index, file2.Attrib);
1868 file2.IsAnti = inDb.IsItemAnti(index);
1869 // file2.IsAux = inDb.IsItemAux(index);
1870 }
1871
1872 HRESULT Update(
1873 DECL_EXTERNAL_CODECS_LOC_VARS
1874 IInStream *inStream,
1875 const CDbEx *db,
1876 CObjectVector<CUpdateItem> &updateItems,
1877 // const CObjectVector<CTreeFolder> &treeFolders,
1878 // const CUniqBlocks &secureBlocks,
1879 ISequentialOutStream *seqOutStream,
1880 IArchiveUpdateCallback *updateCallback,
1881 const CUpdateOptions &options)
1882 {
1883 UInt64 numSolidFiles = options.NumSolidFiles;
1884 if (numSolidFiles == 0)
1885 numSolidFiles = 1;
1886
1887 Z7_DECL_CMyComPtr_QI_FROM(
1888 IArchiveUpdateCallbackFile,
1889 opCallback, updateCallback)
1890
1891 Z7_DECL_CMyComPtr_QI_FROM(
1892 IArchiveExtractCallbackMessage2,
1893 extractCallback, updateCallback)
1894
1895 /*
1896 Z7_DECL_CMyComPtr_QI_FROM(
1897 IArchiveUpdateCallbackArcProp,
1898 reportArcProp, updateCallback)
1899 */
1900
1901 // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
1902
1903 CMyComPtr<IStreamSetRestriction> v_StreamSetRestriction;
1904 {
1905 Z7_DECL_CMyComPtr_QI_FROM(
1906 IOutStream,
1907 outStream, seqOutStream)
1908 if (!outStream)
1909 return E_NOTIMPL;
1910 const UInt64 sfxBlockSize = (db && !options.RemoveSfxBlock) ?
1911 db->ArcInfo.StartPosition: 0;
1912 seqOutStream->QueryInterface(IID_IStreamSetRestriction, (void **)&v_StreamSetRestriction);
1913 if (v_StreamSetRestriction)
1914 {
1915 UInt64 offset = 0;
1916 RINOK(outStream->Seek(0, STREAM_SEEK_CUR, &offset))
1917 RINOK(v_StreamSetRestriction->SetRestriction(
1918 outStream ? offset + sfxBlockSize : 0,
1919 outStream ? offset + sfxBlockSize + k_StartHeadersRewriteSize : 0))
1920 }
1921 outStream.Release();
1922 if (sfxBlockSize != 0)
1923 {
1924 RINOK(WriteRange(inStream, seqOutStream, 0, sfxBlockSize, NULL))
1925 }
1926 }
1927
1928 CIntArr fileIndexToUpdateIndexMap;
1929 UInt64 complexity = 0;
1930 bool isThere_UnknownSize = false;
1931 UInt64 inSizeForReduce2 = 0;
1932
1933 #ifndef Z7_NO_CRYPTO
1934 bool needEncryptedRepack = false;
1935 #endif
1936
1937 CRecordVector<CFilterMode2> filters;
1938 CObjectVector<CSolidGroup> groups;
1939
1940 #ifndef Z7_ST
1941 bool thereAreRepacks = false;
1942 #endif
1943
1944 bool useFilters = options.UseFilters;
1945 if (useFilters)
1946 {
1947 const CCompressionMethodMode &method = *options.Method;
1948
1949 FOR_VECTOR (i, method.Methods)
1950 {
1951 /* IsFilterMethod() knows only built-in codecs
1952 FIXME: we should check IsFilter status for external filters too */
1953 if (IsFilterMethod(method.Methods[i].Id))
1954 {
1955 useFilters = false;
1956 break;
1957 }
1958 }
1959 }
1960
1961 if (db)
1962 {
1963 fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
1964 unsigned i;
1965
1966 for (i = 0; i < db->Files.Size(); i++)
1967 fileIndexToUpdateIndexMap[i] = -1;
1968
1969 for (i = 0; i < updateItems.Size(); i++)
1970 {
1971 int index = updateItems[i].IndexInArchive;
1972 if (index != -1)
1973 fileIndexToUpdateIndexMap[(unsigned)index] = (int)i;
1974 }
1975
1976 for (i = 0; i < db->NumFolders; i++)
1977 {
1978 CNum indexInFolder = 0;
1979 CNum numCopyItems = 0;
1980 const CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
1981 UInt64 repackSize = 0;
1982
1983 for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
1984 {
1985 if (fi >= db->Files.Size())
1986 return E_FAIL;
1987
1988 const CFileItem &file = db->Files[fi];
1989 if (file.HasStream)
1990 {
1991 indexInFolder++;
1992 const int updateIndex = fileIndexToUpdateIndexMap[fi];
1993 if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData)
1994 {
1995 numCopyItems++;
1996 repackSize += file.Size;
1997 }
1998 }
1999 }
2000
2001 if (numCopyItems == 0)
2002 continue;
2003
2004 CFolderRepack rep;
2005 rep.FolderIndex = i;
2006 rep.NumCopyFiles = numCopyItems;
2007 CFolderEx f;
2008 db->ParseFolderEx(i, f);
2009
2010 #ifndef Z7_NO_CRYPTO
2011 const bool isEncrypted = f.IsEncrypted();
2012 #endif
2013 const bool needCopy = (numCopyItems == numUnpackStreams);
2014 const bool extractFilter = (useFilters || needCopy);
2015
2016 const unsigned groupIndex = Get_FilterGroup_for_Folder(filters, f, extractFilter);
2017
2018 while (groupIndex >= groups.Size())
2019 groups.AddNew();
2020
2021 groups[groupIndex].folderRefs.Add(rep);
2022
2023 if (needCopy)
2024 complexity += db->GetFolderFullPackSize(i);
2025 else
2026 {
2027 #ifndef Z7_ST
2028 thereAreRepacks = true;
2029 #endif
2030 complexity += repackSize;
2031 if (inSizeForReduce2 < repackSize)
2032 inSizeForReduce2 = repackSize;
2033 #ifndef Z7_NO_CRYPTO
2034 if (isEncrypted)
2035 needEncryptedRepack = true;
2036 #endif
2037 }
2038 }
2039 }
2040
2041 UInt64 inSizeForReduce = 0;
2042 {
2043 const bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0);
2044 FOR_VECTOR (i, updateItems)
2045 {
2046 const CUpdateItem &ui = updateItems[i];
2047 if (ui.NewData)
2048 {
2049 if (ui.Size == (UInt64)(Int64)-1)
2050 isThere_UnknownSize = true;
2051 else
2052 {
2053 complexity += ui.Size;
2054 if (isSolid)
2055 inSizeForReduce += ui.Size;
2056 else if (inSizeForReduce < ui.Size)
2057 inSizeForReduce = ui.Size;
2058 }
2059 }
2060 }
2061 }
2062
2063 if (isThere_UnknownSize)
2064 inSizeForReduce = (UInt64)(Int64)-1;
2065 else
2066 RINOK(updateCallback->SetTotal(complexity))
2067
2068 if (inSizeForReduce < inSizeForReduce2)
2069 inSizeForReduce = inSizeForReduce2;
2070
2071
2072 CMyComPtr2_Create<ICompressProgressInfo, CLocalProgress> lps;
2073 lps->Init(updateCallback, true);
2074
2075 #ifndef Z7_ST
2076
2077 CStreamBinder sb;
2078 /*
2079 if (options.MultiThreadMixer)
2080 {
2081 RINOK(sb.CreateEvents());
2082 }
2083 */
2084
2085 #endif
2086
2087 CThreadDecoder threadDecoder(options.MultiThreadMixer);
2088
2089 #ifndef Z7_ST
2090 if (options.MultiThreadMixer && thereAreRepacks)
2091 {
2092 #ifdef Z7_EXTERNAL_CODECS
2093 threadDecoder._externalCodecs = _externalCodecs;
2094 #endif
2095 const WRes wres = threadDecoder.Create();
2096 if (wres != 0)
2097 return HRESULT_FROM_WIN32(wres);
2098 }
2099 #endif
2100
2101 {
2102 CAnalysis analysis;
2103 // analysis.Need_ATime = options.Need_ATime;
2104 int analysisLevel = options.AnalysisLevel;
2105 // (analysisLevel < 0) means default level (5)
2106 if (analysisLevel < 0)
2107 analysisLevel = 5;
2108 if (analysisLevel != 0)
2109 {
2110 analysis.Callback = opCallback;
2111 analysis.ParseWav = true;
2112 if (analysisLevel >= 5)
2113 {
2114 analysis.ParseExe = true;
2115 analysis.ParseExeUnix = true;
2116 // analysis.ParseNoExt = true;
2117 if (analysisLevel >= 7)
2118 {
2119 analysis.ParseNoExt = true;
2120 if (analysisLevel >= 9)
2121 analysis.ParseAll = true;
2122 }
2123 }
2124 }
2125
2126 // ---------- Split files to groups ----------
2127
2128 const CCompressionMethodMode &method = *options.Method;
2129
2130 FOR_VECTOR (i, updateItems)
2131 {
2132 const CUpdateItem &ui = updateItems[i];
2133 if (!ui.NewData || !ui.HasStream())
2134 continue;
2135
2136 CFilterMode2 fm;
2137 if (useFilters)
2138 {
2139 // analysis.ATime_Defined = false;
2140 RINOK(analysis.GetFilterGroup(i, ui, fm))
2141 /*
2142 if (analysis.ATime_Defined)
2143 {
2144 ui.ATime = FILETIME_To_UInt64(analysis.ATime);
2145 ui.ATime_WasReadByAnalysis = true;
2146 }
2147 */
2148 }
2149 fm.Encrypted = method.PasswordIsDefined;
2150
2151 const unsigned groupIndex = GetGroup(filters, fm);
2152 while (groupIndex >= groups.Size())
2153 groups.AddNew();
2154 groups[groupIndex].Indices.Add(i);
2155 }
2156 }
2157
2158
2159 #ifndef Z7_NO_CRYPTO
2160
2161 CCryptoGetTextPassword *getPasswordSpec = NULL;
2162 CMyComPtr<ICryptoGetTextPassword> getTextPassword;
2163 if (needEncryptedRepack)
2164 {
2165 getPasswordSpec = new CCryptoGetTextPassword;
2166 getTextPassword = getPasswordSpec;
2167
2168 #ifndef Z7_ST
2169 threadDecoder.getTextPassword = getPasswordSpec;
2170 #endif
2171
2172 if (options.Method->PasswordIsDefined)
2173 getPasswordSpec->Password = options.Method->Password;
2174 else
2175 {
2176 Z7_DECL_CMyComPtr_QI_FROM(
2177 ICryptoGetTextPassword,
2178 getDecoderPassword, updateCallback)
2179 if (!getDecoderPassword)
2180 return E_NOTIMPL;
2181 CMyComBSTR password;
2182 RINOK(getDecoderPassword->CryptoGetTextPassword(&password))
2183 if (password)
2184 getPasswordSpec->Password = password;
2185 }
2186 }
2187
2188 #endif
2189
2190 // ---------- Compress ----------
2191
2192 COutArchive archive;
2193 CArchiveDatabaseOut newDatabase;
2194
2195 RINOK(archive.Create_and_WriteStartPrefix(seqOutStream))
2196
2197 /*
2198 CIntVector treeFolderToArcIndex;
2199 treeFolderToArcIndex.Reserve(treeFolders.Size());
2200 for (i = 0; i < treeFolders.Size(); i++)
2201 treeFolderToArcIndex.Add(-1);
2202 // ---------- Write Tree (only AUX dirs) ----------
2203 for (i = 1; i < treeFolders.Size(); i++)
2204 {
2205 const CTreeFolder &treeFolder = treeFolders[i];
2206 CFileItem file;
2207 CFileItem2 file2;
2208 file2.Init();
2209 int secureID = 0;
2210 if (treeFolder.UpdateItemIndex < 0)
2211 {
2212 // we can store virtual dir item wuthout attrib, but we want all items have attrib.
2213 file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
2214 file2.IsAux = true;
2215 }
2216 else
2217 {
2218 const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
2219 // if item is not dir, then it's parent for alt streams.
2220 // we will write such items later
2221 if (!ui.IsDir)
2222 continue;
2223 secureID = ui.SecureIndex;
2224 if (ui.NewProps)
2225 UpdateItem_To_FileItem(ui, file, file2);
2226 else
2227 GetFile(*db, ui.IndexInArchive, file, file2);
2228 }
2229 file.Size = 0;
2230 file.HasStream = false;
2231 file.IsDir = true;
2232 file.Parent = treeFolder.Parent;
2233
2234 treeFolderToArcIndex[i] = newDatabase.Files.Size();
2235 newDatabase.AddFile(file, file2, treeFolder.Name);
2236
2237 if (totalSecureDataSize != 0)
2238 newDatabase.SecureIDs.Add(secureID);
2239 }
2240 */
2241
2242 {
2243 /* ---------- Write non-AUX dirs and Empty files ---------- */
2244 CUIntVector emptyRefs;
2245
2246 unsigned i;
2247
2248 for (i = 0; i < updateItems.Size(); i++)
2249 {
2250 const CUpdateItem &ui = updateItems[i];
2251 if (ui.NewData)
2252 {
2253 if (ui.HasStream())
2254 continue;
2255 }
2256 else if (ui.IndexInArchive != -1 && db->Files[(unsigned)ui.IndexInArchive].HasStream)
2257 continue;
2258 /*
2259 if (ui.TreeFolderIndex >= 0)
2260 continue;
2261 */
2262 emptyRefs.Add(i);
2263 }
2264
2265 emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
2266
2267 for (i = 0; i < emptyRefs.Size(); i++)
2268 {
2269 const CUpdateItem &ui = updateItems[emptyRefs[i]];
2270 CFileItem file;
2271 CFileItem2 file2;
2272 UString name;
2273 if (ui.NewProps)
2274 {
2275 UpdateItem_To_FileItem(ui, file, file2);
2276 file.CrcDefined = false;
2277 name = ui.Name;
2278 }
2279 else
2280 {
2281 GetFile(*db, (unsigned)ui.IndexInArchive, file, file2);
2282 db->GetPath((unsigned)ui.IndexInArchive, name);
2283 }
2284
2285 /*
2286 if (totalSecureDataSize != 0)
2287 newDatabase.SecureIDs.Add(ui.SecureIndex);
2288 file.Parent = ui.ParentFolderIndex;
2289 */
2290 newDatabase.AddFile(file, file2, name);
2291 }
2292 }
2293
2294 lps->ProgressOffset = 0;
2295
2296 {
2297 // ---------- Sort Filters ----------
2298 FOR_VECTOR (i, filters)
2299 {
2300 filters[i].GroupIndex = i;
2301 }
2302 filters.Sort2();
2303 }
2304
2305 for (unsigned groupIndex = 0; groupIndex < filters.Size(); groupIndex++)
2306 {
2307 const CFilterMode2 &filterMode = filters[groupIndex];
2308
2309 CCompressionMethodMode method = *options.Method;
2310 {
2311 const HRESULT res = MakeExeMethod(method, filterMode,
2312 // bcj2_IsAllowed:
2313 #ifdef Z7_ST
2314 false
2315 #else
2316 options.MaxFilter && options.MultiThreadMixer
2317 #endif
2318 , options.DisabledFilterIDs);
2319
2320 RINOK(res)
2321 }
2322
2323 if (filterMode.Encrypted)
2324 {
2325 if (!method.PasswordIsDefined)
2326 {
2327 #ifndef Z7_NO_CRYPTO
2328 if (getPasswordSpec)
2329 method.Password = getPasswordSpec->Password;
2330 #endif
2331 method.PasswordIsDefined = true;
2332 }
2333 }
2334 else
2335 {
2336 method.PasswordIsDefined = false;
2337 method.Password.Empty();
2338 }
2339
2340 CEncoder encoder(method);
2341
2342 // ---------- Repack and copy old solid blocks ----------
2343
2344 const CSolidGroup &group = groups[filterMode.GroupIndex];
2345
2346 FOR_VECTOR (folderRefIndex, group.folderRefs)
2347 {
2348 const CFolderRepack &rep = group.folderRefs[folderRefIndex];
2349
2350 const unsigned folderIndex = rep.FolderIndex;
2351
2352 const CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
2353
2354 if (rep.NumCopyFiles == numUnpackStreams)
2355 {
2356 if (opCallback)
2357 {
2358 RINOK(opCallback->ReportOperation(
2359 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2360 NUpdateNotifyOp::kReplicate))
2361
2362 // ---------- Copy old solid block ----------
2363 {
2364 CNum indexInFolder = 0;
2365 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2366 {
2367 if (db->Files[fi].HasStream)
2368 {
2369 indexInFolder++;
2370 RINOK(opCallback->ReportOperation(
2371 NEventIndexType::kInArcIndex, (UInt32)fi,
2372 NUpdateNotifyOp::kReplicate))
2373 }
2374 }
2375 }
2376 }
2377
2378 const UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
2379 RINOK(WriteRange(inStream, archive.SeqStream,
2380 db->GetFolderStreamPos(folderIndex, 0), packSize, lps))
2381 lps->ProgressOffset += packSize;
2382
2383 const unsigned folderIndex_New = newDatabase.Folders.Size();
2384 CFolder &folder = newDatabase.Folders.AddNew();
2385 // v23.01: we copy FolderCrc, if FolderCrc was used
2386 if (db->FolderCRCs.ValidAndDefined(folderIndex))
2387 newDatabase.FolderUnpackCRCs.SetItem(folderIndex_New,
2388 true, db->FolderCRCs.Vals[folderIndex]);
2389
2390 db->ParseFolderInfo(folderIndex, folder);
2391 const CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
2392 FOR_VECTOR (j, folder.PackStreams)
2393 {
2394 newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
2395 // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
2396 // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
2397 }
2398
2399 size_t indexStart = db->FoToCoderUnpackSizes[folderIndex];
2400 const size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
2401 for (; indexStart < indexEnd; indexStart++)
2402 newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes.ConstData()[indexStart]);
2403 }
2404 else
2405 {
2406 // ---------- Repack old solid block ----------
2407
2408 CBoolVector extractStatuses;
2409
2410 CNum indexInFolder = 0;
2411
2412 if (opCallback)
2413 {
2414 RINOK(opCallback->ReportOperation(
2415 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2416 NUpdateNotifyOp::kRepack))
2417 }
2418
2419 /* We could reduce data size of decoded folder, if we don't need to repack
2420 last files in folder. But the gain in speed is small in most cases.
2421 So we unpack full folder. */
2422
2423 UInt64 sizeToEncode = 0;
2424
2425 /*
2426 UInt64 importantUnpackSize = 0;
2427 unsigned numImportantFiles = 0;
2428 UInt64 decodeSize = 0;
2429 */
2430
2431 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2432 {
2433 bool needExtract = false;
2434 const CFileItem &file = db->Files[fi];
2435
2436 if (file.HasStream)
2437 {
2438 indexInFolder++;
2439 const int updateIndex = fileIndexToUpdateIndexMap[fi];
2440 if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData)
2441 needExtract = true;
2442 // decodeSize += file.Size;
2443 }
2444
2445 extractStatuses.Add(needExtract);
2446 if (needExtract)
2447 {
2448 sizeToEncode += file.Size;
2449 /*
2450 numImportantFiles = extractStatuses.Size();
2451 importantUnpackSize = decodeSize;
2452 */
2453 }
2454 }
2455
2456 // extractStatuses.DeleteFrom(numImportantFiles);
2457
2458 unsigned startPackIndex = newDatabase.PackSizes.Size();
2459 UInt64 curUnpackSize;
2460 {
2461 CMyComPtr<ISequentialInStream> sbInStream;
2462 CRepackStreamBase *repackBase;
2463 CFolderInStream2 *FosSpec2 = NULL;
2464
2465 CRepackInStreamWithSizes *inStreamSizeCountSpec = new CRepackInStreamWithSizes;
2466 CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec;
2467 {
2468 #ifndef Z7_ST
2469 if (options.MultiThreadMixer)
2470 {
2471 repackBase = threadDecoder.FosSpec;
2472 CMyComPtr<ISequentialOutStream> sbOutStream;
2473 sb.CreateStreams2(sbInStream, sbOutStream);
2474 RINOK(sb.Create_ReInit())
2475
2476 threadDecoder.FosSpec->_stream = sbOutStream;
2477
2478 threadDecoder.InStream = inStream;
2479 threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
2480 threadDecoder.Folders = (const CFolders *)db;
2481 threadDecoder.FolderIndex = folderIndex;
2482
2483 // threadDecoder.UnpackSize = importantUnpackSize;
2484 // threadDecoder.send_UnpackSize = true;
2485 }
2486 else
2487 #endif
2488 {
2489 FosSpec2 = new CFolderInStream2;
2490 FosSpec2->Init();
2491 sbInStream = FosSpec2;
2492 repackBase = FosSpec2;
2493
2494 #ifndef Z7_NO_CRYPTO
2495 bool isEncrypted = false;
2496 bool passwordIsDefined = false;
2497 UString password;
2498 #endif
2499
2500 CMyComPtr<ISequentialInStream> decodedStream;
2501 bool dataAfterEnd_Error = false;
2502
2503 const HRESULT res = threadDecoder.Decoder.Decode(
2504 EXTERNAL_CODECS_LOC_VARS
2505 inStream,
2506 db->ArcInfo.DataStartPosition, // db->GetFolderStreamPos(folderIndex, 0);,
2507 *db, folderIndex,
2508 // &importantUnpackSize, // *unpackSize
2509 NULL, // *unpackSize : FULL unpack
2510
2511 NULL, // *outStream
2512 NULL, // *compressProgress
2513
2514 &decodedStream
2515 , dataAfterEnd_Error
2516
2517 Z7_7Z_DECODER_CRYPRO_VARS
2518 #ifndef Z7_ST
2519 , false // mtMode
2520 , 1 // numThreads
2521 , 0 // memUsage
2522 #endif
2523 );
2524
2525 RINOK(res)
2526 if (!decodedStream)
2527 return E_FAIL;
2528
2529 FosSpec2->_inStream = decodedStream;
2530 }
2531
2532 repackBase->_db = db;
2533 repackBase->_opCallback = opCallback;
2534 repackBase->_extractCallback = extractCallback;
2535
2536 UInt32 startIndex = db->FolderStartFileIndex[folderIndex];
2537 RINOK(repackBase->Init(startIndex, &extractStatuses))
2538
2539 inStreamSizeCountSpec->_db = db;
2540 inStreamSizeCountSpec->Init(sbInStream, startIndex, &extractStatuses);
2541
2542 #ifndef Z7_ST
2543 if (options.MultiThreadMixer)
2544 {
2545 WRes wres = threadDecoder.Start();
2546 if (wres != 0)
2547 return HRESULT_FROM_WIN32(wres);
2548 }
2549 #endif
2550 }
2551
2552 // curUnpackSize = sizeToEncode;
2553
2554 HRESULT encodeRes = encoder.Encode1(
2555 EXTERNAL_CODECS_LOC_VARS
2556 inStreamSizeCount,
2557 // NULL,
2558 &inSizeForReduce,
2559 sizeToEncode, // expectedDataSize
2560 newDatabase.Folders.AddNew(),
2561 // newDatabase.CoderUnpackSizes, curUnpackSize,
2562 archive.SeqStream, newDatabase.PackSizes, lps);
2563
2564 if (encodeRes == k_My_HRESULT_CRC_ERROR)
2565 return E_FAIL;
2566
2567 curUnpackSize = inStreamSizeCountSpec->GetSize();
2568
2569 if (encodeRes == S_OK)
2570 {
2571 encoder.Encode_Post(curUnpackSize, newDatabase.CoderUnpackSizes);
2572 }
2573
2574 #ifndef Z7_ST
2575 if (options.MultiThreadMixer)
2576 {
2577 // 16.00: hang was fixed : for case if decoding was not finished.
2578 // We close CBinderInStream and it calls CStreamBinder::CloseRead()
2579 inStreamSizeCount.Release();
2580 sbInStream.Release();
2581
2582 {
2583 const WRes wres = threadDecoder.WaitExecuteFinish();
2584 if (wres != 0)
2585 return HRESULT_FROM_WIN32(wres);
2586 }
2587
2588 const HRESULT decodeRes = threadDecoder.Result;
2589 // if (res == k_My_HRESULT_CRC_ERROR)
2590 if (decodeRes == S_FALSE || threadDecoder.dataAfterEnd_Error)
2591 {
2592 if (extractCallback)
2593 {
2594 RINOK(extractCallback->ReportExtractResult(
2595 NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex],
2596 // NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2597 (decodeRes != S_OK ?
2598 NExtract::NOperationResult::kDataError :
2599 NExtract::NOperationResult::kDataAfterEnd)))
2600 }
2601 if (decodeRes != S_OK)
2602 return E_FAIL;
2603 }
2604 RINOK(decodeRes)
2605 if (encodeRes == S_OK)
2606 if (sb.ProcessedSize != sizeToEncode)
2607 encodeRes = E_FAIL;
2608 }
2609 else
2610 #endif
2611 {
2612 if (FosSpec2->Result == S_FALSE)
2613 {
2614 if (extractCallback)
2615 {
2616 RINOK(extractCallback->ReportExtractResult(
2617 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2618 NExtract::NOperationResult::kDataError))
2619 }
2620 return E_FAIL;
2621 }
2622 RINOK(FosSpec2->Result)
2623 }
2624
2625 RINOK(encodeRes)
2626 RINOK(repackBase->CheckFinishedState())
2627
2628 if (curUnpackSize != sizeToEncode)
2629 return E_FAIL;
2630 }
2631
2632 for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2633 lps->OutSize += newDatabase.PackSizes[startPackIndex];
2634 lps->InSize += curUnpackSize;
2635 }
2636
2637 newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
2638
2639 CNum indexInFolder = 0;
2640 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2641 {
2642 if (db->Files[fi].HasStream)
2643 {
2644 indexInFolder++;
2645 const int updateIndex = fileIndexToUpdateIndexMap[fi];
2646 if (updateIndex >= 0)
2647 {
2648 const CUpdateItem &ui = updateItems[(unsigned)updateIndex];
2649 if (ui.NewData)
2650 continue;
2651
2652 UString name;
2653 CFileItem file;
2654 CFileItem2 file2;
2655 GetFile(*db, fi, file, file2);
2656
2657 if (ui.NewProps)
2658 {
2659 UpdateItem_To_FileItem2(ui, file2);
2660 file.IsDir = ui.IsDir;
2661 name = ui.Name;
2662 }
2663 else
2664 db->GetPath(fi, name);
2665
2666 /*
2667 file.Parent = ui.ParentFolderIndex;
2668 if (ui.TreeFolderIndex >= 0)
2669 treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2670 if (totalSecureDataSize != 0)
2671 newDatabase.SecureIDs.Add(ui.SecureIndex);
2672 */
2673 newDatabase.AddFile(file, file2, name);
2674 }
2675 }
2676 }
2677 }
2678
2679
2680 // ---------- Compress files to new solid blocks ----------
2681
2682 const unsigned numFiles = group.Indices.Size();
2683 if (numFiles == 0)
2684 continue;
2685 CRecordVector<CRefItem> refItems;
2686 refItems.ClearAndSetSize(numFiles);
2687 // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1
2688 const bool sortByType = options.UseTypeSorting;
2689
2690 unsigned i;
2691
2692 for (i = 0; i < numFiles; i++)
2693 refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
2694
2695 CSortParam sortParam;
2696 // sortParam.TreeFolders = &treeFolders;
2697 sortParam.SortByType = sortByType;
2698 refItems.Sort(CompareUpdateItems, (void *)&sortParam);
2699
2700 CObjArray<UInt32> indices(numFiles);
2701
2702 for (i = 0; i < numFiles; i++)
2703 {
2704 const UInt32 index = refItems[i].Index;
2705 indices[i] = index;
2706 /*
2707 const CUpdateItem &ui = updateItems[index];
2708 CFileItem file;
2709 if (ui.NewProps)
2710 UpdateItem_To_FileItem(ui, file);
2711 else
2712 file = db.Files[ui.IndexInArchive];
2713 if (file.IsAnti || file.IsDir)
2714 return E_FAIL;
2715 newDatabase.Files.Add(file);
2716 */
2717 }
2718
2719 for (i = 0; i < numFiles;)
2720 {
2721 UInt64 totalSize = 0;
2722 unsigned numSubFiles;
2723
2724 const wchar_t *prevExtension = NULL;
2725
2726 for (numSubFiles = 0; i + numSubFiles < numFiles && numSubFiles < numSolidFiles; numSubFiles++)
2727 {
2728 const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
2729 totalSize += ui.Size;
2730 if (totalSize > options.NumSolidBytes)
2731 break;
2732 if (options.SolidExtension)
2733 {
2734 const int slashPos = ui.Name.ReverseFind_PathSepar();
2735 const int dotPos = ui.Name.ReverseFind_Dot();
2736 const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : (unsigned)(dotPos + 1));
2737 if (numSubFiles == 0)
2738 prevExtension = ext;
2739 else if (!StringsAreEqualNoCase(ext, prevExtension))
2740 break;
2741 }
2742 }
2743
2744 if (numSubFiles < 1)
2745 numSubFiles = 1;
2746
2747 RINOK(lps->SetCur())
2748
2749 /*
2750 const unsigned folderIndex = newDatabase.NumUnpackStreamsVector.Size();
2751
2752 if (opCallback)
2753 {
2754 RINOK(opCallback->ReportOperation(
2755 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2756 NUpdateNotifyOp::kAdd));
2757 }
2758 */
2759
2760
2761 CMyComPtr2_Create<ISequentialInStream, CFolderInStream> inStreamSpec; // solidInStream;
2762
2763 // inStreamSpec->_reportArcProp = reportArcProp;
2764
2765 inStreamSpec->Need_CTime = options.Need_CTime;
2766 inStreamSpec->Need_ATime = options.Need_ATime;
2767 inStreamSpec->Need_MTime = options.Need_MTime;
2768 inStreamSpec->Need_Attrib = options.Need_Attrib;
2769 // inStreamSpec->Need_Crc = options.Need_Crc;
2770
2771 inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
2772
2773 unsigned startPackIndex = newDatabase.PackSizes.Size();
2774 // UInt64 curFolderUnpackSize = totalSize;
2775 // curFolderUnpackSize = (UInt64)(Int64)-1; // for debug
2776 const UInt64 expectedDataSize = totalSize;
2777
2778 // const unsigned folderIndex_New = newDatabase.Folders.Size();
2779
2780 RINOK(encoder.Encode1(
2781 EXTERNAL_CODECS_LOC_VARS
2782 inStreamSpec,
2783 // NULL,
2784 &inSizeForReduce,
2785 expectedDataSize, // expected size
2786 newDatabase.Folders.AddNew(),
2787 // newDatabase.CoderUnpackSizes, curFolderUnpackSize,
2788 archive.SeqStream, newDatabase.PackSizes, lps))
2789
2790 if (!inStreamSpec->WasFinished())
2791 return E_FAIL;
2792
2793 /*
2794 if (inStreamSpec->Need_FolderCrc)
2795 newDatabase.FolderUnpackCRCs.SetItem(folderIndex_New,
2796 true, inStreamSpec->GetFolderCrc());
2797 */
2798
2799 const UInt64 curFolderUnpackSize = inStreamSpec->Get_TotalSize_for_Coder();
2800 encoder.Encode_Post(curFolderUnpackSize, newDatabase.CoderUnpackSizes);
2801
2802 UInt64 packSize = 0;
2803 // const UInt32 numStreams = newDatabase.PackSizes.Size() - startPackIndex;
2804 for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2805 packSize += newDatabase.PackSizes[startPackIndex];
2806 lps->OutSize += packSize;
2807
2808 // for ()
2809 // newDatabase.PackCRCsDefined.Add(false);
2810 // newDatabase.PackCRCs.Add(0);
2811
2812 CNum numUnpackStreams = 0;
2813 UInt64 skippedSize = 0;
2814 UInt64 procSize = 0;
2815 // unsigned numProcessedFiles = 0;
2816
2817 for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++)
2818 {
2819 const CUpdateItem &ui = updateItems[indices[i + subIndex]];
2820 CFileItem file;
2821 CFileItem2 file2;
2822 UString name;
2823 if (ui.NewProps)
2824 {
2825 UpdateItem_To_FileItem(ui, file, file2);
2826 name = ui.Name;
2827 }
2828 else
2829 {
2830 GetFile(*db, (unsigned)ui.IndexInArchive, file, file2);
2831 db->GetPath((unsigned)ui.IndexInArchive, name);
2832 }
2833 if (file2.IsAnti || file.IsDir)
2834 return E_FAIL;
2835
2836 /*
2837 CFileItem &file = newDatabase.Files[
2838 startFileIndexInDatabase + i + subIndex];
2839 */
2840 if (!inStreamSpec->Processed[subIndex])
2841 {
2842 // we don't add file here
2843 skippedSize += ui.Size;
2844 continue; // comment it for debug
2845 // name += ".locked"; // for debug
2846 }
2847
2848 // if (inStreamSpec->Need_Crc)
2849 file.Crc = inStreamSpec->CRCs[subIndex];
2850 file.Size = inStreamSpec->Sizes[subIndex];
2851
2852 procSize += file.Size;
2853 // if (file.Size >= 0) // for debug: test purposes
2854 if (file.Size != 0)
2855 {
2856 file.CrcDefined = true; // inStreamSpec->Need_Crc;
2857 file.HasStream = true;
2858 numUnpackStreams++;
2859 }
2860 else
2861 {
2862 file.CrcDefined = false;
2863 file.HasStream = false;
2864 }
2865
2866 if (inStreamSpec->TimesDefined[subIndex])
2867 {
2868 if (inStreamSpec->Need_CTime)
2869 { file2.CTimeDefined = true; file2.CTime = inStreamSpec->CTimes[subIndex]; }
2870 if (inStreamSpec->Need_ATime
2871 // && !ui.ATime_WasReadByAnalysis
2872 )
2873 { file2.ATimeDefined = true; file2.ATime = inStreamSpec->ATimes[subIndex]; }
2874 if (inStreamSpec->Need_MTime)
2875 { file2.MTimeDefined = true; file2.MTime = inStreamSpec->MTimes[subIndex]; }
2876 if (inStreamSpec->Need_Attrib)
2877 {
2878 file2.AttribDefined = true;
2879 file2.Attrib = inStreamSpec->Attribs[subIndex];
2880 }
2881 }
2882
2883 /*
2884 file.Parent = ui.ParentFolderIndex;
2885 if (ui.TreeFolderIndex >= 0)
2886 treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2887 if (totalSecureDataSize != 0)
2888 newDatabase.SecureIDs.Add(ui.SecureIndex);
2889 */
2890 /*
2891 if (reportArcProp)
2892 {
2893 RINOK(ReportItemProps(reportArcProp, ui.IndexInClient, file.Size,
2894 file.CrcDefined ? &file.Crc : NULL))
2895 }
2896 */
2897
2898 // numProcessedFiles++;
2899 newDatabase.AddFile(file, file2, name);
2900 }
2901
2902 /*
2903 // for debug:
2904 // we can write crc to folders area, if folder contains only one file
2905 if (numUnpackStreams == 1 && numSubFiles == 1)
2906 {
2907 const CFileItem &file = newDatabase.Files.Back();
2908 if (file.CrcDefined)
2909 newDatabase.FolderUnpackCRCs.SetItem(folderIndex_New, true, file.Crc);
2910 }
2911 */
2912
2913 /*
2914 // it's optional check to ensure that sizes are correct
2915 if (inStreamSpec->TotalSize_for_Coder != curFolderUnpackSize)
2916 return E_FAIL;
2917 */
2918 // if (inStreamSpec->AlignLog == 0)
2919 {
2920 if (procSize != curFolderUnpackSize)
2921 return E_FAIL;
2922 }
2923 // else
2924 {
2925 /*
2926 {
2927 const CFolder &old = newDatabase.Folders.Back();
2928 CFolder &folder = newDatabase.Folders.AddNew();
2929 {
2930 const unsigned numBonds = old.Bonds.Size();
2931 folder.Bonds.SetSize(numBonds + 1);
2932 for (unsigned k = 0; k < numBonds; k++)
2933 folder.Bonds[k] = old.Bonds[k];
2934 CBond &bond = folder.Bonds[numBonds];
2935 bond.PackIndex = 0;
2936 bond.UnpackIndex = 0;
2937 }
2938 {
2939 const unsigned numCoders = old.Coders.Size();
2940 folder.Coders.SetSize(numCoders + 1);
2941 for (unsigned k = 0; k < numCoders; k++)
2942 folder.Coders[k] = old.Coders[k];
2943 CCoderInfo &cod = folder.Coders[numCoders];
2944 cod.Props.Alloc(1);
2945 cod.Props[0] = (Byte)inStreamSpec->AlignLog;
2946 cod.NumStreams = 1;
2947 }
2948 {
2949 const unsigned numPackStreams = old.Coders.Size();
2950 folder.Coders.SetSize(numPackStreams);
2951 for (unsigned k = 0; k < numPackStreams; k++)
2952 folder.PackStreams[k] = old.PackStreams[k];
2953 }
2954 }
2955 newDatabase.Folders.Delete(newDatabase.Folders.Size() - 2);
2956 */
2957 }
2958
2959
2960 lps->InSize += procSize;
2961 // lps->InSize += curFolderUnpackSize;
2962
2963 // numUnpackStreams = 0 is very bad case for locked files
2964 // v3.13 doesn't understand it.
2965 newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
2966 i += numSubFiles;
2967
2968 if (skippedSize != 0 && complexity >= skippedSize)
2969 {
2970 complexity -= skippedSize;
2971 RINOK(updateCallback->SetTotal(complexity))
2972 }
2973
2974 /*
2975 if (reportArcProp)
2976 {
2977 PROPVARIANT prop;
2978 prop.vt = VT_EMPTY;
2979 prop.wReserved1 = 0;
2980 {
2981 NWindows::NCOM::PropVarEm_Set_UInt32(&prop, numProcessedFiles);
2982 RINOK(reportArcProp->ReportProp(
2983 NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidNumSubFiles, &prop));
2984 }
2985 {
2986 NWindows::NCOM::PropVarEm_Set_UInt64(&prop, curFolderUnpackSize);
2987 RINOK(reportArcProp->ReportProp(
2988 NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidSize, &prop));
2989 }
2990 {
2991 NWindows::NCOM::PropVarEm_Set_UInt64(&prop, packSize);
2992 RINOK(reportArcProp->ReportProp(
2993 NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidPackSize, &prop));
2994 }
2995 {
2996 NWindows::NCOM::PropVarEm_Set_UInt32(&prop, numStreams);
2997 RINOK(reportArcProp->ReportProp(
2998 NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidNumStreams, &prop));
2999 }
3000 RINOK(reportArcProp->ReportFinished(NEventIndexType::kBlockIndex, (UInt32)folderIndex, NUpdate::NOperationResult::kOK));
3001 }
3002 */
3003 /*
3004 if (opCallback)
3005 {
3006 RINOK(opCallback->ReportOperation(
3007 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
3008 NUpdateNotifyOp::kOpFinished));
3009 }
3010 */
3011 }
3012 }
3013
3014 RINOK(lps->SetCur())
3015
3016 /*
3017 fileIndexToUpdateIndexMap.ClearAndFree();
3018 groups.ClearAndFree();
3019 */
3020
3021 /*
3022 for (i = 0; i < newDatabase.Files.Size(); i++)
3023 {
3024 CFileItem &file = newDatabase.Files[i];
3025 file.Parent = treeFolderToArcIndex[file.Parent];
3026 }
3027
3028 if (totalSecureDataSize != 0)
3029 {
3030 newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
3031 size_t pos = 0;
3032 newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
3033 for (i = 0; i < secureBlocks.Sorted.Size(); i++)
3034 {
3035 const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
3036 size_t size = buf.GetCapacity();
3037 if (size != 0)
3038 memcpy(newDatabase.SecureBuf + pos, buf, size);
3039 newDatabase.SecureSizes.Add((UInt32)size);
3040 pos += size;
3041 }
3042 }
3043 */
3044
3045 {
3046 const unsigned numFolders = newDatabase.Folders.Size();
3047 if (newDatabase.NumUnpackStreamsVector.Size() != numFolders
3048 || newDatabase.FolderUnpackCRCs.Defs.Size() > numFolders)
3049 return E_FAIL;
3050 newDatabase.FolderUnpackCRCs.if_NonEmpty_FillResidue_with_false(numFolders);
3051 }
3052
3053 updateItems.ClearAndFree();
3054 newDatabase.ReserveDown();
3055
3056 if (opCallback)
3057 RINOK(opCallback->ReportOperation(NEventIndexType::kNoIndex, (UInt32)(Int32)-1, NUpdateNotifyOp::kHeader))
3058
3059 RINOK(archive.WriteDatabase(EXTERNAL_CODECS_LOC_VARS
3060 newDatabase, options.HeaderMethod, options.HeaderOptions))
3061
3062 if (v_StreamSetRestriction)
3063 RINOK(v_StreamSetRestriction->SetRestriction(0, 0))
3064
3065 return S_OK;
3066 }
3067
3068 }}
3069