Lines Matching +full:libzstd +full:- +full:size
5 * This source code is licensed under both the BSD-style license (found in the
8 * You may select, at your option, one of the above-listed licenses.
17 # pragma warning(disable : 4204) /* non-constant aggregate initializer */
23 /*-*************************************
108 /*-*************************************
128 /*-************************************
129 * Signal (Ctrl-C trapping)
162 /*-*********************************************************
238 /*-*************************************
259 return fCtx->nbFilesTotal <= 1 || g_display_prefs.displayLevel >= 3; in FIO_shouldDisplayFileSummary()
264 int const shouldDisplay = (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1); in FIO_shouldDisplayMultipleFileSummary()
265 assert(shouldDisplay || FIO_shouldDisplayFileSummary(fCtx) || fCtx->nbFilesProcessed == 0); in FIO_shouldDisplayMultipleFileSummary()
270 /*-*************************************
283 ret->compressionType = FIO_zstdCompression; in FIO_createPreferences()
284 ret->overwrite = 0; in FIO_createPreferences()
285 ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT; in FIO_createPreferences()
286 ret->dictIDFlag = 1; in FIO_createPreferences()
287 ret->checksumFlag = 1; in FIO_createPreferences()
288 ret->removeSrcFile = 0; in FIO_createPreferences()
289 ret->memLimit = 0; in FIO_createPreferences()
290 ret->nbWorkers = 1; in FIO_createPreferences()
291 ret->blockSize = 0; in FIO_createPreferences()
292 ret->overlapLog = FIO_OVERLAP_LOG_NOTSET; in FIO_createPreferences()
293 ret->adaptiveMode = 0; in FIO_createPreferences()
294 ret->rsyncable = 0; in FIO_createPreferences()
295 …ret->minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() do… in FIO_createPreferences()
296 …ret->maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doe… in FIO_createPreferences()
297 ret->ldmFlag = 0; in FIO_createPreferences()
298 ret->ldmHashLog = 0; in FIO_createPreferences()
299 ret->ldmMinMatch = 0; in FIO_createPreferences()
300 ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; in FIO_createPreferences()
301 ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET; in FIO_createPreferences()
302 ret->streamSrcSize = 0; in FIO_createPreferences()
303 ret->targetCBlockSize = 0; in FIO_createPreferences()
304 ret->srcSizeHint = 0; in FIO_createPreferences()
305 ret->testMode = 0; in FIO_createPreferences()
306 ret->literalCompressionMode = ZSTD_ps_auto; in FIO_createPreferences()
307 ret->excludeCompressedFiles = 0; in FIO_createPreferences()
308 ret->allowBlockDevices = 0; in FIO_createPreferences()
309 ret->asyncIO = AIO_supported(); in FIO_createPreferences()
310 ret->passThrough = -1; in FIO_createPreferences()
319 ret->currFileIdx = 0; in FIO_createContext()
320 ret->hasStdinInput = 0; in FIO_createContext()
321 ret->hasStdoutOutput = 0; in FIO_createContext()
322 ret->nbFilesTotal = 1; in FIO_createContext()
323 ret->nbFilesProcessed = 0; in FIO_createContext()
324 ret->totalBytesInput = 0; in FIO_createContext()
325 ret->totalBytesOutput = 0; in FIO_createContext()
340 /*-*************************************
349 /*-*************************************
355 …ype(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = co… in FIO_setCompressionType()
357 void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } in FIO_overwriteMode()
359 void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse) { prefs->sparseFileSupport = sparse; } in FIO_setSparseWrite()
361 void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; } in FIO_setDictIDFlag()
363 void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checks… in FIO_setChecksumFlag()
365 void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag) { prefs->removeSrcFile = (flag!=0); } in FIO_setRemoveSrcFile()
367 void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; } in FIO_setMemLimit()
371 if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); in FIO_setNbWorkers()
373 prefs->nbWorkers = nbWorkers; in FIO_setNbWorkers()
376 void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->ex… in FIO_setExcludeCompressedFile()
378 void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockD… in FIO_setAllowBlockDevices()
381 if (blockSize && prefs->nbWorkers==0) in FIO_setBlockSize()
382 DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); in FIO_setBlockSize()
383 prefs->blockSize = blockSize; in FIO_setBlockSize()
387 if (overlapLog && prefs->nbWorkers==0) in FIO_setOverlapLog()
388 DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); in FIO_setOverlapLog()
389 prefs->overlapLog = overlapLog; in FIO_setOverlapLog()
393 if ((adapt>0) && (prefs->nbWorkers==0)) in FIO_setAdaptiveMode()
395 prefs->adaptiveMode = adapt; in FIO_setAdaptiveMode()
399 prefs->useRowMatchFinder = useRowMatchFinder; in FIO_setUseRowMatchFinder()
403 if ((rsyncable>0) && (prefs->nbWorkers==0)) in FIO_setRsyncable()
405 prefs->rsyncable = rsyncable; in FIO_setRsyncable()
409 prefs->streamSrcSize = streamSrcSize; in FIO_setStreamSrcSize()
413 prefs->targetCBlockSize = targetCBlockSize; in FIO_setTargetCBlockSize()
417 prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); in FIO_setSrcSizeHint()
421 prefs->testMode = (testMode!=0); in FIO_setTestMode()
427 prefs->literalCompressionMode = mode; in FIO_setLiteralCompressionMode()
435 prefs->minAdaptLevel = minCLevel; in FIO_setAdaptMin()
440 prefs->maxAdaptLevel = maxCLevel; in FIO_setAdaptMax()
444 prefs->ldmFlag = (ldmFlag>0); in FIO_setLdmFlag()
448 prefs->ldmHashLog = ldmHashLog; in FIO_setLdmHashLog()
452 prefs->ldmMinMatch = ldmMinMatch; in FIO_setLdmMinMatch()
456 prefs->ldmBucketSizeLog = ldmBucketSizeLog; in FIO_setLdmBucketSizeLog()
461 prefs->ldmHashRateLog = ldmHashRateLog; in FIO_setLdmHashRateLog()
466 prefs->patchFromMode = value != 0; in FIO_setPatchFromMode()
471 prefs->contentSize = value != 0; in FIO_setContentSize()
476 prefs->asyncIO = value; in FIO_setAsyncIOFlag()
485 prefs->passThrough = (value != 0); in FIO_setPassThroughFlag()
490 prefs->mmapDict = value; in FIO_setMMapDict()
496 fCtx->hasStdoutOutput = value; in FIO_setHasStdoutOutput()
501 fCtx->nbFilesTotal = value; in FIO_setNbFilesTotal()
506 for ( ; i < filenames->tableSize; ++i) { in FIO_determineHasStdinInput()
507 if (!strcmp(stdinmark, filenames->fileNames[i])) { in FIO_determineHasStdinInput()
508 fCtx->hasStdinInput = 1; in FIO_determineHasStdinInput()
514 /*-*************************************
518 * @result : Unlink `fileName`, even if it's read-only */
527 DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path); in FIO_removeFile()
531 /* windows doesn't allow remove read-only files, in FIO_removeFile()
541 * condition : `srcFileName` must be non-NULL. `prefs` may be NULL.
545 int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0; in FIO_openSrcFile()
555 DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n", in FIO_openSrcFile()
564 DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", in FIO_openSrcFile()
577 * condition : `dstFileName` must be non-NULL.
586 if (prefs->testMode) return NULL; /* do not open file in test mode */ in FIO_openDstFile()
592 if (prefs->sparseFileSupport == 1) { in FIO_openDstFile()
593 prefs->sparseFileSupport = 0; in FIO_openDstFile()
594 … DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); in FIO_openDstFile()
606 if (prefs->sparseFileSupport == 1) { in FIO_openDstFile()
607 prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT; in FIO_openDstFile()
609 prefs->sparseFileSupport = 0; in FIO_openDstFile()
624 if (!prefs->overwrite) { in FIO_openDstFile()
632 …IL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput)) in FIO_openDstFile()
646 if (fd != -1) { in FIO_openDstFile()
653 if (fd != -1) { in FIO_openDstFile()
660 /* An increased buffer size can provide a significant performance in FIO_openDstFile()
662 * size that's not 0 is not defined in ANSI C, but is defined in an in FIO_openDstFile()
665 * 2. Libc ignores the size when buf is NULL, in which case in FIO_openDstFile()
670 * this is a more cost-effective solution than managing the buffers in FIO_openDstFile()
698 * allocates a buffer, pointed by `dict->dictBuffer`,
700 * @return : loaded size
707 void** bufferPtr = &dict->dictBuffer; in FIO_setDictBufferMalloc()
724 size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; in FIO_setDictBufferMalloc()
746 munmap(dict->dictBuffer, dict->dictBufferSize); in FIO_munmap()
747 dict->dictBuffer = NULL; in FIO_munmap()
748 dict->dictBufferSize = 0; in FIO_munmap()
754 void** bufferPtr = &dict->dictBuffer; in FIO_setDictBufferMMap()
765 if (fileHandle == -1) { in FIO_setDictBufferMMap()
771 size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; in FIO_setDictBufferMMap()
788 UnmapViewOfFile(dict->dictBuffer); in FIO_munmap()
789 CloseHandle(dict->dictHandle); in FIO_munmap()
790 dict->dictBuffer = NULL; in FIO_munmap()
791 dict->dictBufferSize = 0; in FIO_munmap()
797 void** bufferPtr = &dict->dictBuffer; in FIO_setDictBufferMMap()
814 size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; in FIO_setDictBufferMMap()
829 dict->dictHandle = fileHandle; in FIO_setDictBufferMMap()
838 free(dict->dictBuffer); in FIO_munmap()
839 dict->dictBuffer = NULL; in FIO_munmap()
840 dict->dictBufferSize = 0; in FIO_munmap()
845 if (dict->dictBufferType == FIO_mallocDict) { in FIO_freeDict()
846 free(dict->dictBuffer); in FIO_freeDict()
847 dict->dictBuffer = NULL; in FIO_freeDict()
848 dict->dictBufferSize = 0; in FIO_freeDict()
849 } else if (dict->dictBufferType == FIO_mmapDict) { in FIO_freeDict()
857 dict->dictBufferType = dictBufferType; in FIO_initDict()
858 if (dict->dictBufferType == FIO_mallocDict) { in FIO_initDict()
859 dict->dictBufferSize = FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat); in FIO_initDict()
860 } else if (dict->dictBufferType == FIO_mmapDict) { in FIO_initDict()
861 dict->dictBufferSize = FIO_setDictBufferMMap(dict, fileName, prefs, dictFileStat); in FIO_initDict()
940 if (outDirName[strlen(outDirName)-1] == separator) { in FIO_createFilename_fromOutDir()
967 unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize)); in FIO_adjustMemLimitForPatchFromMode()
970 EXM_THROW(42, "Using --patch-from with stdin requires --stream-size"); in FIO_adjustMemLimitForPatchFromMode()
978 …* This function handles logic when processing multiple files with -o or -c, displaying the appropr…
981 * If output is stdout or test mode is active, check that `--rm` disabled.
987 * display a warning message, then disable --rm if it's set.
989 * If -f is specified or if output is stdout, just proceed.
990 * If output is set with -o, prompt for confirmation.
994 if (fCtx->hasStdoutOutput) { in FIO_multiFilesConcatWarning()
995 if (prefs->removeSrcFile) in FIO_multiFilesConcatWarning()
1002 if (prefs->testMode) { in FIO_multiFilesConcatWarning()
1003 if (prefs->removeSrcFile) in FIO_multiFilesConcatWarning()
1012 if (fCtx->nbFilesTotal == 1) return 0; in FIO_multiFilesConcatWarning()
1013 assert(fCtx->nbFilesTotal > 1); in FIO_multiFilesConcatWarning()
1017 if (fCtx->hasStdoutOutput) { in FIO_multiFilesConcatWarning()
1024 /* multi-input into single output : --rm is not allowed */ in FIO_multiFilesConcatWarning()
1025 if (prefs->removeSrcFile) { in FIO_multiFilesConcatWarning()
1027 prefs->removeSrcFile = 0; in FIO_multiFilesConcatWarning()
1030 if (fCtx->hasStdoutOutput) return 0; in FIO_multiFilesConcatWarning()
1031 if (prefs->overwrite) return 0; in FIO_multiFilesConcatWarning()
1033 /* multiple files concatenated into single destination file using -o without -f */ in FIO_multiFilesConcatWarning()
1041 … return UTIL_requireUserConfirmation("Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput); in FIO_multiFilesConcatWarning()
1048 i.size = s; in setInBuffer()
1057 o.size = s; in setOutBuffer()
1082 return hashLog - btScale; in ZSTD_cycleLog()
1096 comprParams->windowLog = MAX(ZSTD_WINDOWLOG_MIN, MIN(ZSTD_WINDOWLOG_MAX, fileWindowLog)); in FIO_adjustParamsForPatchFromMode()
1098 if (!prefs->ldmFlag) in FIO_adjustParamsForPatchFromMode()
1103 …DISPLAYLEVEL(3, "[Optimal parser notes] Consider the following to improve patch size at the cost o… in FIO_adjustParamsForPatchFromMode()
1104 DISPLAYLEVEL(3, "- Use --single-thread mode in the zstd cli\n"); in FIO_adjustParamsForPatchFromMode()
1105 DISPLAYLEVEL(3, "- Set a larger targetLength (e.g. --zstd=targetLength=4096)\n"); in FIO_adjustParamsForPatchFromMode()
1106 DISPLAYLEVEL(3, "- Set a larger chainLog (e.g. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX); in FIO_adjustParamsForPatchFromMode()
1114 int useMMap = prefs->mmapDict == ZSTD_ps_enable; in FIO_createCResources()
1115 int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable; in FIO_createCResources()
1130 if (prefs->patchFromMode) { in FIO_createCResources()
1132 unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize; in FIO_createCResources()
1133 useMMap |= dictSize > prefs->memLimit; in FIO_createCResources()
1148 if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog) in FIO_createCResources()
1151 …etParameter(ress.cctx, ZSTD_c_contentSizeFlag, prefs->contentSize) ); /* always enable content si… in FIO_createCResources()
1152 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) ); in FIO_createCResources()
1153 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) ); in FIO_createCResources()
1156 /* max compressed block size */ in FIO_createCResources()
1157 … CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetCBlockSize, (int)prefs->targetCBlockSize) ); in FIO_createCResources()
1158 /* source size hint */ in FIO_createCResources()
1159 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_srcSizeHint, (int)prefs->srcSizeHint) ); in FIO_createCResources()
1161 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) ); in FIO_createCResources()
1162 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) ); in FIO_createCResources()
1163 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) ); in FIO_createCResources()
1164 if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { in FIO_createCResources()
1165 … CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) ); in FIO_createCResources()
1167 if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) { in FIO_createCResources()
1168 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) ); in FIO_createCResources()
1170 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_useRowMatchFinder, prefs->useRowMatchFinder)); in FIO_createCResources()
1179 …CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompres… in FIO_createCResources()
1181 /* multi-threading */ in FIO_createCResources()
1183 DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); in FIO_createCResources()
1184 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) ); in FIO_createCResources()
1185 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) ); in FIO_createCResources()
1186 if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) { in FIO_createCResources()
1187 DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog); in FIO_createCResources()
1188 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) ); in FIO_createCResources()
1190 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) ); in FIO_createCResources()
1193 if (prefs->patchFromMode) { in FIO_createCResources()
1204 FIO_freeDict(&(ress->dict)); in FIO_freeCResources()
1205 AIO_WritePool_free(ress->writeCtx); in FIO_freeCResources()
1206 AIO_ReadPool_free(ress->readCtx); in FIO_freeCResources()
1207 ZSTD_freeCStream(ress->cctx); /* never fails */ in FIO_freeCResources()
1235 writeJob = AIO_WritePool_acquireJob(ress->writeCtx); in FIO_compressGzFrame()
1238 strm.next_out = (Bytef*)writeJob->buffer; in FIO_compressGzFrame()
1239 strm.avail_out = (uInt)writeJob->bufferSize; in FIO_compressGzFrame()
1244 AIO_ReadPool_fillBuffer(ress->readCtx, ZSTD_CStreamInSize()); in FIO_compressGzFrame()
1245 if (ress->readCtx->srcBufferLoaded == 0) break; in FIO_compressGzFrame()
1246 inFileSize += ress->readCtx->srcBufferLoaded; in FIO_compressGzFrame()
1247 strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer; in FIO_compressGzFrame()
1248 strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded; in FIO_compressGzFrame()
1254 AIO_ReadPool_consumeBytes(ress->readCtx, availBefore - strm.avail_in); in FIO_compressGzFrame()
1259 { size_t const cSize = writeJob->bufferSize - strm.avail_out; in FIO_compressGzFrame()
1261 writeJob->usedBufferSize = cSize; in FIO_compressGzFrame()
1264 strm.next_out = (Bytef*)writeJob->buffer; in FIO_compressGzFrame()
1265 strm.avail_out = (uInt)writeJob->bufferSize; in FIO_compressGzFrame()
1281 { size_t const cSize = writeJob->bufferSize - strm.avail_out; in FIO_compressGzFrame()
1283 writeJob->usedBufferSize = cSize; in FIO_compressGzFrame()
1286 strm.next_out = (Bytef*)writeJob->buffer; in FIO_compressGzFrame()
1287 strm.avail_out = (uInt)writeJob->bufferSize; in FIO_compressGzFrame()
1300 AIO_WritePool_sparseWriteEnd(ress->writeCtx); in FIO_compressGzFrame()
1334 writeJob =AIO_WritePool_acquireJob(ress->writeCtx); in FIO_compressLzmaFrame()
1335 strm.next_out = (BYTE*)writeJob->buffer; in FIO_compressLzmaFrame()
1336 strm.avail_out = writeJob->bufferSize; in FIO_compressLzmaFrame()
1342 size_t const inSize = AIO_ReadPool_fillBuffer(ress->readCtx, ZSTD_CStreamInSize()); in FIO_compressLzmaFrame()
1343 if (ress->readCtx->srcBufferLoaded == 0) action = LZMA_FINISH; in FIO_compressLzmaFrame()
1345 strm.next_in = (BYTE const*)ress->readCtx->srcBuffer; in FIO_compressLzmaFrame()
1346 strm.avail_in = ress->readCtx->srcBufferLoaded; in FIO_compressLzmaFrame()
1352 AIO_ReadPool_consumeBytes(ress->readCtx, availBefore - strm.avail_in); in FIO_compressLzmaFrame()
1358 { size_t const compBytes = writeJob->bufferSize - strm.avail_out; in FIO_compressLzmaFrame()
1360 writeJob->usedBufferSize = compBytes; in FIO_compressLzmaFrame()
1363 strm.next_out = (BYTE*)writeJob->buffer; in FIO_compressLzmaFrame()
1364 strm.avail_out = writeJob->bufferSize; in FIO_compressLzmaFrame()
1381 AIO_WritePool_sparseWriteEnd(ress->writeCtx); in FIO_compressLzmaFrame()
1408 IOJob_t* writeJob = AIO_WritePool_acquireJob(ress->writeCtx); in FIO_compressLz4Frame()
1416 assert(blockSize <= ress->readCtx->base.jobBufferSize); in FIO_compressLz4Frame()
1427 assert(LZ4F_compressBound(blockSize, &prefs) <= writeJob->bufferSize); in FIO_compressLz4Frame()
1430 size_t headerSize = LZ4F_compressBegin(ctx, writeJob->buffer, writeJob->bufferSize, &prefs); in FIO_compressLz4Frame()
1434 writeJob->usedBufferSize = headerSize; in FIO_compressLz4Frame()
1439 inFileSize += AIO_ReadPool_fillBuffer(ress->readCtx, blockSize); in FIO_compressLz4Frame()
1442 while (ress->readCtx->srcBufferLoaded) { in FIO_compressLz4Frame()
1443 size_t inSize = MIN(blockSize, ress->readCtx->srcBufferLoaded); in FIO_compressLz4Frame()
1444 size_t const outSize = LZ4F_compressUpdate(ctx, writeJob->buffer, writeJob->bufferSize, in FIO_compressLz4Frame()
1445 ress->readCtx->srcBuffer, inSize, NULL); in FIO_compressLz4Frame()
1461 writeJob->usedBufferSize = outSize; in FIO_compressLz4Frame()
1465 AIO_ReadPool_consumeBytes(ress->readCtx, inSize); in FIO_compressLz4Frame()
1466 inFileSize += AIO_ReadPool_fillBuffer(ress->readCtx, blockSize); in FIO_compressLz4Frame()
1470 headerSize = LZ4F_compressEnd(ctx, writeJob->buffer, writeJob->bufferSize, NULL); in FIO_compressLz4Frame()
1475 writeJob->usedBufferSize = headerSize; in FIO_compressLz4Frame()
1483 AIO_WritePool_sparseWriteEnd(ress->writeCtx); in FIO_compressLz4Frame()
1497 IOJob_t *writeJob = AIO_WritePool_acquireJob(ressPtr->writeCtx); in FIO_compressZstdFrame()
1523 } else if (prefs->streamSrcSize > 0) { in FIO_compressZstdFrame()
1524 /* unknown source size; use the declared stream size */ in FIO_compressZstdFrame()
1525 pledgedSrcSize = prefs->streamSrcSize; in FIO_compressZstdFrame()
1526 CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); in FIO_compressZstdFrame()
1534 if (prefs->ldmFlag) { in FIO_compressZstdFrame()
1535 … /* If long mode is set without a window size libzstd will set this size internally */ in FIO_compressZstdFrame()
1552 … ZSTD_inBuffer inBuff = setInBuffer( ress.readCtx->srcBuffer, ress.readCtx->srcBufferLoaded, 0 ); in FIO_compressZstdFrame()
1556 if ((ress.readCtx->srcBufferLoaded == 0) || (*readsize == fileSize)) in FIO_compressZstdFrame()
1560 while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ in FIO_compressZstdFrame()
1564 ZSTD_outBuffer outBuff = setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 ); in FIO_compressZstdFrame()
1567 AIO_ReadPool_consumeBytes(ress.readCtx, inBuff.pos - oldIPos); in FIO_compressZstdFrame()
1575 …DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u by… in FIO_compressZstdFrame()
1576 … (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); in FIO_compressZstdFrame()
1578 writeJob->usedBufferSize = outBuff.pos; in FIO_compressZstdFrame()
1584 if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) { in FIO_compressZstdFrame()
1592 unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced; in FIO_compressZstdFrame()
1593 unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed; in FIO_compressZstdFrame()
1595 assert(prefs->nbWorkers >= 1); in FIO_compressZstdFrame()
1624 …if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers … in FIO_compressZstdFrame()
1629 … unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested; in FIO_compressZstdFrame()
1630 … unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed; in FIO_compressZstdFrame()
1631 … unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced; in FIO_compressZstdFrame()
1632 … unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed; in FIO_compressZstdFrame()
1635 …DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produce… in FIO_compressZstdFrame()
1656 … if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel; in FIO_compressZstdFrame()
1662 compressionLevel --; in FIO_compressZstdFrame()
1663 … if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel; in FIO_compressZstdFrame()
1664 compressionLevel -= (compressionLevel == 0); /* skip 0 */ in FIO_compressZstdFrame()
1671 } /* if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) */ in FIO_compressZstdFrame()
1677 …L_HumanReadableSize_t const buffered_hrs = UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed); in FIO_compressZstdFrame()
1688 … "(L%i) Buffered:%5.*f%s - Consumed:%5.*f%s - Compressed:%5.*f%s => %.2f%% ", in FIO_compressZstdFrame()
1696 if (fCtx->nbFilesTotal > 1) { in FIO_compressZstdFrame()
1698 /* Ensure that the string we print is roughly the same size each time */ in FIO_compressZstdFrame()
1700 const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; in FIO_compressZstdFrame()
1702 … fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName); in FIO_compressZstdFrame()
1705 … fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); in FIO_compressZstdFrame()
1714 } /* while ((inBuff.pos != inBuff.size) */ in FIO_compressZstdFrame()
1723 AIO_WritePool_sparseWriteEnd(ressPtr->writeCtx); in FIO_compressZstdFrame()
1748 switch (prefs->compressionType) { in FIO_compressFilename_internal()
1759 …zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", in FIO_compressFilename_internal()
1767 …ssLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO… in FIO_compressFilename_internal()
1770 … %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", in FIO_compressFilename_internal()
1777 …= FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &reads… in FIO_compressFilename_internal()
1780 …zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", in FIO_compressFilename_internal()
1787 fCtx->totalBytesInput += (size_t)readsize; in FIO_compressFilename_internal()
1788 fCtx->totalBytesOutput += (size_t)compressedfilesize; in FIO_compressFilename_internal()
1794 DISPLAY_SUMMARY("%-20s : (%6.*f%s => %6.*f%s, %s) \n", in FIO_compressFilename_internal()
1800 DISPLAY_SUMMARY("%-20s :%6.2f%% (%6.*f%s => %6.*f%s, %s) \n", in FIO_compressFilename_internal()
1811 double const cpuLoad_s = (double)(cpuEnd - cpuStart) / CLOCKS_PER_SEC; in FIO_compressFilename_internal()
1815 DISPLAYLEVEL(4, "%-20s : Completed in %.2f sec (cpu load : %.0f%%)\n", in FIO_compressFilename_internal()
1823 * open dstFileName, or pass-through if ress.file != NULL,
1825 * Manages source removal (--rm) and file permissions transfer.
1842 int dstFd = -1; in FIO_compressFilename_dstFile()
1863 * and the user presses Ctrl-C when asked if they wish to overwrite. in FIO_compressFilename_dstFile()
1897 /* List used to compare file extensions (used with --exclude-compressed flag)
2041 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); in FIO_compressFilename_srcFile()
2053 /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used in FIO_compressFilename_srcFile()
2057 …if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensi… in FIO_compressFilename_srcFile()
2083 if ( prefs->removeSrcFile /* --rm */ in FIO_compressFilename_srcFile()
2100 /* Necessary to avoid warnings since -O3 will omit the above `assert` */ in checked_index()
2111 static const char* sparseOptions[3] = {" --no-sparse", "", " --sparse"}; in FIO_displayCompressionParameters()
2112 static const char* checkSumOptions[3] = {" --no-check", "", " --check"}; in FIO_displayCompressionParameters()
2113 …static const char* rowMatchFinderOptions[3] = {"", " --no-row-match-finder", " --row-match-finder"… in FIO_displayCompressionParameters()
2114 …static const char* compressLiteralsOptions[3] = {"", " --compress-literals", " --no-compress-liter… in FIO_displayCompressionParameters()
2118 DISPLAY("--format=%s", formatOptions[prefs->compressionType]); in FIO_displayCompressionParameters()
2119 DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport)); in FIO_displayCompressionParameters()
2120 DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID"); in FIO_displayCompressionParameters()
2121 DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag)); in FIO_displayCompressionParameters()
2122 DISPLAY(" --block-size=%d", prefs->blockSize); in FIO_displayCompressionParameters()
2123 if (prefs->adaptiveMode) in FIO_displayCompressionParameters()
2124 DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel); in FIO_displayCompressionParameters()
2125 DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder)); in FIO_displayCompressionParameters()
2126 DISPLAY("%s", prefs->rsyncable ? " --rsyncable" : ""); in FIO_displayCompressionParameters()
2127 if (prefs->streamSrcSize) in FIO_displayCompressionParameters()
2128 DISPLAY(" --stream-size=%u", (unsigned) prefs->streamSrcSize); in FIO_displayCompressionParameters()
2129 if (prefs->srcSizeHint) in FIO_displayCompressionParameters()
2130 DISPLAY(" --size-hint=%d", prefs->srcSizeHint); in FIO_displayCompressionParameters()
2131 if (prefs->targetCBlockSize) in FIO_displayCompressionParameters()
2132 DISPLAY(" --target-compressed-block-size=%u", (unsigned) prefs->targetCBlockSize); in FIO_displayCompressionParameters()
2133 DISPLAY("%s", INDEX(compressLiteralsOptions, prefs->literalCompressionMode)); in FIO_displayCompressionParameters()
2134 DISPLAY(" --memory=%u", prefs->memLimit ? prefs->memLimit : 128 MB); in FIO_displayCompressionParameters()
2135 DISPLAY(" --threads=%d", prefs->nbWorkers); in FIO_displayCompressionParameters()
2136 DISPLAY("%s", prefs->excludeCompressedFiles ? " --exclude-compressed" : ""); in FIO_displayCompressionParameters()
2137 DISPLAY(" --%scontent-size", prefs->contentSize ? "" : "no-"); in FIO_displayCompressionParameters()
2165 … dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ in FIO_determineCompressedName()
2216 * or into a destination folder (specified with -O)
2230 FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal), in FIO_compressMultipleFilenames()
2246 for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { in FIO_compressMultipleFilenames()
2247 …mpressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compre… in FIO_compressMultipleFilenames()
2248 if (!status) fCtx->nbFilesProcessed++; in FIO_compressMultipleFilenames()
2257 …UTIL_mirrorSourceFilesDirectories(inFileNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootD… in FIO_compressMultipleFilenames()
2259 for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { in FIO_compressMultipleFilenames()
2260 const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx]; in FIO_compressMultipleFilenames()
2268 …DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot compress '%s' into '%s' \n", srcFileName, outMir… in FIO_compressMultipleFilenames()
2276 if (!status) fCtx->nbFilesProcessed++; in FIO_compressMultipleFilenames()
2281 FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal); in FIO_compressMultipleFilenames()
2285 UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput); in FIO_compressMultipleFilenames()
2286 … UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); in FIO_compressMultipleFilenames()
2289 if (fCtx->totalBytesInput == 0) { in FIO_compressMultipleFilenames()
2291 fCtx->nbFilesProcessed, in FIO_compressMultipleFilenames()
2296 fCtx->nbFilesProcessed, in FIO_compressMultipleFilenames()
2297 (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, in FIO_compressMultipleFilenames()
2325 int useMMap = prefs->mmapDict == ZSTD_ps_enable; in FIO_createDResources()
2326 int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable; in FIO_createDResources()
2334 if (prefs->patchFromMode){ in FIO_createDResources()
2336 useMMap |= dictSize > prefs->memLimit; in FIO_createDResources()
2337 FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */); in FIO_createDResources()
2344 CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) ); in FIO_createDResources()
2345 CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag)); in FIO_createDResources()
2354 if (prefs->patchFromMode){ in FIO_createDResources()
2374 /* FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
2379 IOJob_t *writeJob = AIO_WritePool_acquireJob(ress->writeCtx); in FIO_passThrough()
2380 AIO_ReadPool_fillBuffer(ress->readCtx, blockSize); in FIO_passThrough()
2382 while(ress->readCtx->srcBufferLoaded) { in FIO_passThrough()
2384 writeSize = MIN(blockSize, ress->readCtx->srcBufferLoaded); in FIO_passThrough()
2385 assert(writeSize <= writeJob->bufferSize); in FIO_passThrough()
2386 memcpy(writeJob->buffer, ress->readCtx->srcBuffer, writeSize); in FIO_passThrough()
2387 writeJob->usedBufferSize = writeSize; in FIO_passThrough()
2389 AIO_ReadPool_consumeBytes(ress->readCtx, writeSize); in FIO_passThrough()
2390 AIO_ReadPool_fillBuffer(ress->readCtx, blockSize); in FIO_passThrough()
2392 assert(ress->readCtx->reachedEof); in FIO_passThrough()
2394 AIO_WritePool_sparseWriteEnd(ress->writeCtx); in FIO_passThrough()
2399 * detailed error message when requested window size is too large */
2413 err = ZSTD_getFrameHeader(&header, ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded); in FIO_zstdErrorHelp()
2416 … unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); in FIO_zstdErrorHelp()
2417 assert(prefs->memLimit > 0); in FIO_zstdErrorHelp()
2418 DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u \n", in FIO_zstdErrorHelp()
2419 srcFileName, windowSize, prefs->memLimit); in FIO_zstdErrorHelp()
2421 … unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0)); in FIO_zstdErrorHelp()
2423 DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB \n", in FIO_zstdErrorHelp()
2432 * @return : size of decoded zstd frame, or an error code
2434 #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
2439 U64 alreadyDecoded) /* for multi-frames streams */ in FIO_decompressZstdFrame()
2442 IOJob_t *writeJob = AIO_WritePool_acquireJob(ress->writeCtx); in FIO_decompressZstdFrame()
2444 /* display last 20 characters only when not --verbose */ in FIO_decompressZstdFrame()
2447 srcFileName += srcFileLength-20; in FIO_decompressZstdFrame()
2450 ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only); in FIO_decompressZstdFrame()
2453 AIO_ReadPool_fillBuffer(ress->readCtx, ZSTD_FRAMEHEADERSIZE_MAX); in FIO_decompressZstdFrame()
2457 …ZSTD_inBuffer inBuff = setInBuffer( ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 ); in FIO_decompressZstdFrame()
2458 ZSTD_outBuffer outBuff= setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 ); in FIO_decompressZstdFrame()
2459 size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); in FIO_decompressZstdFrame()
2470 writeJob->usedBufferSize = outBuff.pos; in FIO_decompressZstdFrame()
2473 if (fCtx->nbFilesTotal > 1) { in FIO_decompressZstdFrame()
2476 const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; in FIO_decompressZstdFrame()
2479 …fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffi… in FIO_decompressZstdFrame()
2482 … fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix); in FIO_decompressZstdFrame()
2485 DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s... ", in FIO_decompressZstdFrame()
2489 AIO_ReadPool_consumeBytes(ress->readCtx, inBuff.pos); in FIO_decompressZstdFrame()
2495 if (ress->readCtx->srcBufferLoaded < toDecode) { in FIO_decompressZstdFrame()
2496 size_t const readSize = AIO_ReadPool_fillBuffer(ress->readCtx, toDecode); in FIO_decompressZstdFrame()
2506 AIO_WritePool_sparseWriteEnd(ress->writeCtx); in FIO_decompressZstdFrame()
2531 writeJob = AIO_WritePool_acquireJob(ress->writeCtx); in FIO_decompressGzFrame()
2532 strm.next_out = (Bytef*)writeJob->buffer; in FIO_decompressGzFrame()
2533 strm.avail_out = (uInt)writeJob->bufferSize; in FIO_decompressGzFrame()
2534 strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded; in FIO_decompressGzFrame()
2535 strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer; in FIO_decompressGzFrame()
2540 AIO_ReadPool_consumeAndRefill(ress->readCtx); in FIO_decompressGzFrame()
2541 if (ress->readCtx->srcBufferLoaded == 0) flush = Z_FINISH; in FIO_decompressGzFrame()
2542 strm.next_in = (z_const unsigned char*)ress->readCtx->srcBuffer; in FIO_decompressGzFrame()
2543 strm.avail_in = (uInt)ress->readCtx->srcBufferLoaded; in FIO_decompressGzFrame()
2554 { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; in FIO_decompressGzFrame()
2556 writeJob->usedBufferSize = decompBytes; in FIO_decompressGzFrame()
2559 strm.next_out = (Bytef*)writeJob->buffer; in FIO_decompressGzFrame()
2560 strm.avail_out = (uInt)writeJob->bufferSize; in FIO_decompressGzFrame()
2566 AIO_ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded - strm.avail_in); in FIO_decompressGzFrame()
2574 AIO_WritePool_sparseWriteEnd(ress->writeCtx); in FIO_decompressGzFrame()
2606 writeJob = AIO_WritePool_acquireJob(ress->writeCtx); in FIO_decompressLzmaFrame()
2607 strm.next_out = (BYTE*)writeJob->buffer; in FIO_decompressLzmaFrame()
2608 strm.avail_out = writeJob->bufferSize; in FIO_decompressLzmaFrame()
2609 strm.next_in = (BYTE const*)ress->readCtx->srcBuffer; in FIO_decompressLzmaFrame()
2610 strm.avail_in = ress->readCtx->srcBufferLoaded; in FIO_decompressLzmaFrame()
2615 AIO_ReadPool_consumeAndRefill(ress->readCtx); in FIO_decompressLzmaFrame()
2616 if (ress->readCtx->srcBufferLoaded == 0) action = LZMA_FINISH; in FIO_decompressLzmaFrame()
2617 strm.next_in = (BYTE const*)ress->readCtx->srcBuffer; in FIO_decompressLzmaFrame()
2618 strm.avail_in = ress->readCtx->srcBufferLoaded; in FIO_decompressLzmaFrame()
2631 { size_t const decompBytes = writeJob->bufferSize - strm.avail_out; in FIO_decompressLzmaFrame()
2633 writeJob->usedBufferSize = decompBytes; in FIO_decompressLzmaFrame()
2636 strm.next_out = (BYTE*)writeJob->buffer; in FIO_decompressLzmaFrame()
2637 strm.avail_out = writeJob->bufferSize; in FIO_decompressLzmaFrame()
2642 AIO_ReadPool_consumeBytes(ress->readCtx, ress->readCtx->srcBufferLoaded - strm.avail_in); in FIO_decompressLzmaFrame()
2645 AIO_WritePool_sparseWriteEnd(ress->writeCtx); in FIO_decompressLzmaFrame()
2666 writeJob = AIO_WritePool_acquireJob(ress->writeCtx); in FIO_decompressLz4Frame()
2671 size_t decodedBytes = writeJob->bufferSize; in FIO_decompressLz4Frame()
2675 AIO_ReadPool_fillBuffer(ress->readCtx, nextToLoad); in FIO_decompressLz4Frame()
2676 if(!ress->readCtx->srcBufferLoaded) break; /* reached end of file */ in FIO_decompressLz4Frame()
2678 …while ((pos < ress->readCtx->srcBufferLoaded) || fullBufferDecoded) { /* still to read, or still … in FIO_decompressLz4Frame()
2680 size_t remaining = ress->readCtx->srcBufferLoaded - pos; in FIO_decompressLz4Frame()
2681 decodedBytes = writeJob->bufferSize; in FIO_decompressLz4Frame()
2682 …nextToLoad = LZ4F_decompress(dCtx, writeJob->buffer, &decodedBytes, (char*)(ress->readCtx->srcBuff… in FIO_decompressLz4Frame()
2690 assert(pos <= ress->readCtx->srcBufferLoaded); in FIO_decompressLz4Frame()
2691 fullBufferDecoded = decodedBytes == writeJob->bufferSize; in FIO_decompressLz4Frame()
2696 writeJob->usedBufferSize = decodedBytes; in FIO_decompressLz4Frame()
2705 AIO_ReadPool_consumeBytes(ress->readCtx, pos); in FIO_decompressLz4Frame()
2714 AIO_WritePool_sparseWriteEnd(ress->writeCtx); in FIO_decompressLz4Frame()
2734 int passThrough = prefs->passThrough; in FIO_decompressFrames()
2736 if (passThrough == -1) { in FIO_decompressFrames()
2737 /* If pass-through mode is not explicitly enabled or disabled, in FIO_decompressFrames()
2741 passThrough = prefs->overwrite && !strcmp(dstFileName, stdoutmark); in FIO_decompressFrames()
2747 /* check magic number -> version */ in FIO_decompressFrames()
2751 buf = (const BYTE*)ress.readCtx->srcBuffer; in FIO_decompressFrames()
2752 if (ress.readCtx->srcBufferLoaded==0) { in FIO_decompressFrames()
2760 if (ress.readCtx->srcBufferLoaded < toRead) { /* not enough input to check magic number */ in FIO_decompressFrames()
2767 if (ZSTD_isFrame(buf, ress.readCtx->srcBufferLoaded)) { in FIO_decompressFrames()
2777 …td: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFile… in FIO_decompressFrames()
2787 … %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFile… in FIO_decompressFrames()
2796 …zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFile… in FIO_decompressFrames()
2807 fCtx->totalBytesOutput += (size_t)filesize; in FIO_decompressFrames()
2810 DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName, filesize); in FIO_decompressFrames()
2816 open `dstFileName`, or pass-through if writeCtx's file is already != 0,
2833 if ((AIO_WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) { in FIO_decompressDstFile()
2852 * and the user presses Ctrl-C when asked if they wish to overwrite. in FIO_decompressDstFile()
2899 DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); in FIO_decompressSrcFile()
2928 if ( prefs->removeSrcFile /* --rm */ in FIO_decompressSrcFile()
3002 … dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ in FIO_determineDstName()
3021 "Specify it with -o dstFileName. Ignoring.\n", in FIO_determineDstName()
3040 "Specify it with -o dstFileName. Ignoring.\n", in FIO_determineDstName()
3069 dstFileNameEndPos = sfnSize - srcSuffixLen; in FIO_determineDstName()
3102 if (!prefs->testMode) { in FIO_decompressMultipleFilenames()
3107 for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { in FIO_decompressMultipleFilenames()
3108 … status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); in FIO_decompressMultipleFilenames()
3109 if (!status) fCtx->nbFilesProcessed++; in FIO_decompressMultipleFilenames()
3112 if ((!prefs->testMode) && (AIO_WritePool_closeFile(ress.writeCtx))) in FIO_decompressMultipleFilenames()
3117 …UTIL_mirrorSourceFilesDirectories(srcNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirN… in FIO_decompressMultipleFilenames()
3119 … for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { /* create dstFileName */ in FIO_decompressMultipleFilenames()
3120 const char* const srcFileName = srcNamesTable[fCtx->currFileIdx]; in FIO_decompressMultipleFilenames()
3128 …DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot decompress '%s' into '%s'\n", srcFileName, outMi… in FIO_decompressMultipleFilenames()
3135 if (!status) fCtx->nbFilesProcessed++; in FIO_decompressMultipleFilenames()
3139 FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal); in FIO_decompressMultipleFilenames()
3145 fCtx->nbFilesProcessed, (unsigned long long)fCtx->totalBytesOutput); in FIO_decompressMultipleFilenames()
3153 * .zst file info (--list command)
3195 && (info->compressedSize > 0) in FIO_analyzeFrames()
3196 && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { in FIO_analyzeFrames()
3198 unsigned long long file_size = (unsigned long long) info->compressedSize; in FIO_analyzeFrames()
3200 "Error: seeked to position %llu, which is beyond file size of %llu\n", in FIO_analyzeFrames()
3215 info->decompUnavailable = 1; in FIO_analyzeFrames()
3217 info->decompressedSize += frameContentSize; in FIO_analyzeFrames()
3221 if (info->dictID != 0 && info->dictID != header.dictID) { in FIO_analyzeFrames()
3223 info->dictID = 0; in FIO_analyzeFrames()
3225 info->dictID = header.dictID; in FIO_analyzeFrames()
3227 info->windowSize = header.windowSize; in FIO_analyzeFrames()
3230 …ROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size"); in FIO_analyzeFrames()
3231 ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0, in FIO_analyzeFrames()
3258 info->usesCheck = 1; in FIO_analyzeFrames()
3259 ERROR_IF(fread(info->checksum, 1, 4, srcFile) != 4, in FIO_analyzeFrames()
3262 info->numActualFrames++; in FIO_analyzeFrames()
3267 long const seek = (long)(8 + frameSize - numBytesRead); in FIO_analyzeFrames()
3270 info->numSkippableFrames++; in FIO_analyzeFrames()
3290 info->compressedSize = UTIL_getFileSizeStat(&srcFileStat); in getFileInfo_fileConfirmed()
3294 info->nbFiles = 1; in getFileInfo_fileConfirmed()
3315 UTIL_HumanReadableSize_t const window_hrs = UTIL_makeHumanReadableSize(info->windowSize); in displayInfo()
3316 … UTIL_HumanReadableSize_t const compressed_hrs = UTIL_makeHumanReadableSize(info->compressedSize); in displayInfo()
3317 …UTIL_HumanReadableSize_t const decompressed_hrs = UTIL_makeHumanReadableSize(info->decompressedSiz… in displayInfo()
3318 …double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/(double)in… in displayInfo()
3319 const char* const checkString = (info->usesCheck ? "XXH64" : "None"); in displayInfo()
3321 if (!info->decompUnavailable) { in displayInfo()
3323 info->numSkippableFrames + info->numActualFrames, in displayInfo()
3324 info->numSkippableFrames, in displayInfo()
3330 info->numSkippableFrames + info->numActualFrames, in displayInfo()
3331 info->numSkippableFrames, in displayInfo()
3337 DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); in displayInfo()
3338 if (info->numSkippableFrames) in displayInfo()
3339 DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); in displayInfo()
3340 DISPLAYOUT("DictID: %u\n", info->dictID); in displayInfo()
3341 DISPLAYOUT("Window Size: %.*f%s (%llu B)\n", in displayInfo()
3343 (unsigned long long)info->windowSize); in displayInfo()
3344 DISPLAYOUT("Compressed Size: %.*f%s (%llu B)\n", in displayInfo()
3346 (unsigned long long)info->compressedSize); in displayInfo()
3347 if (!info->decompUnavailable) { in displayInfo()
3348 DISPLAYOUT("Decompressed Size: %.*f%s (%llu B)\n", in displayInfo()
3350 (unsigned long long)info->decompressedSize); in displayInfo()
3354 if (info->usesCheck && info->numActualFrames == 1) { in displayInfo()
3356 info->checksum[3], info->checksum[2], in displayInfo()
3357 info->checksum[1], info->checksum[0] in displayInfo()
3422 1, "zstd: --list does not support reading from standard input"); in FIO_listMultipleFiles()
3427 DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n"); in FIO_listMultipleFiles()
3440 /* --list each file, and check for any error */ in FIO_listMultipleFiles()
3450 DISPLAYOUT("----------------------------------------------------------------- \n"); in FIO_listMultipleFiles()