Lines Matching +full:- +full:- +full:with +full:- +full:zstd
3 set -e # exit immediately on error
4 # set -x # print commands before execution (debug)
19 zstd() { function
20 if [ -z "$EXE_PREFIX" ]; then
28 if [ -z "$EXE_PREFIX" ]; then
36 if [ -n "$3" ]; then
43 if [ -n "$4" ]; then
49 rm -f tmp1 tmp2
50 println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel"
52 datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2
53 $DIFF -q tmp1 tmp2
57 if [ -n "$3" ]; then
64 if [ -n "$4" ]; then
70 rm -f tmp.zst tmp.md5.1 tmp.md5.2
71 … println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d"
74 zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2
75 $DIFF -q tmp.md5.1 tmp.md5.2
79 dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1"
86 if [ -z "${size}" ]; then
95 UNAME=${UNAME:-$(uname)}
96 GREP=${GREP:-grep}
99 SunOS) DIFF=${DIFF:-gdiff} ;;
100 *) DIFF=${DIFF:-diff} ;;
104 if [ -t 0 ] && [ -t 1 ]
108 isTerminal=${isTerminal:-$detectedTerminal}
122 Darwin) MD5SUM="md5 -r" ;;
123 NetBSD) MD5SUM="md5 -n" ;;
128 MTIME="stat -c %Y"
130 Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
140 GET_PERMS="stat -c %a"
142 Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;;
159 if [ -z "${ZSTD_BIN}" ]; then
161 ZSTD_BIN="$PRGDIR/zstd"
165 if [ -z "${DATAGEN_BIN}" ]; then
174 [ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the z…
175 …-n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointi…
178 if echo hello | zstd -v -T2 2>&1 > $INTOVOID | $GREP -q 'multi-threading is disabled'
186 zstd -vvV
191 zstd -h
192 zstd -H
193 zstd -V
195 zstd -f tmp # trivial compression case, creates tmp.zst
196 zstd -f -z tmp
197 zstd -f -k tmp
198 zstd -f -C tmp
200 zstd -df tmp.zst # trivial decompression case (overwrites tmp)
201 println "test : too large compression level => auto-fix"
202 zstd -99 -f tmp # too large compression level, automatic sized down
203 zstd -5000000000 -f tmp && die "too large numeric value : must fail"
204 println "test : --fast aka negative compression levels"
205 zstd --fast -f tmp # == -1
206 zstd --fast=3 -f tmp # == -3
207 zstd --fast=200000 -f tmp # too low compression level, automatic fixed
208 zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail"
209 zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0"
211 zstd --fast=9999999999 -f tmp && die "should have refused numeric value"
212 println "test : set compression level with environment variable ZSTD_CLEVEL"
214 ZSTD_CLEVEL=12 zstd -f tmp # positive compression level
215 ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level
216 ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign
217 ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting
218 ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting
219 ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting
220 ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting
221 ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting
222 ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting
223 println "test : override ZSTD_CLEVEL with command line option"
224 ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option
231 zstd tmp -c > tmpCompressed
232 zstd tmp --stdout > tmpCompressed # long command format
234 println "test : compress to named file (-o)"
235 rm -f tmpCompressed
236 zstd tmp -o tmpCompressed
237 test -f tmpCompressed # file must be created
240 zstd tmp -fo tmpCompressed
242 println "test : -c + -o : last one wins"
243 rm -f tmpOut
244 zstd tmp -c > tmpCompressed -o tmpOut
245 test -f tmpOut # file must be created
246 rm -f tmpCompressed
247 zstd tmp -o tmpOut -c > tmpCompressed
248 test -f tmpCompressed # file must be created
252 zstd tmp2 -fo && die "-o must be followed by filename "
254 println bob | zstd | zstd -d
257 println bob | zstd && die "should have refused : compressed data to terminal"
258 println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting …
259 zstd -d > $INTOVOID && die "should have refused : compressed data from terminal"
261 println "test : null-length file roundtrip"
262 println -n '' | zstd - --stdout | zstd -d --stdout
263 println "test : ensure small file doesn't add 3-bytes null block"
264 datagen -g1 > tmp1
265 zstd tmp1 -c | wc -c | $GREP "14"
266 zstd < tmp1 | wc -c | $GREP "14"
267 println "test : decompress file with wrong suffix (must fail)"
268 zstd -d tmpCompressed && die "wrong suffix error not detected!"
269 zstd -df tmp && die "should have refused : wrong extension"
271 zstd -d tmpCompressed -c > tmpResult # decompression using stdout
272 zstd --decompress tmpCompressed -c > tmpResult
273 zstd --decompress tmpCompressed --stdout > tmpResult
275 zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout
276 zstd -dc - < tmp.zst > $INTOVOID
277 zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used
278 zstd -d - < tmp.zst > $INTOVOID
280 datagen -g500K > tmplimit
281 zstd -f tmplimit
282 zstd -d -f tmplimit.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed"
283 zstd -d -f tmplimit.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than a…
284 zstd -d -f tmplimit.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than all…
285 zstd -d -f tmplimit.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more me…
286 rm -f tmplimit tmplimit.zst
288 zstd -q tmp && die "overwrite check failed!"
290 zstd -q -f tmp
291 zstd -q --force tmp
293 rm -f tmpro tmpro.zst
297 zstd -q tmpro && die "should have refused to overwrite read-only file"
298 zstd -q -f tmpro
299 println "test: --no-progress flag"
300 zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress
301 zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress
302 println "test: --progress flag"
303 zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes'
304 zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes'
305 zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes'
306 rm -f tmpro tmpro.zst
308 zstd tmp -fo tmp && die "zstd compression overwrote the input file"
309 zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file"
311 zstd nothere && die "zstd hasn't detected that input file does not exist"
312 println "test: --[no-]compress-literals"
313 zstd tmp -c --no-compress-literals -1 | zstd -t
314 zstd tmp -c --no-compress-literals --fast=1 | zstd -t
315 zstd tmp -c --no-compress-literals -19 | zstd -t
316 zstd tmp -c --compress-literals -1 | zstd -t
317 zstd tmp -c --compress-literals --fast=1 | zstd -t
318 zstd tmp -c --compress-literals -19 | zstd -t
319 zstd -b --fast=1 -i0e1 tmp --compress-literals
320 zstd -b --fast=1 -i0e1 tmp --no-compress-literals
321 println "test: --no-check for decompression"
322 zstd -f tmp -o tmp_corrupt.zst --check
323 zstd -f tmp -o tmp.zst --no-check
324 printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) c…
325 zstd -d -f tmp_corrupt.zst --no-check
326 zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides
327 zstd -d -f tmp.zst --no-check
330 if [ -n "$(which readelf)" ]; then
332 readelf -lW "$ZSTD_BIN" | $GREP 'GNU_STACK .* RW ' || die "zstd binary has executable stack!"
339 println "test : single-thread "
340 zstd --fast --single-thread tmp -o tmpMT0
342 zstd --fast -T1 tmp -o tmpMT1
344 zstd --fast -T2 tmp -o tmpMT2
345 println "test : 16-thread "
346 zstd --fast -T16 tmp -o tmpMT3
347 println "test : 127-thread "
348 zstd --fast -T127 tmp -o tmpMT4
349 println "test : 128-thread "
350 zstd --fast -T128 tmp -o tmpMT5
352 zstd --fast -4294967295 tmp -o tmpMT6
353 println "test : numeric value overflows 32-bit unsigned int "
354 zstd --fast -4294967296 tmp -o tmptest9 && die "max allowed numeric value is 4294967295"
358 zstd -f tmp # trivial compression case, creates tmp.zst
360 zstd -d -f -T1 tmp.zst
361 println "note : decompression does not support -T mode, but execution support"
362 rm -rf tmpMT*
364 println "\n===> --fast_argument test "
367 zstd -f tmp # trivial compression case, creates tmp.zst
368 println "test: --fast=1"
369 zstd --fast=1 -f tmp
370 println "test: --fast=99"
371 zstd --fast=99 -f tmp
372 println "test: Invalid value -- negative number"
373 zstd --fast=-1 -f tmp && die "error: Invalid value -- negative number"
374 println "test: Invalid value -- zero"
375 zstd --fast=0 -f tmp && die "error: Invalid value -- 0 number"
376 println "test: max allowed numeric argument of --fast is 4294967295"
377 zstd --fast=4294967295 -f tmp
378 println "test: numeric value overflows 32-bit unsigned int "
379 zstd --fast=4294967296 -f tmp && die "max allowed argument of --fast is 4294967295"
381 println "\n===> --exclude-compressed flag"
382 rm -rf precompressedFilterTestDir
383 mkdir -p precompressedFilterTestDir
386 zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
389 zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
390 test ! -f precompressedFilterTestDir/input.5.zst.zst
391 test ! -f precompressedFilterTestDir/input.6.zst.zst
394 if [ $file2timestamp -ge $file1timestamp ]; then
401 zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
402 # zstd should compress input.zstbar
403 test -f precompressedFilterTestDir/input.zstbar.zst
404 # Check without the --exclude-compressed flag
405 zstd --long --rm -r precompressedFilterTestDir
406 # Files should get compressed again without the --exclude-compressed flag.
407 test -f precompressedFilterTestDir/input.5.zst.zst
408 test -f precompressedFilterTestDir/input.6.zst.zst
414 zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
415 test ! -f precompressedFilterTestDir/input.flac.zst
416 test ! -f precompressedFilterTestDir/input.mov.zst
417 test ! -f precompressedFilterTestDir/input.mp3.zst
418 zstd --long --rm -r precompressedFilterTestDir
419 test -f precompressedFilterTestDir/input.flac.zst
420 test -f precompressedFilterTestDir/input.mov.zst
421 test -f precompressedFilterTestDir/input.mp3.zst
422 rm -rf precompressedFilterTestDir
430 zstd tmpPrompt -f
431 zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite"
432 zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f
433 zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated
436 echo 'yes' | zstd tmpPrompt -v -o tmpPrompt.zst # accept piped "y" input to force overwrite when u…
437 echo 'yes' | zstd < tmpPrompt -v -o tmpPrompt.zst && die "should have aborted immediately and faile…
438 zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and faile…
444 # combination of -r with empty list of input file
445 zstd -c -r < tmp > tmp.zst
447 # combination of -r with empty folder
448 mkdir -p tmpEmptyDir
449 zstd -r tmpEmptyDir
450 rm -rf tmpEmptyDir
454 zstd -f --rm tmp
455 test ! -f tmp # tmp should no longer be present
456 zstd -f -d --rm tmp.zst
457 test ! -f tmp.zst # tmp.zst should no longer be present
458 println "test: --rm is disabled when output is stdout"
459 test -f tmp
460 zstd --rm tmp -c > $INTOVOID
461 test -f tmp # tmp shall still be there
462 zstd --rm tmp --stdout > $INTOVOID
463 test -f tmp # tmp shall still be there
464 zstd -f --rm tmp -c > $INTOVOID
465 test -f tmp # tmp shall still be there
466 zstd -f tmp -c > $INTOVOID --rm
467 test -f tmp # tmp shall still be there
468 println "test: --rm is disabled when multiple inputs are concatenated into a single output"
470 zstd --rm tmp tmp2 -c > $INTOVOID
471 test -f tmp
472 test -f tmp2
473 rm -f tmp3.zst
474 echo 'y' | zstd -v tmp tmp2 -o tmp3.zst --rm # prompt for confirmation
475 test -f tmp
476 test -f tmp2
477 zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt
478 test -f tmp
479 test -f tmp2
480 zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate"
481 println "test: --rm is active with -o when single input"
482 rm -f tmp2.zst
483 zstd --rm tmp2 -o tmp2.zst
484 test -f tmp2.zst
485 test ! -f tmp2
486 println "test: -c followed by -o => -o wins, so --rm remains active" # (#3719)
489 zstd --rm tmp2 -c > $INTOVOID -o tmp2.zst
490 test ! -f tmp2
491 println "test: -o followed by -c => -c wins, so --rm is disabled" # (#3719)
494 zstd -v --rm tmp2 -o tmp2.zst -c > tmp3.zst
495 test -f tmp2
496 test -f tmp3.zst
497 println "test : should quietly not remove non-regular file"
499 zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
500 $GREP "Refusing to remove non-regular file" tmplog && die
501 rm -f tmplog
502 zstd tmp -f -o "$INTOVOID" 2>&1 | $GREP "Refusing to remove non-regular file" && die
503 println "test : --rm on stdin"
504 println a | zstd --rm > $INTOVOID # --rm should remain silent
505 rm -f tmp
506 zstd -f tmp && die "tmp not present : should have failed"
507 test ! -f tmp.zst # tmp.zst should not be created
508 println "test : -d -f do not delete destination when source is not present"
510 zstd -d -f tmp.zst && die "attempt to decompress a non existing file"
511 test -f tmp # destination file should still be present
512 println "test : -f do not delete destination when source is not present"
513 rm -f tmp # erase source file
515 zstd -f tmp && die "attempt to compress a non existing file"
516 test -f tmp.zst # destination file should still be present
517 rm -rf tmp* # may also erase tmp* directory from previous failed run
521 # the following test verifies that the decoder is compatible with RLE as first block
522 # older versions of zstd cli are not able to decode such corner case.
523 # As a consequence, the zstd cli do not generate them, to maintain compatibility with older version…
525 zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst"
526 $DIFF -s tmp1 tmp
529 zstd -d -o tmp2 "$TESTDIR/golden-decompression/empty-block.zst"
530 $DIFF -s tmp2 tmp_empty
532 zstd -t "$TESTDIR/golden-decompression/zeroSeq_2B.zst"
534 zstd -t "$TESTDIR/golden-decompression-errors/zeroSeq_extraneous.zst" && die "invalid Sequences sec…
536 rm -f tmp*
541 zstd tmp1 tmp2 -o "$INTOVOID" -f
542 zstd tmp1 tmp2 -c | zstd -t
543 echo 'y' | zstd -v tmp1 tmp2 -o tmp.zst
544 test ! -f tmp1.zst
545 test ! -f tmp2.zst
546 zstd tmp1 tmp2
547 zstd -t tmp1.zst tmp2.zst
548 zstd -dc tmp1.zst tmp2.zst
549 zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f
550 echo 'y' | zstd -v -d tmp1.zst tmp2.zst -o tmp
552 zstd tmp1 tmp2 -f -o tmpexists
553 zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite"
557 echo 'y' | zstd -v tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst
558 test -f tmp_rm1
559 test -f tmp_rm2
561 echo 'Y' | zstd -v -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm
562 test -f tmp_rm3.zst
563 test -f tmp_rm4.zst
565 zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID
567 if [ "$?" -eq 139 ]; then
570 test -f tmpexists1
571 test -f tmpexists
573 datagen -s1 > tmp1 2> $INTOVOID
574 datagen -s2 -g100K > tmp2 2> $INTOVOID
575 datagen -s3 -g1M > tmp3 2> $INTOVOID
577 zstd -f tmp*
578 test -f tmp1.zst
579 test -f tmp2.zst
580 test -f tmp3.zst
581 rm -f tmp1 tmp2 tmp3
583 zstd -df ./*.zst
584 test -f tmp1
585 test -f tmp2
586 test -f tmp3
588 zstd -c tmp1 tmp2 tmp3 > tmpall
589 test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst)
592 zstd -dc tmpall* > tmpdec
593 test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
595 zstd -f tmp1 notHere tmp2 && die "missing file not detected!"
596 rm -f tmp*
600 println "\n===> zstd fifo named pipe test "
603 # note : fifo test doesn't work in combination with `dd` or `cat`
605 zstd tmp_named_pipe -o tmp_compressed
606 zstd -d -o tmp_decompressed tmp_compressed
607 $DIFF -s tmp_original tmp_decompressed
608 rm -rf tmp*
611 println "\n===> zstd created file permissions tests"
613 rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove
623 println "test : copy 666 permissions in file -> file compression "
624 zstd -f tmp1 -o tmp1.zst
626 println "test : copy 666 permissions in file -> file decompression "
627 zstd -f -d tmp1.zst -o tmp1.out
630 rm -f tmp1.zst tmp1.out
632 println "test : copy 400 permissions in file -> file compression (write to a read-only file) "
635 zstd -f tmp1 -o tmp1.zst
637 println "test : copy 400 permissions in file -> file decompression (write to a read-only file) "
638 zstd -f -d tmp1.zst -o tmp1
641 rm -f tmp1.zst tmp1.out
644 zstd -f -o tmp1.zst < tmp1
647 zstd -f -d -o tmp1.out < tmp1.zst
650 rm -f tmp1.zst tmp1.out
653 zstd -f tmp1 tmp2 -o tmp1.zst
657 zstd -f -d tmp1.zst tmp2.zst -o tmp1.out
660 rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out
662 println "test : check permissions on pre-existing output file in compression "
666 zstd -f tmp1 -o tmp1.zst
668 println "test : check permissions on pre-existing output file in decompression "
672 zstd -f -d tmp1.zst -o tmp1.out
678 rm -f tmp1.zst tmp1.out
681 zstd -f -o tmp1.zst < tmp1
685 zstd -f -d -o tmp1.out < tmp1.zst
688 rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out
692 if [ -n "$DEVNULLRIGHTS" ] ; then
697 sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions
698 sudoZstd tmp -c > $INTOVOID
699 zstd tmp -f -o tmp.zst
700 sudoZstd -d tmp.zst -c > $INTOVOID
701 sudoZstd -d tmp.zst -o $INTOVOID
702 ls -las $INTOVOID | $GREP "rw-rw-rw-"
705 if [ -n "$READFROMBLOCKDEVICE" ] ; then
706 # This creates a temporary block device, which is only possible on unix-y
709 println "\n===> checking that zstd can read from a block device"
710 datagen -g65536 > tmp.img
711 sudo losetup -fP tmp.img
712 LOOP_DEV=$(losetup -a | $GREP 'tmp\.img' | cut -f1 -d:)
713 [ -z "$LOOP_DEV" ] && die "failed to get loopback device"
714 sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f"
715 sudoZstd -f $LOOP_DEV -c > tmp.img.zst
716 zstd -d tmp.img.zst -o tmp.img.copy
717 sudo losetup -d $LOOP_DEV
718 $DIFF -s tmp.img tmp.img.copy || die "round trip failed"
719 rm -f tmp.img tmp.img.zst tmp.img.copy
722 println "\n===> zstd created file timestamp tests"
724 touch -m -t 200001010000.00 tmp
725 println "test : copy mtime in file -> file compression "
726 zstd -f tmp -o tmp.zst
728 println "test : copy mtime in file -> file decompression "
729 zstd -f -d tmp.zst -o tmp.out
731 rm -f tmp
733 println "\n===> compress multiple files into an output directory, --output-dir-flat"
742 zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir
743 test -f tmpOutDir/tmp1.zst
744 test -f tmpOutDir/tmp2.zst
745 println "test : decompress multiple files into an output directory, --output-dir-flat"
747 zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
748 test -f tmpOutDirDecomp/tmp2
749 test -f tmpOutDirDecomp/tmp1
750 rm -f tmpOutDirDecomp/*
751 zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp
752 test -f tmpOutDirDecomp/tmp2
753 test -f tmpOutDirDecomp/tmp1
754 rm -rf tmp*
757 …==> compress multiple files into an output directory and mirror input folder, --output-dir-mirror"
758 println "test --output-dir-mirror" > tmp1
759 mkdir -p tmpInputTestDir/we/.../..must/go/deeper..
761 zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir
762 test -f tmpOutDir/tmp1.zst
763 test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst
766 …zstd -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "…
767 …zstd -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input can…
768 …zstd -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "inp…
769 test ! -d non-exist
771 println "test: compress input dir should succeed with benign uses of '..'"
772 zstd -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout
773 test -d tmpout
775 println "test : decompress multiple files into an output directory, --output-dir-mirror"
776 zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp
777 test -f tmpOutDirDecomp/tmpOutDir/tmp1
778 test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2
781 …zstd -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "i…
782 test ! -d non-exist
784 rm -rf tmp*
788 println "test : compress multiple files reading them from a file, --filelist=FILE"
793 zstd -f --filelist=tmp_fileList
794 test -f tmp2.zst
795 test -f tmp1.zst
797 println "test : alternate syntax: --filelist FILE"
798 zstd -f --filelist tmp_fileList
799 test -f tmp2.zst
800 test -f tmp1.zst
802 println "test : reading file list from a symlink, --filelist=FILE"
803 rm -f *.zst
804 ln -s tmp_fileList tmp_symLink
805 zstd -f --filelist=tmp_symLink
806 test -f tmp2.zst
807 test -f tmp1.zst
809 println "test : compress multiple files reading them from multiple files, --filelist=FILE"
810 rm -f *.zst
815 zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2
816 test -f tmp1.zst
817 test -f tmp2.zst
818 test -f tmp3.zst
819 test -f tmp4.zst
821 println "test : decompress multiple files reading them from a file, --filelist=FILE"
822 rm -f tmp1 tmp2
825 zstd -d -f --filelist=tmpZst
826 test -f tmp1
827 test -f tmp2
829 println "test : decompress multiple files reading them from multiple files, --filelist=FILE"
830 rm -f tmp1 tmp2 tmp3 tmp4
833 zstd -d -f --filelist=tmpZst --filelist=tmpZst2
834 test -f tmp1
835 test -f tmp2
836 test -f tmp3
837 test -f tmp4
839 println "test : survive the list of files with too long filenames (--filelist=FILE)"
840 datagen -g5M > tmp_badList
841 zstd -qq -f --filelist=tmp_badList && die "should have failed : file name length is too long" # pr…
843 println "test : survive a list of files which is text garbage (--filelist=FILE)"
845 zstd -qq -f --filelist=tmp_badList && die "should have failed : list is text garbage" # printing v…
847 println "test : survive a list of files which is binary garbage (--filelist=FILE)"
848 datagen -P0 -g1M > tmp_badList
849 zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's av…
851 println "test : try to overflow internal list of files (--filelist=FILE)"
854 zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal…
855 rm -rf tmp*
857 println "\n===> --[no-]content-size tests"
860 zstd -f tmp_contentsize
861 zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:"
862 zstd -f --no-content-size tmp_contentsize
863 zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die
864 zstd -f --content-size tmp_contentsize
865 zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:"
866 zstd -f --content-size --no-content-size tmp_contentsize
867 zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die
868 rm -rf tmp*
870 println "test : show-default-cparams regular"
872 zstd --show-default-cparams -f tmp
873 zstd --show-default-cparams -d tmp.zst && die "error: can't use --show-default-cparams in decompres…
874 rm -rf tmp*
876 println "test : show-default-cparams recursive"
878 datagen -g15000 > tmp_files/tmp1
879 datagen -g129000 > tmp_files/tmp2
880 datagen -g257000 > tmp_files/tmp3
881 zstd --show-default-cparams -f -r tmp_files
882 rm -rf tmp*
886 zstd -vv tmp 2>&1 | \
887 $GREP -q -- "--zstd=wlog=[0-9]*,clog=[0-9]*,hlog=[0-9]*,slog=[0-9]*,mml=[0-9]*,tlen=[0-9]*,strat=[0…
888 rm -rf tmp*
891 println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not d…
892 println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not d…
893 println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not d…
894 println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound…
895 test ! -f tmp.zst # tmp.zst should not be created
896 roundTripTest -g512K
897 roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
898 roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
899 roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLeng…
900 roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=…
901 roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
902 roundTripTest -g64K "19 --zstd=strat=9" # btultra2
905 println "\n===> Pass-Through mode "
906 println "Hello world 1!" | zstd -df
907 println "Hello world 2!" | zstd -dcf
909 zstd -dcf tmp1
910 println "" | zstd -df > tmp1
912 $DIFF -q tmp1 tmp2
913 println "1" | zstd -df > tmp1
915 $DIFF -q tmp1 tmp2
916 println "12" | zstd -df > tmp1
918 $DIFF -q tmp1 tmp2
919 rm -rf tmp*
926 zstd -c hello.tmp > hello.zst
927 zstd -c world.tmp > world.zst
928 zstd -c hello.tmp world.tmp > helloworld.zst
929 zstd -dc helloworld.zst > result.tmp
932 zstd -dc helloworld.zst > result.tmp
936 zstd -c hello.tmp > hello.zst --no-check
937 zstd -c world.tmp > world.zst --no-check
938 cat hello.zst world.zst > helloworld.zstd
939 zstd -dc helloworld.zst > result.tmp
942 ln -sf "$ZSTD_BIN" zstdcat
945 ln -s helloworld.zst helloworld.link.zst
948 rm -f zstdcat
949 rm -f result.tmp
951 ln -sf "$ZSTD_BIN" zcat
956 rm -f zcat
957 rm -f ./*.tmp ./*.zstd
964 println "println foo | zstd > /dev/full"
965 println foo | zstd > /dev/full && die "write error not detected!"
966 println "println foo | zstd | zstd -d > /dev/full"
967 println foo | zstd | zstd -d > /dev/full && die "write error not detected!"
976 rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst
978 ln -s hello.tmp world.tmp
979 ln -s hello.tmp world2.tmp
980 zstd world.tmp hello.tmp || true
981 test -f hello.tmp.zst # regular file should have been compressed!
982 test ! -f world.tmp.zst # symbolic link should not have been compressed!
983 zstd world.tmp || true
984 test ! -f world.tmp.zst # symbolic link should not have been compressed!
985 zstd world.tmp world2.tmp || true
986 test ! -f world.tmp.zst # symbolic link should not have been compressed!
987 test ! -f world2.tmp.zst # symbolic link should not have been compressed!
988 zstd world.tmp hello.tmp -f
989 test -f world.tmp.zst # symbolic link should have been compressed with --force
990 rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst
997 datagen -g5M -P100 > tmpSparse
998 zstd tmpSparse -c | zstd -dv -o tmpSparseRegen
999 $DIFF -s tmpSparse tmpSparseRegen
1000 zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse
1001 $DIFF -s tmpSparse tmpOutSparse
1002 zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse
1003 $DIFF -s tmpSparse tmpOutNoSparse
1004 ls -ls tmpSparse* # look at file size and block size on disk
1005 datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not …
1006 datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd
1007 ls -ls tmpSparseOdd # look at file size and block size on disk
1008 println "\n Sparse Compatibility with Console :"
1009 println "Hello World 1 !" | zstd | zstd -d -c
1010 println "Hello World 2 !" | zstd | zstd -d | cat
1011 println "\n Sparse Compatibility with Append :"
1012 datagen -P100 -g1M > tmpSparse1M
1014 zstd -v -f tmpSparse1M -o tmpSparseCompressed
1015 zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
1016 zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
1017 ls -ls tmpSparse* # look at file size and block size on disk
1019 rm -f tmpSparse*
1022 println "\n===> stream-size mode"
1024 datagen -g11000 > tmp
1026 file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst)
1027 stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c)
1028 if [ "$stream_size" -gt "$file_size" ]; then
1032 cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000
1033 zstd -df tmp.zst -o tmp_decompress
1036 cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size"
1038 println "\n===> zstd zero weight dict test "
1039 rm -f tmp*
1040 cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input
1041 zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input
1042 zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp
1044 rm -rf tmp*
1046 println "\n===> zstd (valid) zero weight dict test "
1047 rm -f tmp*
1048 # 0 has a non-zero weight in the dictionary
1050 zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input
1051 zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp
1053 rm -rf tmp*
1055 println "\n===> size-hint mode"
1057 datagen -g11000 > tmp
1058 datagen -g11000 > tmp2
1061 file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst)
1062 stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c)
1063 if [ "$stream_size" -ge "$file_size" ]; then
1067 cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000
1068 zstd -df tmp.zst -o tmp_decompress
1070 println "test : hinted streaming compression with dictionary"
1071 cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict
1072 println "test : multiple file compression with hints and dictionary"
1073 zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2
1074 zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp
1075 zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2
1079 cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high
1080 cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low
1081 cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high
1082 cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low
1084 cat tmp | zstd -14 -f --size-hint=11K | zstd -t
1085 cat tmp | zstd -14 -f --size-hint=11KB | zstd -t
1086 cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t
1087 cat tmp | zstd -14 -f --size-hint=1M | zstd -t
1088 cat tmp | zstd -14 -f --size-hint=1MB | zstd -t
1089 cat tmp | zstd -14 -f --size-hint=1MiB | zstd -t
1093 println "- Test high/low compressibility corpus training"
1094 datagen -g12M -P90 > tmpCorpusHighCompress
1095 datagen -g12M -P5 > tmpCorpusLowCompress
1096 zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress
1097 zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress
1098 rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress
1099 println "- Test with raw dict (content only) "
1101 datagen -g1M | $MD5SUM > tmp1
1102 datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2
1103 $DIFF -q tmp1 tmp2
1104 println "- Create first dictionary "
1106 zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
1108 println "- Test dictionary compression with tmpDict as an input file and dictionary"
1109 zstd -f tmpDict -D tmpDict && die "compression error not detected!"
1110 println "- Dictionary compression roundtrip"
1111 zstd -f tmp -D tmpDict
1112 zstd -d tmp.zst -D tmpDict -fo result
1114 println "- Dictionary compression with hlog < clog"
1115 zstd -6f tmp -D tmpDict --zstd=clog=25,hlog=23
1116 println "- Dictionary compression with btlazy2 strategy"
1117 zstd -f tmp -D tmpDict --zstd=strategy=6
1118 zstd -d tmp.zst -D tmpDict -fo result
1120 if [ -e /proc/self/fd/0 ]; then
1121 println "- Test rejecting irregular dictionary file"
1122 cat tmpDict | zstd -f tmp -D /proc/self/fd/0 && die "Piped dictionary should fail!"
1123 cat tmpDict | zstd -d tmp.zst -D /proc/self/fd/0 -f && die "Piped dictionary should fail!"
1125 if [ -n "$hasMT" ]
1127 println "- Test dictionary compression with multithreading "
1128 datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict # fails with v1.3.2
1130 println "- Create second (different) dictionary "
1131 zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
1132 zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
1133 println "- Create dictionary with short dictID"
1134 zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
1136 println "- Create dictionary with wrong dictID parameter order (must fail)"
1137 zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID mus…
1138 println "- Create dictionary with size limit"
1139 zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v
1140 println "- Create dictionary with small size limit"
1141 zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v
1142 println "- Create dictionary with wrong parameter order (must fail)"
1143 zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdi…
1144 println "- Compress without dictID"
1145 zstd -f tmp -D tmpDict1 --no-dictID
1146 zstd -d tmp.zst -D tmpDict -fo result
1148 println "- Compress multiple files with dictionary"
1149 rm -rf dirTestDict
1155 zstd -f --rm dirTestDict/* -D tmpDictC
1156 zstd -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default
1158 …Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's …
1159 *) $MD5SUM -c tmph1 ;;
1161 rm -rf dirTestDict
1162 println "- dictionary builder on bogus input"
1164 zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source"
1165 datagen -P0 -g10M > tmp
1166 zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise"
1167 println "- Test -o before --train"
1168 rm -f tmpDict dictionary
1169 zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c
1170 test -f tmpDict
1171 zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c
1172 test -f dictionary
1173 if [ -n "$hasMT" ]
1175 println "- Create dictionary with multithreading enabled"
1176 zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
1178 rm -f tmp* dictionary
1180 println "- Test --memory for dictionary compression"
1181 datagen -g12M -P90 > tmpCorpusHighCompress
1182 zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary trai…
1183 zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimit…
1191 println "- Create first dictionary"
1192 zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
1194 zstd -f tmp -D tmpDict
1195 zstd -d tmp.zst -D tmpDict -fo result
1197 println "- Create second (different) dictionary"
1198 zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
1199 zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
1200 zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file"
1201 println "- Create dictionary with short dictID"
1202 zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
1204 println "- Create dictionaries with shrink-dict flag enabled"
1205 zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
1206 zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
1207 zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
1208 zstd --train-fastcover=shrink=5,steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict3
1209 println "- Create dictionary with size limit"
1210 zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
1211 println "- Create dictionary using all samples for both training and testing"
1212 zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1213 println "- Create dictionary using f=16"
1214 zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1215 zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary…
1216 println "- Create dictionary using accel=2"
1217 zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1218 println "- Create dictionary using accel=10"
1219 zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1220 println "- Create dictionary with multithreading"
1221 zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1222 println "- Test -o before --train-fastcover"
1223 rm -f tmpDict dictionary
1224 zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
1225 test -f tmpDict
1226 zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
1227 test -f dictionary
1228 rm -f tmp* dictionary
1235 println "- Create first dictionary"
1236 zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
1238 zstd -f tmp -D tmpDict
1239 zstd -d tmp.zst -D tmpDict -fo result
1241 zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)"
1242 println "- Create second (different) dictionary"
1243 zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
1244 zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
1245 println "- Create dictionary with short dictID"
1246 zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
1248 println "- Create dictionary with size limit"
1249 zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
1250 println "- Test -o before --train-legacy"
1251 rm -f tmpDict dictionary
1252 zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c
1253 test -f tmpDict
1254 zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c
1255 test -f dictionary
1256 rm -f tmp* dictionary
1263 zstd tmp1
1264 zstd -t tmp1.zst
1265 zstd --test tmp1.zst
1267 zstd -t ./*.zst
1269 zstd -t ./* && die "bad files not detected !"
1270 zstd -t tmp1 && die "bad file not detected !"
1272 zstd -t tmp2.zst && die "bad file not detected !"
1273 datagen -g0 > tmp3
1274 zstd -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad
1275 println "test --rm and --test combined "
1276 zstd -t --rm tmp1.zst
1277 test -f tmp1.zst # check file is still present
1279 zstd -t tmp1.zst tmp2.zst --rm
1280 test -f tmp1.zst # check file is still present
1281 test -f tmp2.zst # check file is still present
1282 split -b16384 tmp1.zst tmpSplit.
1283 zstd -t tmpSplit.* && die "bad file not detected !"
1284 datagen | zstd -c | zstd -t
1289 zstd -t -r "$TESTDIR/golden-decompression"
1290 zstd -c -r "$TESTDIR/golden-compression" | zstd -t
1291 zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http"…
1298 zstd -bi0 tmp1
1300 zstd -i0b0e3 tmp1
1302 zstd -bi0 --fast tmp1
1303 println "with recursive and quiet modes"
1304 zstd -rqi0b1e2 tmp1
1306 zstd -f tmp1
1307 zstd -b -d -i0 tmp1.zst
1308 println "benchmark can fail - decompression on invalid data"
1309 zstd -b -d -i0 tmp1 && die "invalid .zst data => benchmark should have failed"
1312 zstd --format=gzip -V || GZIPMODE=0
1313 if [ $GZIPMODE -eq 1 ]; then
1314 println "benchmark mode is only compatible with zstd"
1315 zstd --format=gzip -b tmp1 && die "-b should be incompatible with gzip format!"
1318 println "\n===> zstd compatibility tests "
1321 rm -f tmp.zst
1322 zstd --format=zstd -f tmp
1323 test -f tmp.zst
1329 zstd --format=gzip -V || GZIPMODE=0
1330 if [ $GZIPMODE -eq 1 ]; then
1333 gzip -V || GZIPEXE=0
1334 if [ $GZIPEXE -eq 1 ]; then
1336 zstd --format=gzip -f tmp
1337 gzip -t -v tmp.gz
1338 gzip -f tmp
1339 zstd -d -f -v tmp.gz
1340 rm -f tmp*
1351 if [ $GZIPMODE -eq 1 ]; then
1353 zstd -f --format=gzip tmp
1354 zstd -f tmp
1355 cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp
1356 truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !"
1357 rm -f tmp*
1362 if [ $GZIPMODE -eq 1 ]; then
1364 rm -f tmp.zst
1365 zstd --format=gzip --format=zstd -f tmp
1366 test -f tmp.zst
1372 zstd --format=xz -V || LZMAMODE=0
1373 if [ $LZMAMODE -eq 1 ]; then
1376 xz -Q -V && lzma -Q -V || XZEXE=0
1377 if [ $XZEXE -eq 1 ]; then
1378 println "Testing zstd xz and lzma support"
1380 zstd --format=lzma -f tmp
1381 zstd --format=xz -f tmp
1382 xz -Q -t -v tmp.xz
1383 xz -Q -t -v tmp.lzma
1384 xz -Q -f -k tmp
1385 lzma -Q -f -k --lzma1 tmp
1386 zstd -d -f -v tmp.xz
1387 zstd -d -f -v tmp.lzma
1388 rm -f tmp*
1390 ln -s "$ZSTD_BIN" ./xz
1391 ln -s "$ZSTD_BIN" ./unxz
1392 ln -s "$ZSTD_BIN" ./lzma
1393 ln -s "$ZSTD_BIN" ./unlzma
1397 xz -Q -d tmp.xz
1399 lzma -Q -d tmp.lzma
1401 xz -Q tmp
1402 ./xz -d tmp.xz
1403 lzma -Q tmp
1404 ./lzma -d tmp.lzma
1405 rm -f xz unxz lzma unlzma
1406 rm -f tmp*
1417 if [ $LZMAMODE -eq 1 ]; then
1419 zstd -f --format=xz tmp
1420 zstd -f --format=lzma tmp
1421 zstd -f tmp
1422 cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp
1423 truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !"
1424 truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !"
1425 rm -f tmp*
1433 zstd --format=lz4 -V || LZ4MODE=0
1434 if [ $LZ4MODE -eq 1 ]; then
1437 lz4 -V || LZ4EXE=0
1438 if [ $LZ4EXE -eq 1 ]; then
1440 zstd --format=lz4 -f tmp
1441 lz4 -t -v tmp.lz4
1442 lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout
1443 zstd -d -f -v tmp.lz4
1444 rm -f tmp*
1453 if [ $LZ4MODE -eq 1 ]; then
1456 zstd -f --format=lz4 tmp
1457 zstd -f tmp
1458 cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp
1459 truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !"
1460 rm -f tmp*
1468 ! zstd -d tmp.abc 2> tmplg
1470 if [ $GZIPMODE -ne 1 ]; then
1474 if [ $LZMAMODE -ne 1 ]; then
1479 if [ $LZ4MODE -ne 1 ]; then
1484 zstd tmp1 -o tmp1.zstd
1485 zstd -d -f tmp1.zstd # support .zstd suffix even though it's not the default suffix
1489 rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd
1492 tar -cf tmp.tar tmp
1493 zstd tmp.tar -o tmp.tzst
1494 rm -f tmp.tar
1495 zstd -d tmp.tzst
1496 [ -e tmp.tar ] || die ".tzst failed to decompress to .tar!"
1497 rm -f tmp.tar tmp.tzst
1499 if [ $GZIPMODE -eq 1 ]; then
1500 tar -f - -c tmp | gzip > tmp.tgz
1501 zstd -d tmp.tgz
1502 [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!"
1503 rm -f tmp.tar tmp.tgz
1506 if [ $LZMAMODE -eq 1 ]; then
1507 tar -f - -c tmp | zstd --format=xz > tmp.txz
1508 zstd -d tmp.txz
1509 [ -e tmp.tar ] || die ".txz failed to decompress to .tar!"
1510 rm -f tmp.tar tmp.txz
1513 if [ $LZ4MODE -eq 1 ]; then
1514 tar -f - -c tmp | zstd --format=lz4 > tmp.tlz4
1515 zstd -d tmp.tlz4
1516 [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!"
1517 rm -f tmp.tar tmp.tlz4
1521 ! zstd -d tmp.t
1522 ! zstd -d tmp.tz
1523 ! zstd -d tmp.tzs
1526 println "\n===> zstd round-trip tests "
1529 roundTripTest -g15K # TableID==3
1530 roundTripTest -g127K # TableID==2
1531 roundTripTest -g255K # TableID==1
1532 roundTripTest -g522K # TableID==0
1533 roundTripTest -g519K 6 # greedy, hash chain
1534 roundTripTest -g517K 16 # btlazy2
1535 roundTripTest -g516K 19 # btopt
1537 fileRoundTripTest -g500K
1539 println "\n===> zstd long distance matching round-trip tests "
1540 roundTripTest -g0 "2 --single-thread --long"
1541 roundTripTest -g1000K "1 --single-thread --long"
1542 roundTripTest -g517K "6 --single-thread --long"
1543 roundTripTest -g516K "16 --single-thread --long"
1544 roundTripTest -g518K "19 --single-thread --long"
1545 roundTripTest -g2M "22 --single-thread --ultra --long"
1546 fileRoundTripTest -g5M "3 --single-thread --long"
1549 roundTripTest -g96K "5 --single-thread"
1550 if [ -n "$hasMT" ]
1552 println "\n===> zstdmt round-trip tests "
1553 roundTripTest -g4M "1 -T0"
1554 roundTripTest -g4M "1 -T0 --auto-threads=physical"
1555 roundTripTest -g4M "1 -T0 --auto-threads=logical"
1556 roundTripTest -g8M "3 -T2"
1557 roundTripTest -g8000K "2 --threads=2"
1558 fileRoundTripTest -g4M "19 -T2 -B1M"
1560 println "\n===> zstdmt long distance matching round-trip tests "
1561 roundTripTest -g8M "3 --long=24 -T2"
1565 ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting
1566 ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting
1567 ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting
1568 ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting
1569 ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting
1570 ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting
1571 …ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default se…
1572 ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage
1573 ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread
1576 rm -f mt_tmp*
1579 datagen -g2MB > tmp
1580 refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c)
1581 ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c)
1582 ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c)
1583 if [ "$refSize" -eq "$ov9Size" ]; then
1587 if [ "$refSize" -eq "$ov1Size" ]; then
1591 if [ "$ov9Size" -ge "$ov1Size" ]; then
1600 rm -f tmp*
1602 println "\n===> zstd --list/-l single frame tests "
1606 zstd tmp*
1607 zstd -l ./*.zst
1608 zstd -lv ./*.zst | $GREP "Decompressed Size:" # check that decompressed size is present in header
1609 zstd --list ./*.zst
1610 zstd --list -v ./*.zst
1612 println "\n===> zstd --list/-l multiple frame tests "
1615 zstd -l ./*.zst
1616 zstd -lv ./*.zst
1618 println "\n===> zstd --list/-l error detection tests "
1619 zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file"
1620 zstd --list tmp* && die "-l must fail on non-zstd file"
1621 zstd -lv tmp1* && die "-l must fail on non-zstd file"
1622 zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file"
1625 TEST_DATA_FILE=truncatable-input.txt
1627 TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst
1628 datagen -g50000 > $TEST_DATA_FILE
1629 zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE
1631 zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file"
1633 rm -f $TEST_DATA_FILE
1634 rm -f $FULL_COMPRESSED_FILE
1635 rm -f $TRUNCATED_COMPRESSED_FILE
1637 println "\n===> zstd --list/-l errors when presented with stdin / no files"
1638 zstd -l && die "-l must fail on empty list of files"
1639 zstd -l - && die "-l does not work on stdin"
1640 zstd -l < tmp1.zst && die "-l does not work on stdin"
1641 zstd -l - < tmp1.zst && die "-l does not work on stdin"
1642 zstd -l - tmp1.zst && die "-l does not work on stdin"
1643 zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin"
1644 zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : z…
1646 println "\n===> zstd --list/-l test with null files "
1647 datagen -g0 > tmp5
1648 zstd tmp5
1649 zstd -l tmp5.zst
1650 zstd -l tmp5* && die "-l must fail on non-zstd file"
1651 zstd -lv tmp5.zst | $GREP "Decompressed Size: 0 B (0 B)" # check that 0 size is present in header
1652 zstd -lv tmp5* && die "-l must fail on non-zstd file"
1654 println "\n===> zstd --list/-l test with no content size field "
1655 datagen -g513K | zstd > tmp6.zst
1656 zstd -l tmp6.zst
1657 zstd -lv tmp6.zst | $GREP "Decompressed Size:" && die "Field :Decompressed Size: should not be ava…
1659 println "\n===> zstd --list/-l test with no checksum "
1660 zstd -f --no-check tmp1
1661 zstd -l tmp1.zst
1662 zstd -lv tmp1.zst
1664 println "\n===> zstd trace tests "
1665 zstd -f --trace tmp.trace tmp1
1666 zstd -f --trace tmp.trace tmp1 tmp2 tmp3
1667 zstd -f --trace tmp.trace tmp1 tmp2 tmp3 -o /dev/null
1668 zstd -f --trace tmp.trace tmp1 tmp2 tmp3 --single-thread
1669 zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null
1670 zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null --single-thread
1671 zstd --trace tmp.trace -t tmp1.zst
1672 zstd --trace tmp.trace -t tmp1.zst tmp2.zst
1673 zstd -f --trace tmp.trace -d tmp1.zst
1674 zstd -f --trace tmp.trace -d tmp1.zst tmp2.zst tmp3.zst
1675 zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1
1676 zstd -b1e10i0 --trace tmp.trace tmp1
1677 zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3
1679 rm -f tmp*
1682 println "\n===> zstd long distance matching tests "
1683 roundTripTest -g0 " --single-thread --long"
1684 roundTripTest -g9M "2 --single-thread --long"
1686 roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB"
1687 roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB"
1688 roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB"
1689 roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB"
1692 if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -ne "1" ]; then
1693 println "\n===> zstd long distance matching with optimal parser compressed size tests "
1694 optCSize16=$(datagen -g511K | zstd -16 -c | wc -c)
1695 longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c)
1696 optCSize19=$(datagen -g2M | zstd -19 -c | wc -c)
1697 longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c)
1698 optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c)
1699 longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c)
1700 if [ "$longCSize16" -gt "$optCSize16" ]; then
1701 echo using --long on compression level 16 should not cause compressed size regression
1703 elif [ "$longCSize19" -gt "$optCSize19" ]; then
1704 echo using --long on compression level 19 should not cause compressed size regression
1706 elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then
1707 … echo using --long on compression level 19 with wLog=23 should not cause compressed size regression
1712 println "\n===> zstd asyncio tests "
1715 datagen -g2M -s$2 >> tmp_uncompressed
1716 datagen -g2M -s$2 | zstd -1 --format=$1 >> tmp_compressed.zst
1725 roundTripTest -g2M "3 --asyncio --format=$1"
1726 roundTripTest -g2M "3 --no-asyncio --format=$1"
1729 rm -f tmp_compressed tmp_uncompressed
1730 testAsyncIO zstd
1731 addTwoFrames zstd
1732 if [ $GZIPMODE -eq 1 ]; then
1736 if [ $LZMAMODE -eq 1 ]; then
1740 if [ $LZ4MODE -eq 1 ]; then
1745 zstd -d tmp_compressed.zst --asyncio -c | $MD5SUM > tmp1
1746 $DIFF -q tmp1 tmp2
1748 zstd -d tmp_compressed.zst --no-asyncio -c | $MD5SUM > tmp1
1749 $DIFF -q tmp1 tmp2
1751 if [ "$1" != "--test-large-data" ]; then
1760 if [ -n "$hasMT" ]
1763 roundTripTest -g270000000 " --adapt"
1764 roundTripTest -g27000000 " --adapt=min=1,max=4"
1765 roundTripTest -g27000000 " --adapt=min=-2,max=-1"
1766 println "===> test: --adapt must fail on incoherent bounds "
1768 zstd --adapt= tmp && die "invalid compression parameter"
1769 zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds"
1772 roundTripTest -g10M " --rsyncable"
1773 roundTripTest -g10M " --rsyncable -B100K"
1774 println "===> test: --rsyncable must fail with --single-thread"
1775 zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread"
1778 println "\n===> patch-from=origin tests"
1779 datagen -g1000 -P50 > tmp_dict
1780 datagen -g1000 -P10 > tmp_patch
1781 zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff
1782 zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon
1783 $DIFF -s tmp_patch_recon tmp_patch
1785 println "\n===> alternate syntax: patch-from origin"
1786 zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff
1787 zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon
1788 $DIFF -s tmp_patch_recon tmp_patch
1789 rm -rf tmp_*
1791 println "\n===> patch-from recursive tests"
1796 zstd --patch-from=tmp_dict -r tmp_dir && die
1797 rm -rf tmp*
1799 println "\n===> patch-from long mode trigger larger file test"
1800 if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -eq "1" ]; then
1802 datagen -g10000000 > tmp_dict
1803 datagen -g10000000 > tmp_patch
1805 datagen -g5000000 > tmp_dict
1806 datagen -g5000000 > tmp_patch
1808 zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | $GREP "long mode automatically triggered"
1809 rm -rf tmp*
1811 println "\n===> patch-from very large dictionary and file test"
1812 datagen -g550000000 -P0 > tmp_dict
1813 datagen -g100000000 -P1 > tmp_patch
1814 zstd --long=30 -1f --patch-from tmp_dict tmp_patch
1815 zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon
1816 $DIFF -s tmp_patch_recon tmp_patch
1817 rm -rf tmp*
1819 println "\n===> patch-from --stream-size test"
1820 datagen -g1000 -P50 > tmp_dict
1821 datagen -g1000 -P10 > tmp_patch
1822 cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die
1823 cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff
1824 rm -rf tmp*
1828 roundTripTest -g270000000 1
1829 roundTripTest -g250000000 2
1830 roundTripTest -g230000000 3
1832 roundTripTest -g140000000 -P60 4
1833 roundTripTest -g130000000 -P62 5
1834 roundTripTest -g120000000 -P65 6
1836 roundTripTest -g70000000 -P70 7
1837 roundTripTest -g60000000 -P71 8
1838 roundTripTest -g50000000 -P73 9
1840 roundTripTest -g35000000 -P75 10
1841 roundTripTest -g30000000 -P76 11
1842 roundTripTest -g25000000 -P78 12
1844 roundTripTest -g18000013 -P80 13
1845 roundTripTest -g18000014 -P80 14
1846 roundTripTest -g18000015 -P81 15
1847 roundTripTest -g18000016 -P84 16
1848 roundTripTest -g18000017 -P88 17
1849 roundTripTest -g18000018 -P94 18
1850 roundTripTest -g18000019 -P96 19
1852 roundTripTest -g8M "19 --long"
1854 roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25"
1855 roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an over…
1857 fileRoundTripTest -g4193M -P99 1
1860 println "\n===> zstd long, long distance matching round-trip tests "
1861 roundTripTest -g270000000 "1 --single-thread --long"
1862 roundTripTest -g130000000 -P60 "5 --single-thread --long"
1863 roundTripTest -g35000000 -P70 "8 --single-thread --long"
1864 roundTripTest -g18000001 -P80 "18 --single-thread --long"
1866 roundTripTest -g700M -P50 "1 --single-thread --long=29"
1867 roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28"
1870 if [ -n "$hasMT" ]
1872 println "\n===> zstdmt long round-trip tests "
1873 roundTripTest -g80000000 -P99 "19 -T2" " "
1874 roundTripTest -g5000000000 -P99 "1 -T2" " "
1875 roundTripTest -g500000000 -P97 "1 -T999" " "
1876 fileRoundTripTest -g4103M -P98 " -T0" " "
1877 roundTripTest -g400000000 -P97 "1 --long=24 -T2" " "
1878 # Exposes the bug in https://github.com/facebook/zstd/pull/1678
1881 # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000"
1891 println "- Create first dictionary"
1892 zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
1894 zstd -f tmp -D tmpDict
1895 zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the sa…
1896 zstd -d tmp.zst -D tmpDict -fo result
1898 zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
1899 println "- Create second (different) dictionary"
1900 zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
1901 zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
1902 println "- Create dictionary using shrink-dict flag"
1903 zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
1904 zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
1905 zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
1906 zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3
1907 println "- Create dictionary with short dictID"
1908 zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
1910 println "- Create dictionary with size limit"
1911 zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
1912 println "- Compare size of dictionary from 90% training samples with 80% training samples"
1913 zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1914 zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1915 println "- Create dictionary using all samples for both training and testing"
1916 zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1917 println "- Test -o before --train-cover"
1918 rm -f tmpDict dictionary
1919 zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
1920 test -f tmpDict
1921 zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
1922 test -f dictionary
1923 rm -f tmp* dictionary
1925 rm -f tmp*