1 /*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12 /*-************************************
13 * Compiler specific
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
19 #endif
20
21
22 /*-************************************
23 * Includes
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #include <time.h> /* time_t, time(), to randomize seed */
29 #include <assert.h> /* assert */
30 #include "timefn.h" /* UTIL_time_t, UTIL_getTime */
31 #include "mem.h"
32 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
33 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
34 #include "zstd.h" /* ZSTD_compressBound */
35 #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
36 #include "zdict.h" /* ZDICT_trainFromBuffer */
37 #include "datagen.h" /* RDG_genBuffer */
38 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
39 #include "xxhash.h" /* XXH64_* */
40 #include "seqgen.h"
41 #include "util.h"
42 #include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
43 #include "external_matchfinder.h" /* zstreamSequenceProducer, EMF_testCase */
44
45 /*-************************************
46 * Constants
47 **************************************/
48 #define KB *(1U<<10)
49 #define MB *(1U<<20)
50 #define GB *(1U<<30)
51
52 static const int nbTestsDefault = 10000;
53 static const U32 g_cLevelMax_smallTests = 10;
54 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
55 #define FUZ_COMPRESSIBILITY_DEFAULT 50
56 static const U32 prime32 = 2654435761U;
57
58
59 /*-************************************
60 * Display Macros
61 **************************************/
62 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
63 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \
64 DISPLAY(__VA_ARGS__); \
65 if (g_displayLevel>=4) fflush(stderr); }
66 static U32 g_displayLevel = 2;
67
68 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
69 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
70
71 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
72 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
73 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
74 if (g_displayLevel>=4) fflush(stderr); } }
75
76 static U64 g_clockTime = 0;
77
78
79 /*-*******************************************************
80 * Check macros
81 *********************************************************/
82 #undef MIN
83 #undef MAX
84 #define MIN(a,b) ((a)<(b)?(a):(b))
85 #define MAX(a,b) ((a)>(b)?(a):(b))
86 /*! FUZ_rand() :
87 @return : a 27 bits random value, from a 32-bits `seed`.
88 `seed` is also modified */
89 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(U32 * seedPtr)90 static U32 FUZ_rand(U32* seedPtr)
91 {
92 static const U32 prime2 = 2246822519U;
93 U32 rand32 = *seedPtr;
94 rand32 *= prime32;
95 rand32 += prime2;
96 rand32 = FUZ_rotl32(rand32, 13);
97 *seedPtr = rand32;
98 return rand32 >> 5;
99 }
100
101 #define CHECK(cond, ...) { \
102 if (cond) { \
103 DISPLAY("Error => "); \
104 DISPLAY(__VA_ARGS__); \
105 DISPLAY(" (seed %u, test nb %u, line %u) \n", \
106 (unsigned)seed, testNb, __LINE__); \
107 goto _output_error; \
108 } }
109
110 #define CHECK_Z(f) { \
111 size_t const err = f; \
112 CHECK(ZSTD_isError(err), "%s : %s ", \
113 #f, ZSTD_getErrorName(err)); \
114 }
115
116 #define CHECK_RET(ret, cond, ...) { \
117 if (cond) { \
118 DISPLAY("Error %llu => ", (unsigned long long)ret); \
119 DISPLAY(__VA_ARGS__); \
120 DISPLAY(" (line %u)\n", __LINE__); \
121 return ret; \
122 } }
123
124 #define CHECK_RET_Z(f) { \
125 size_t const err = f; \
126 CHECK_RET(err, ZSTD_isError(err), "%s : %s ", \
127 #f, ZSTD_getErrorName(err)); \
128 }
129
130
131 /*======================================================
132 * Basic Unit tests
133 *======================================================*/
134
135 typedef struct {
136 void* start;
137 size_t size;
138 size_t filled;
139 } buffer_t;
140
141 static const buffer_t kBuffNull = { NULL, 0 , 0 };
142
FUZ_freeDictionary(buffer_t dict)143 static void FUZ_freeDictionary(buffer_t dict)
144 {
145 free(dict.start);
146 }
147
FUZ_createDictionary(const void * src,size_t srcSize,size_t blockSize,size_t requestedDictSize)148 static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
149 {
150 buffer_t dict = kBuffNull;
151 size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
152 size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
153 if (!blockSizes) return kBuffNull;
154 dict.start = malloc(requestedDictSize);
155 if (!dict.start) { free(blockSizes); return kBuffNull; }
156 { size_t nb;
157 for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
158 blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
159 }
160 { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
161 free(blockSizes);
162 if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; }
163 dict.size = requestedDictSize;
164 dict.filled = dictSize;
165 return dict;
166 }
167 }
168
169 /* Round trips data and updates xxh with the decompressed data produced */
SEQ_roundTrip(ZSTD_CCtx * cctx,ZSTD_DCtx * dctx,XXH64_state_t * xxh,void * data,size_t size,ZSTD_EndDirective endOp)170 static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
171 XXH64_state_t* xxh, void* data, size_t size,
172 ZSTD_EndDirective endOp)
173 {
174 static BYTE compressed[1024];
175 static BYTE uncompressed[1024];
176
177 ZSTD_inBuffer cin = {data, size, 0};
178 size_t cret;
179
180 do {
181 ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
182 ZSTD_inBuffer din = { compressed, 0, 0 };
183 ZSTD_outBuffer dout = { uncompressed, 0, 0 };
184
185 cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
186 if (ZSTD_isError(cret))
187 return cret;
188
189 din.size = cout.pos;
190 while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
191 size_t dret;
192
193 dout.pos = 0;
194 dout.size = sizeof(uncompressed);
195 dret = ZSTD_decompressStream(dctx, &dout, &din);
196 if (ZSTD_isError(dret))
197 return dret;
198 XXH64_update(xxh, dout.dst, dout.pos);
199 if (dret == 0)
200 break;
201 }
202 } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
203 return 0;
204 }
205
206 /* Generates some data and round trips it */
SEQ_generateRoundTrip(ZSTD_CCtx * cctx,ZSTD_DCtx * dctx,XXH64_state_t * xxh,SEQ_stream * seq,SEQ_gen_type type,unsigned value)207 static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
208 XXH64_state_t* xxh, SEQ_stream* seq,
209 SEQ_gen_type type, unsigned value)
210 {
211 static BYTE data[1024];
212 size_t gen;
213
214 do {
215 SEQ_outBuffer sout = {data, sizeof(data), 0};
216 size_t ret;
217 gen = SEQ_gen(seq, type, value, &sout);
218
219 ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
220 if (ZSTD_isError(ret))
221 return ret;
222 } while (gen != 0);
223
224 return 0;
225 }
226
getCCtxParams(ZSTD_CCtx * zc,ZSTD_parameters * savedParams)227 static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
228 {
229 int value;
230 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
231 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
232 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
233 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
234 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
235 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
236 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
237 savedParams->cParams.strategy = value;
238
239 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
240 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
241 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
242 savedParams->fParams.noDictIDFlag = !value;
243 return 0;
244 }
245
badParameters(ZSTD_CCtx * zc,ZSTD_parameters const savedParams)246 static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
247 {
248 ZSTD_parameters params;
249 if (ZSTD_isError(getCCtxParams(zc, ¶ms))) return 10;
250 CHECK_RET(1, params.cParams.windowLog != savedParams.cParams.windowLog, "windowLog");
251 CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
252 CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
253 CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
254 CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
255 CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
256
257 CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
258 CHECK_RET(8, params.fParams.contentSizeFlag != savedParams.fParams.contentSizeFlag, "contentSizeFlag");
259 CHECK_RET(9, params.fParams.noDictIDFlag != savedParams.fParams.noDictIDFlag, "noDictIDFlag");
260 return 0;
261 }
262
basicUnitTests(U32 seed,double compressibility,int bigTests)263 static int basicUnitTests(U32 seed, double compressibility, int bigTests)
264 {
265 size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
266 void* CNBuffer = malloc(CNBufferSize);
267 size_t const skippableFrameSize = 200 KB;
268 size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
269 void* compressedBuffer = malloc(compressedBufferSize);
270 size_t const decodedBufferSize = CNBufferSize;
271 void* decodedBuffer = malloc(decodedBufferSize);
272 size_t cSize;
273 int testResult = 0;
274 int testNb = 1;
275 U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */
276 ZSTD_CStream* zc = ZSTD_createCStream();
277 ZSTD_DStream* zd = ZSTD_createDStream();
278 ZSTD_CCtx* mtctx = ZSTD_createCCtx();
279
280 ZSTD_inBuffer inBuff, inBuff2;
281 ZSTD_outBuffer outBuff;
282 buffer_t dictionary = kBuffNull;
283 size_t const dictSize = 128 KB;
284 unsigned dictID = 0;
285
286 /* Create compressible test buffer */
287 if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd || !mtctx) {
288 DISPLAY("Not enough memory, aborting \n");
289 goto _output_error;
290 }
291 RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
292
293 CHECK_Z(ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2));
294
295 /* Create dictionary */
296 DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
297 dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB);
298 if (!dictionary.start) {
299 DISPLAY("Error creating dictionary, aborting \n");
300 goto _output_error;
301 }
302 dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
303
304 /* Basic compression test */
305 DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
306 CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
307 outBuff.dst = (char*)(compressedBuffer);
308 outBuff.size = compressedBufferSize;
309 outBuff.pos = 0;
310 inBuff.src = CNBuffer;
311 inBuff.size = CNBufferSize;
312 inBuff.pos = 0;
313 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
314 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
315 { size_t const r = ZSTD_endStream(zc, &outBuff);
316 if (r != 0) goto _output_error; } /* error, or some data not flushed */
317 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos);
318
319 /* generate skippable frame */
320 MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
321 MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
322 cSize = skippableFrameSize + 8;
323
324 /* Basic compression test using dict */
325 DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
326 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
327 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
328 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) );
329 outBuff.dst = (char*)(compressedBuffer)+cSize;
330 assert(compressedBufferSize > cSize);
331 outBuff.size = compressedBufferSize - cSize;
332 outBuff.pos = 0;
333 inBuff.src = CNBuffer;
334 inBuff.size = CNBufferSize;
335 inBuff.pos = 0;
336 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
337 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
338 { size_t const r = ZSTD_endStream(zc, &outBuff);
339 if (r != 0) goto _output_error; } /* error, or some data not flushed */
340 cSize += outBuff.pos;
341 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
342 (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
343
344 /* context size functions */
345 DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
346 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
347 size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams);
348 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */
349 if (ZSTD_isError(cstreamSize)) goto _output_error;
350 if (ZSTD_isError(cdictSize)) goto _output_error;
351 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize));
352 }
353
354 /* context size functions */
355 DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++);
356 { ZSTD_CCtx_params* const params = ZSTD_createCCtxParams();
357 size_t cstreamSize, cctxSize;
358 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) );
359 cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
360 CHECK_Z(cstreamSize);
361 cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
362 CHECK_Z(cctxSize);
363 if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error;
364 ZSTD_freeCCtxParams(params);
365 DISPLAYLEVEL(3, "OK \n");
366 }
367
368 DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
369 { size_t const s = ZSTD_sizeof_CStream(zc);
370 if (ZSTD_isError(s)) goto _output_error;
371 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
372 }
373
374 /* Attempt bad compression parameters */
375 DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++);
376 { size_t r;
377 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
378 params.cParams.minMatch = 2;
379 r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
380 if (!ZSTD_isError(r)) goto _output_error;
381 DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
382 }
383
384 /* skippable frame test */
385 DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
386 CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) );
387 inBuff.src = compressedBuffer;
388 inBuff.size = cSize;
389 inBuff.pos = 0;
390 outBuff.dst = decodedBuffer;
391 outBuff.size = CNBufferSize;
392 outBuff.pos = 0;
393 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
394 DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r);
395 if (r != 0) goto _output_error;
396 }
397 if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */
398 DISPLAYLEVEL(3, "OK \n");
399
400 /* Basic decompression test */
401 inBuff2 = inBuff;
402 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
403 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
404 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */
405 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
406 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
407 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
408 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
409 DISPLAYLEVEL(3, "OK \n");
410
411 /* Reuse without init */
412 DISPLAYLEVEL(3, "test%3i : decompress again without init (reuse previous settings): ", testNb++);
413 outBuff.pos = 0;
414 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
415 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
416 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
417 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
418 DISPLAYLEVEL(3, "OK \n");
419
420 /* check regenerated data is byte exact */
421 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
422 { size_t i;
423 for (i=0; i<CNBufferSize; i++) {
424 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
425 } }
426 DISPLAYLEVEL(3, "OK \n");
427
428 /* check decompression fails early if first bytes are wrong */
429 DISPLAYLEVEL(3, "test%3i : early decompression error if first bytes are incorrect : ", testNb++);
430 { const char buf[3] = { 0 }; /* too short, not enough to start decoding header */
431 ZSTD_inBuffer inb = { buf, sizeof(buf), 0 };
432 size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inb);
433 if (!ZSTD_isError(remaining)) goto _output_error; /* should have errored out immediately (note: this does not test the exact error code) */
434 }
435 DISPLAYLEVEL(3, "OK \n");
436
437 /* context size functions */
438 DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
439 { ZSTD_FrameHeader fhi;
440 const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
441 size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
442 if (gfhError!=0) goto _output_error;
443 DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize);
444 { size_t const s = ZSTD_estimateDStreamSize((size_t)fhi.windowSize)
445 /* uses ZSTD_initDStream_usingDict() */
446 + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
447 if (ZSTD_isError(s)) goto _output_error;
448 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
449 } }
450
451 DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
452 { size_t const s = ZSTD_sizeof_DStream(zd);
453 if (ZSTD_isError(s)) goto _output_error;
454 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
455 }
456
457 /* Decompression by small increment */
458 DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
459 { /* skippable frame */
460 size_t r = 1;
461 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
462 inBuff.src = compressedBuffer;
463 outBuff.dst = decodedBuffer;
464 inBuff.pos = 0;
465 outBuff.pos = 0;
466 while (r) { /* skippable frame */
467 size_t const inSize = (FUZ_rand(&coreSeed) & 15) + 1;
468 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + 1;
469 inBuff.size = inBuff.pos + inSize;
470 outBuff.size = outBuff.pos + outSize;
471 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
472 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream on skippable frame error : %s \n", ZSTD_getErrorName(r));
473 if (ZSTD_isError(r)) goto _output_error;
474 }
475 /* normal frame */
476 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
477 r=1;
478 while (r) {
479 size_t const inSize = FUZ_rand(&coreSeed) & 15;
480 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + (!inSize); /* avoid having both sizes at 0 => would trigger a no_forward_progress error */
481 inBuff.size = inBuff.pos + inSize;
482 outBuff.size = outBuff.pos + outSize;
483 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
484 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(r));
485 if (ZSTD_isError(r)) goto _output_error;
486 }
487 }
488 if (outBuff.pos != CNBufferSize) DISPLAYLEVEL(4, "outBuff.pos != CNBufferSize : should have regenerated same amount ! \n");
489 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
490 if (inBuff.pos != cSize) DISPLAYLEVEL(4, "inBuff.pos != cSize : should have real all input ! \n");
491 if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
492 DISPLAYLEVEL(3, "OK \n");
493
494 /* check regenerated data is byte exact */
495 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
496 { size_t i;
497 for (i=0; i<CNBufferSize; i++) {
498 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
499 } }
500 DISPLAYLEVEL(3, "OK \n");
501
502 /* Decompression forward progress */
503 DISPLAYLEVEL(3, "test%3i : generate error when ZSTD_decompressStream() doesn't progress : ", testNb++);
504 { /* skippable frame */
505 size_t r = 0;
506 int decNb = 0;
507 int const maxDec = 100;
508 inBuff.src = compressedBuffer;
509 inBuff.size = cSize;
510 inBuff.pos = 0;
511
512 outBuff.dst = decodedBuffer;
513 outBuff.pos = 0;
514 outBuff.size = CNBufferSize-1; /* 1 byte missing */
515
516 for (decNb=0; decNb<maxDec; decNb++) {
517 if (r==0) ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
518 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
519 if (ZSTD_isError(r)) break;
520 }
521 if (!ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream should have triggered a no_forward_progress error \n");
522 if (!ZSTD_isError(r)) goto _output_error; /* should have triggered no_forward_progress error */
523 }
524 DISPLAYLEVEL(3, "OK \n");
525
526 DISPLAYLEVEL(3, "test%3i : NULL output and NULL input : ", testNb++);
527 inBuff.src = NULL;
528 inBuff.size = 0;
529 inBuff.pos = 0;
530 outBuff.dst = NULL;
531 outBuff.size = 0;
532 outBuff.pos = 0;
533 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
534 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
535 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
536 outBuff.dst = (char*)(compressedBuffer);
537 outBuff.size = compressedBufferSize;
538 outBuff.pos = 0;
539 { size_t const r = ZSTD_endStream(zc, &outBuff);
540 CHECK(r != 0, "Error or some data not flushed (ret=%i)", ZSTD_getErrorCode(r));
541 }
542 inBuff.src = outBuff.dst;
543 inBuff.size = outBuff.pos;
544 inBuff.pos = 0;
545 outBuff.dst = NULL;
546 outBuff.size = 0;
547 outBuff.pos = 0;
548 CHECK_Z( ZSTD_initDStream(zd) );
549 { size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
550 if (ret != 0) goto _output_error;
551 }
552 DISPLAYLEVEL(3, "OK\n");
553
554 DISPLAYLEVEL(3, "test%3i : NULL output buffer with non-NULL input : ", testNb++);
555 {
556 const char* test = "aa";
557 inBuff.src = test;
558 inBuff.size = 2;
559 inBuff.pos = 0;
560 outBuff.dst = NULL;
561 outBuff.size = 0;
562 outBuff.pos = 0;
563 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
564 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
565 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
566 outBuff.dst = (char*)(compressedBuffer);
567 outBuff.size = compressedBufferSize;
568 outBuff.pos = 0;
569 { size_t const r = ZSTD_endStream(zc, &outBuff);
570 CHECK(r != 0, "Error or some data not flushed (ret=%i)", ZSTD_getErrorCode(r));
571 }
572 inBuff.src = outBuff.dst;
573 inBuff.size = outBuff.pos;
574 inBuff.pos = 0;
575 outBuff.dst = NULL;
576 outBuff.size = 0;
577 outBuff.pos = 0;
578 CHECK_Z( ZSTD_initDStream(zd) );
579 CHECK_Z(ZSTD_decompressStream(zd, &outBuff, &inBuff));
580 }
581
582 DISPLAYLEVEL(3, "OK\n");
583 /* _srcSize compression test */
584 DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
585 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
586 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
587 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
588 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) );
589 outBuff.dst = (char*)(compressedBuffer);
590 outBuff.size = compressedBufferSize;
591 outBuff.pos = 0;
592 inBuff.src = CNBuffer;
593 inBuff.size = CNBufferSize;
594 inBuff.pos = 0;
595 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
596 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
597 { size_t const r = ZSTD_endStream(zc, &outBuff);
598 CHECK(r != 0, "Error or some data not flushed (ret=%i)", ZSTD_getErrorCode(r));
599 }
600 { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
601 CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
602 CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
603 }
604 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
605
606 /* wrong _srcSize compression test */
607 DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
608 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
609 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
610 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
611 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) );
612 outBuff.dst = (char*)(compressedBuffer);
613 outBuff.size = compressedBufferSize;
614 outBuff.pos = 0;
615 inBuff.src = CNBuffer;
616 inBuff.size = CNBufferSize;
617 inBuff.pos = 0;
618 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
619 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
620 { size_t const r = ZSTD_endStream(zc, &outBuff);
621 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
622 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
623
624 /* wrong _srcSize compression test */
625 DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
626 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
627 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
628 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
629 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) );
630 outBuff.dst = (char*)(compressedBuffer);
631 outBuff.size = compressedBufferSize;
632 outBuff.pos = 0;
633 inBuff.src = CNBuffer;
634 inBuff.size = CNBufferSize;
635 inBuff.pos = 0;
636 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
637 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
638 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
639 }
640
641 DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
642 { CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
643 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) );
644 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) );
645 outBuff.dst = (char*)compressedBuffer;
646 outBuff.size = compressedBufferSize;
647 outBuff.pos = 0;
648 inBuff.src = CNBuffer;
649 inBuff.size = CNBufferSize;
650 inBuff.pos = 0;
651 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
652 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
653 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
654 } }
655
656 /* Compression state reuse scenario */
657 DISPLAYLEVEL(3, "test%3i : context reuse : ", testNb++);
658 ZSTD_freeCStream(zc);
659 zc = ZSTD_createCStream();
660 if (zc==NULL) goto _output_error; /* memory allocation issue */
661 /* use 1 */
662 { size_t const inSize = 513;
663 DISPLAYLEVEL(5, "use1 ");
664 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
665 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
666 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
667 inBuff.src = CNBuffer;
668 inBuff.size = inSize;
669 inBuff.pos = 0;
670 outBuff.dst = (char*)(compressedBuffer)+cSize;
671 outBuff.size = ZSTD_compressBound(inSize);
672 outBuff.pos = 0;
673 DISPLAYLEVEL(5, "compress1 ");
674 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
675 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
676 DISPLAYLEVEL(5, "end1 ");
677 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
678 }
679 /* use 2 */
680 { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
681 DISPLAYLEVEL(5, "use2 ");
682 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
683 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
684 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
685 inBuff.src = CNBuffer;
686 inBuff.size = inSize;
687 inBuff.pos = 0;
688 outBuff.dst = (char*)(compressedBuffer)+cSize;
689 outBuff.size = ZSTD_compressBound(inSize);
690 outBuff.pos = 0;
691 DISPLAYLEVEL(5, "compress2 ");
692 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
693 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
694 DISPLAYLEVEL(5, "end2 ");
695 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
696 }
697 DISPLAYLEVEL(3, "OK \n");
698
699 /* Decompression single pass with empty frame */
700 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, 1);
701 CHECK_Z(cSize);
702 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass on empty frame : ", testNb++);
703 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
704 size_t const dctxSize = ZSTD_sizeof_DCtx(dctx);
705 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
706
707 outBuff.dst = decodedBuffer;
708 outBuff.pos = 0;
709 outBuff.size = CNBufferSize;
710
711 inBuff.src = compressedBuffer;
712 inBuff.size = cSize;
713 inBuff.pos = 0;
714 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
715 CHECK_Z(r);
716 CHECK(r != 0, "Entire frame must be decompressed");
717 CHECK(outBuff.pos != 0, "Wrong size!");
718 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
719 }
720 CHECK(dctxSize != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
721 ZSTD_freeDCtx(dctx);
722 }
723 DISPLAYLEVEL(3, "OK \n");
724
725 DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2KB : ", testNb++);
726 {
727 ZSTD_DCtx* dctx = ZSTD_createDCtx();
728 size_t singlePassSize, streamingSize, streaming2KSize;
729
730 {
731 ZSTD_CCtx* cctx = ZSTD_createCCtx();
732 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
733 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18));
734 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
735 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048));
736 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize);
737 CHECK_Z(cSize);
738 ZSTD_freeCCtx(cctx);
739 }
740
741 CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBufferSize, compressedBuffer, cSize));
742 singlePassSize = ZSTD_sizeof_DCtx(dctx);
743 CHECK_Z(singlePassSize);
744
745 inBuff.src = compressedBuffer;
746 inBuff.size = cSize;
747
748 outBuff.dst = decodedBuffer;
749 outBuff.size = decodedBufferSize;
750
751 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048));
752 inBuff.pos = 0;
753 outBuff.pos = 0;
754 {
755 size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
756 CHECK_Z(r);
757 CHECK(r != 0, "Entire frame must be decompressed");
758 }
759 streaming2KSize = ZSTD_sizeof_DCtx(dctx);
760 CHECK_Z(streaming2KSize);
761
762 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters));
763 inBuff.pos = 0;
764 outBuff.pos = 0;
765 {
766 size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
767 CHECK_Z(r);
768 CHECK(r != 0, "Entire frame must be decompressed");
769 }
770 streamingSize = ZSTD_sizeof_DCtx(dctx);
771 CHECK_Z(streamingSize);
772
773 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024));
774 inBuff.pos = 0;
775 outBuff.pos = 0;
776 CHECK(!ZSTD_isError(ZSTD_decompressStream(dctx, &outBuff, &inBuff)), "decompression must fail");
777
778 CHECK(streamingSize < singlePassSize + (1 << 18) + 3 * ZSTD_BLOCKSIZE_MAX, "Streaming doesn't use the right amount of memory");
779 CHECK(streamingSize != streaming2KSize + 3 * (ZSTD_BLOCKSIZE_MAX - 2048), "ZSTD_d_blockSizeMax didn't save the right amount of memory");
780 DISPLAYLEVEL(3, "| %u | %u | %u | ", (unsigned)singlePassSize, (unsigned)streaming2KSize, (unsigned)streamingSize);
781
782 ZSTD_freeDCtx(dctx);
783 }
784 DISPLAYLEVEL(3, "OK \n");
785
786 /* Decompression with ZSTD_d_stableOutBuffer */
787 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1);
788 CHECK_Z(cSize);
789 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
790 size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
791 size_t dctxSize1;
792 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
793
794 outBuff.dst = decodedBuffer;
795 outBuff.pos = 0;
796 outBuff.size = CNBufferSize;
797
798 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass : ", testNb++);
799 inBuff.src = compressedBuffer;
800 inBuff.size = cSize;
801 inBuff.pos = 0;
802 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
803 CHECK_Z(r);
804 CHECK(r != 0, "Entire frame must be decompressed");
805 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
806 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
807 }
808 CHECK(dctxSize0 != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
809 DISPLAYLEVEL(3, "OK \n");
810
811 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer : ", testNb++);
812 outBuff.pos = 0;
813 inBuff.pos = 0;
814 inBuff.size = 0;
815 while (inBuff.pos < cSize) {
816 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
817 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
818 }
819 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
820 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
821 dctxSize1 = ZSTD_sizeof_DCtx(dctx);
822 CHECK(!(dctxSize0 < dctxSize1), "Input buffer allocated");
823 DISPLAYLEVEL(3, "OK \n");
824
825 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer too small : ", testNb++);
826 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
827 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
828 inBuff.src = compressedBuffer;
829 inBuff.size = cSize;
830 inBuff.pos = 0;
831 outBuff.pos = 0;
832 outBuff.size = CNBufferSize - 1;
833 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
834 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall, "Must error but got %s", ZSTD_getErrorName(r));
835 }
836 DISPLAYLEVEL(3, "OK \n");
837
838 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer modified : ", testNb++);
839 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
840 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
841 inBuff.src = compressedBuffer;
842 inBuff.size = cSize - 1;
843 inBuff.pos = 0;
844 outBuff.pos = 0;
845 outBuff.size = CNBufferSize;
846 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
847 ++inBuff.size;
848 outBuff.pos = 0;
849 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
850 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r));
851 }
852 DISPLAYLEVEL(3, "OK \n");
853
854 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++);
855 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
856 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0));
857 outBuff.pos = 0;
858 inBuff.pos = 0;
859 inBuff.size = 0;
860 while (inBuff.pos < cSize) {
861 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
862 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
863 }
864 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
865 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
866 CHECK(!(dctxSize1 < ZSTD_sizeof_DCtx(dctx)), "Output buffer allocated");
867 DISPLAYLEVEL(3, "OK \n");
868
869 ZSTD_freeDCtx(dctx);
870 }
871
872 /* Compression with ZSTD_c_stable{In,Out}Buffer */
873 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
874 ZSTD_inBuffer in;
875 ZSTD_outBuffer out;
876 size_t cctxSize1;
877 size_t cctxSize2;
878 assert(cctx != NULL);
879 in.src = CNBuffer;
880 in.size = CNBufferSize;
881 out.dst = compressedBuffer;
882 out.size = compressedBufferSize;
883 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
884 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++);
885 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
886 CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
887 /* check that compression fits with just a 8-bytes margin */
888 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize+8, CNBuffer, CNBufferSize));
889 CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
890 /* @cctxSize2 : sizeof_CCtx when doing full streaming (no stable in/out) */
891 { ZSTD_CCtx* const cctx2 = ZSTD_createCCtx();
892 assert(cctx2 != NULL);
893 in.pos = out.pos = 0;
894 CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
895 CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
896 CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
897 ZSTD_freeCCtx(cctx2);
898 }
899 /* @cctxSize1 : sizeof_CCtx when doing single-shot compression (no streaming) */
900 { ZSTD_CCtx* const cctx1 = ZSTD_createCCtx();
901 ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
902 size_t cSize3;
903 assert(cctx1 != NULL);
904 params.fParams.checksumFlag = 1;
905 cSize3 = ZSTD_compress_advanced(cctx1, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
906 CHECK_Z(cSize3);
907 CHECK(!(cSize == cSize3), "Must be same compressed size");
908 CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx1)), "Must be same CCtx size");
909 ZSTD_freeCCtx(cctx1);
910 }
911 CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
912 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
913 DISPLAYLEVEL(3, "OK \n");
914
915 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
916 { int stableInBuffer;
917 int stableOutBuffer;
918 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
919 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
920 CHECK(!(stableInBuffer == 0), "Modified");
921 CHECK(!(stableOutBuffer == 0), "Modified");
922 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
923 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
924 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
925 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
926 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
927 CHECK(!(stableInBuffer == 1), "Modified");
928 CHECK(!(stableOutBuffer == 1), "Modified");
929 }
930 DISPLAYLEVEL(3, "OK \n");
931
932 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer : ", testNb++);
933 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
934 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
935 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
936 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
937 in.pos = out.pos = 0;
938 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
939 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
940 DISPLAYLEVEL(3, "OK \n");
941
942 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer allocated size : ", testNb++);
943 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
944 CHECK(!(cctxSize1 == cctxSize), "Must be the same size as single pass");
945 }
946 DISPLAYLEVEL(3, "OK \n");
947
948 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer only : ", testNb++);
949 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
950 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
951 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
952 in.pos = out.pos = 0;
953 out.size = cSize / 4;
954 for (;;) {
955 size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
956 CHECK_Z(ret);
957 if (ret == 0)
958 break;
959 out.size = MIN(out.size + cSize / 4, compressedBufferSize);
960 }
961 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, out.pos));
962 DISPLAYLEVEL(3, "OK \n");
963
964 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++);
965 in.pos = out.pos = 0;
966 out.size = cSize / 4;
967 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
968 in.src = (char const*)in.src + in.pos;
969 in.size -= in.pos;
970 in.pos = 0;
971 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
972 CHECK(!ZSTD_isError(ret), "Must error");
973 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
974 }
975 DISPLAYLEVEL(3, "OK \n");
976
977 /* stableSrc + streaming */
978 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream, flushStream and endStream : ", testNb++);
979 CHECK_Z( ZSTD_initCStream(cctx, 1) );
980 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
981 { ZSTD_inBuffer inBuf;
982 ZSTD_outBuffer outBuf;
983 const size_t nonZeroStartPos = 18;
984 const size_t inputSize = 500;
985 inBuf.src = CNBuffer;
986 inBuf.size = 100;
987 inBuf.pos = nonZeroStartPos;
988 outBuf.dst = (char*)(compressedBuffer)+cSize;
989 outBuf.size = ZSTD_compressBound(inputSize);
990 outBuf.pos = 0;
991 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
992 inBuf.size = 200;
993 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
994 CHECK_Z( ZSTD_flushStream(cctx, &outBuf) );
995 inBuf.size = nonZeroStartPos + inputSize;
996 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
997 CHECK(ZSTD_endStream(cctx, &outBuf) != 0, "compression should be successful and fully flushed");
998 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
999 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
1000 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
1001 CHECK_Z(decSize);
1002 CHECK(decSize != inputSize, "regenerated %u bytes, instead of %u", (unsigned)decSize, (unsigned)inputSize);
1003 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
1004 } }
1005 DISPLAYLEVEL(3, "OK \n");
1006
1007 /* stableSrc + streaming */
1008 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream2, using different end directives : ", testNb++);
1009 CHECK_Z( ZSTD_initCStream(cctx, 1) );
1010 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
1011 { ZSTD_inBuffer inBuf;
1012 ZSTD_outBuffer outBuf;
1013 const size_t nonZeroStartPos = 18;
1014 const size_t inputSize = 500;
1015 inBuf.src = CNBuffer;
1016 inBuf.size = 100;
1017 inBuf.pos = nonZeroStartPos;
1018 outBuf.dst = (char*)(compressedBuffer)+cSize;
1019 outBuf.size = ZSTD_compressBound(inputSize);
1020 outBuf.pos = 0;
1021 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
1022 inBuf.size = 200;
1023 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
1024 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_flush) );
1025 inBuf.size = nonZeroStartPos + inputSize;
1026 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
1027 CHECK( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_end) != 0, "compression should be successful and fully flushed");
1028 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
1029 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
1030 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
1031 CHECK_Z(decSize);
1032 CHECK(decSize != inputSize, "regenerated %u bytes, instead of %u", (unsigned)decSize, (unsigned)inputSize);
1033 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
1034 } }
1035 DISPLAYLEVEL(3, "OK \n");
1036
1037 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer: context size : ", testNb++);
1038 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
1039 DISPLAYLEVEL(4, "cctxSize1=%u; cctxSize=%u; cctxSize2=%u : ", (unsigned)cctxSize1, (unsigned)cctxSize, (unsigned)cctxSize2);
1040 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
1041 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
1042 cctxSize1 = cctxSize;
1043 }
1044 DISPLAYLEVEL(3, "OK \n");
1045
1046 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer only : ", testNb++);
1047 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
1048 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1049 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
1050 in.src = CNBuffer;
1051 in.pos = out.pos = 0;
1052 in.size = MIN(CNBufferSize, 10);
1053 out.size = compressedBufferSize;
1054 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1055 in.pos = 0;
1056 in.size = CNBufferSize - in.size;
1057 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
1058 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, out.pos));
1059 DISPLAYLEVEL(3, "OK \n");
1060
1061 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer modify buffer : ", testNb++);
1062 in.pos = out.pos = 0;
1063 in.size = CNBufferSize;
1064 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1065 in.pos = out.pos = 0;
1066 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1067 CHECK(!ZSTD_isError(ret), "Must have errored");
1068 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
1069 }
1070 DISPLAYLEVEL(3, "OK \n");
1071
1072 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer: context size : ", testNb++);
1073 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
1074 DISPLAYLEVEL(4, "cctxSize1=%u; cctxSize=%u; cctxSize2=%u : ", (unsigned)cctxSize1, (unsigned)cctxSize, (unsigned)cctxSize2);
1075 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
1076 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
1077 }
1078 DISPLAYLEVEL(3, "OK \n");
1079
1080 ZSTD_freeCCtx(cctx);
1081 }
1082
1083 /* CDict scenario */
1084 DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
1085 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
1086 size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
1087 DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError);
1088 if (ZSTD_isError(initError)) goto _output_error;
1089 outBuff.dst = compressedBuffer;
1090 outBuff.size = compressedBufferSize;
1091 outBuff.pos = 0;
1092 inBuff.src = CNBuffer;
1093 inBuff.size = CNBufferSize;
1094 inBuff.pos = 0;
1095 DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
1096 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1097 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1098 { size_t const r = ZSTD_endStream(zc, &outBuff);
1099 DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r);
1100 if (r != 0) goto _output_error; /* error, or some data not flushed */
1101 }
1102 cSize = outBuff.pos;
1103 ZSTD_freeCDict(cdict);
1104 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1105 }
1106
1107 DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
1108 { size_t const s = ZSTD_sizeof_CStream(zc);
1109 if (ZSTD_isError(s)) goto _output_error;
1110 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
1111 }
1112
1113 DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
1114 { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1115 if (dID != dictID) goto _output_error;
1116 DISPLAYLEVEL(4, "OK (%u) \n", dID);
1117 }
1118
1119 /* DDict scenario */
1120 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize);
1121 { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1122 size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
1123 if (ZSTD_isError(initError)) goto _output_error;
1124 outBuff.dst = decodedBuffer;
1125 outBuff.size = CNBufferSize;
1126 outBuff.pos = 0;
1127 inBuff.src = compressedBuffer;
1128 inBuff.size = cSize;
1129 inBuff.pos = 0;
1130 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1131 if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
1132 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
1133 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
1134 ZSTD_freeDDict(ddict);
1135 DISPLAYLEVEL(3, "OK \n");
1136 }
1137
1138 /* Memory restriction */
1139 DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
1140 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
1141 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */
1142 outBuff.dst = decodedBuffer;
1143 outBuff.size = CNBufferSize;
1144 outBuff.pos = 0;
1145 inBuff.src = compressedBuffer;
1146 inBuff.size = cSize;
1147 inBuff.pos = 0;
1148 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1149 if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
1150 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
1151 ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */
1152
1153 DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
1154 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1155 int const maxLevel = 16; /* first level with zstd_opt */
1156 int level;
1157 assert(maxLevel < ZSTD_maxCLevel());
1158 CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) );
1159 for (level = 1; level <= maxLevel; ++level) {
1160 ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level);
1161 size_t const maxSize = MIN(1 MB, CNBufferSize);
1162 size_t size;
1163 for (size = 512; size <= maxSize; size <<= 1) {
1164 U64 const crcOrig = XXH64(CNBuffer, size, 0);
1165 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1166 ZSTD_parameters savedParams;
1167 getCCtxParams(cctx, &savedParams);
1168 outBuff.dst = compressedBuffer;
1169 outBuff.size = compressedBufferSize;
1170 outBuff.pos = 0;
1171 inBuff.src = CNBuffer;
1172 inBuff.size = size;
1173 inBuff.pos = 0;
1174 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
1175 CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
1176 CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
1177 if (inBuff.pos != inBuff.size) goto _output_error;
1178 { ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
1179 ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
1180 CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
1181 if (decIn.pos != decIn.size) goto _output_error;
1182 if (decOut.pos != size) goto _output_error;
1183 { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
1184 if (crcDec != crcOrig) goto _output_error;
1185 } }
1186 ZSTD_freeCCtx(cctx);
1187 }
1188 ZSTD_freeCDict(cdict);
1189 }
1190 ZSTD_freeDCtx(dctx);
1191 }
1192 DISPLAYLEVEL(3, "OK\n");
1193
1194 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1195 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) );
1196 cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB));
1197 CHECK_Z(cSize);
1198 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++);
1199 {
1200 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1201 /* We should fail to decompress without a dictionary. */
1202 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1203 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1204 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1205 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1206 if (!ZSTD_isError(ret)) goto _output_error;
1207 }
1208 /* We should succeed to decompress with the dictionary. */
1209 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1210 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1211 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1212 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1213 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1214 if (in.pos != in.size) goto _output_error;
1215 }
1216 /* The dictionary should persist across calls. */
1217 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1218 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1219 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1220 if (in.pos != in.size) goto _output_error;
1221 }
1222 /* The dictionary should not be cleared by ZSTD_reset_session_only. */
1223 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
1224 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1225 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1226 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1227 if (in.pos != in.size) goto _output_error;
1228 }
1229 /* When we reset the context the dictionary is cleared. */
1230 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1231 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1232 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1233 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1234 if (!ZSTD_isError(ret)) goto _output_error;
1235 }
1236 ZSTD_freeDCtx(dctx);
1237 }
1238 DISPLAYLEVEL(3, "OK \n");
1239
1240 DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
1241 {
1242 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1243 /* We should succeed to decompress with the dictionary. */
1244 ZSTD_resetDStream(dctx);
1245 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1246 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1247 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1248 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1249 if (in.pos != in.size) goto _output_error;
1250 }
1251 /* The dictionary should not be cleared by ZSTD_resetDStream(). */
1252 ZSTD_resetDStream(dctx);
1253 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1254 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1255 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1256 if (in.pos != in.size) goto _output_error;
1257 }
1258 /* The dictionary should be cleared by ZSTD_initDStream(). */
1259 CHECK_Z( ZSTD_initDStream(dctx) );
1260 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1261 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1262 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1263 if (!ZSTD_isError(ret)) goto _output_error;
1264 }
1265 ZSTD_freeDCtx(dctx);
1266 }
1267 DISPLAYLEVEL(3, "OK \n");
1268
1269 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++);
1270 {
1271 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1272 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1273 /* We should succeed to decompress with the ddict. */
1274 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1275 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
1276 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1277 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1278 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1279 if (in.pos != in.size) goto _output_error;
1280 }
1281 /* The ddict should persist across calls. */
1282 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1283 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1284 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1285 if (in.pos != in.size) goto _output_error;
1286 }
1287 /* When we reset the context the ddict is cleared. */
1288 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1289 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1290 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1291 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1292 if (!ZSTD_isError(ret)) goto _output_error;
1293 }
1294 ZSTD_freeDCtx(dctx);
1295 ZSTD_freeDDict(ddict);
1296 }
1297 DISPLAYLEVEL(3, "OK \n");
1298
1299 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
1300 {
1301 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1302 /* We should succeed to decompress with the prefix. */
1303 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1304 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) );
1305 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1306 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1307 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1308 if (in.pos != in.size) goto _output_error;
1309 }
1310 /* The prefix should be cleared after the first compression. */
1311 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1312 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1313 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1314 if (!ZSTD_isError(ret)) goto _output_error;
1315 }
1316 ZSTD_freeDCtx(dctx);
1317 }
1318 DISPLAYLEVEL(3, "OK \n");
1319
1320 DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++);
1321 {
1322 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1323 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1324 size_t ret;
1325 /* We should succeed to decompress with the dictionary. */
1326 CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) );
1327 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1328 /* The dictionary should persist across calls. */
1329 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1330 /* We should succeed to decompress with the ddict. */
1331 CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) );
1332 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1333 /* The ddict should persist across calls. */
1334 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1335 /* When we reset the context the ddict is cleared. */
1336 CHECK_Z( ZSTD_initDStream(dctx) );
1337 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize);
1338 if (!ZSTD_isError(ret)) goto _output_error;
1339 ZSTD_freeDCtx(dctx);
1340 ZSTD_freeDDict(ddict);
1341 }
1342 DISPLAYLEVEL(3, "OK \n");
1343
1344 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
1345 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
1346 ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
1347 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
1348 size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
1349 if (ZSTD_isError(initError)) goto _output_error;
1350 outBuff.dst = compressedBuffer;
1351 outBuff.size = compressedBufferSize;
1352 outBuff.pos = 0;
1353 inBuff.src = CNBuffer;
1354 inBuff.size = CNBufferSize;
1355 inBuff.pos = 0;
1356 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1357 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1358 { size_t const r = ZSTD_endStream(zc, &outBuff);
1359 if (r != 0) goto _output_error; } /* error, or some data not flushed */
1360 cSize = outBuff.pos;
1361 ZSTD_freeCDict(cdict);
1362 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1363 }
1364
1365 DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
1366 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1367 if (did != 0) goto _output_error;
1368 }
1369 DISPLAYLEVEL(3, "OK (not detected) \n");
1370
1371 DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
1372 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1373 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1374 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1375 }
1376
1377 DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
1378 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) );
1379 outBuff.dst = compressedBuffer;
1380 outBuff.size = compressedBufferSize;
1381 outBuff.pos = 0;
1382 inBuff.src = CNBuffer;
1383 inBuff.size = CNBufferSize;
1384 inBuff.pos = 0;
1385 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1386 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1387 cSize = outBuff.pos;
1388 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1389
1390 DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++);
1391 CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) );
1392 outBuff.dst = decodedBuffer;
1393 outBuff.size = CNBufferSize;
1394 outBuff.pos = 0;
1395 inBuff.src = compressedBuffer;
1396 inBuff.size = cSize;
1397 inBuff.pos = 0;
1398 CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
1399 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1400 if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
1401 DISPLAYLEVEL(3, "OK \n");
1402
1403 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
1404 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1405 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1406 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1407 }
1408
1409 DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
1410 outBuff.dst = compressedBuffer;
1411 outBuff.size = compressedBufferSize;
1412 outBuff.pos = 0;
1413 inBuff.src = CNBuffer;
1414 inBuff.size = CNBufferSize;
1415 inBuff.pos = 0;
1416 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1417 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1418 cSize = outBuff.pos;
1419 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1420
1421 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
1422 CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) );
1423 DISPLAYLEVEL(3, "OK \n");
1424
1425 /* Empty srcSize */
1426 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
1427 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1428 params.fParams.contentSizeFlag = 1;
1429 CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
1430 } /* cstream advanced shall write content size = 0 */
1431 outBuff.dst = compressedBuffer;
1432 outBuff.size = compressedBufferSize;
1433 outBuff.pos = 0;
1434 inBuff.src = CNBuffer;
1435 inBuff.size = 0;
1436 inBuff.pos = 0;
1437 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1438 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1439 cSize = outBuff.pos;
1440 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1441 DISPLAYLEVEL(3, "OK \n");
1442
1443 DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++);
1444 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1445 params.fParams.contentSizeFlag = 1;
1446 CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
1447 } /* cstream advanced shall write content size = 0 */
1448 inBuff.src = CNBuffer;
1449 inBuff.size = 0;
1450 inBuff.pos = 0;
1451 outBuff.dst = compressedBuffer;
1452 outBuff.size = compressedBufferSize;
1453 outBuff.pos = 0;
1454 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1455 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1456 cSize = outBuff.pos;
1457 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1458
1459 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1460 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1461 outBuff.dst = compressedBuffer;
1462 outBuff.size = compressedBufferSize;
1463 outBuff.pos = 0;
1464 inBuff.src = CNBuffer;
1465 inBuff.size = 0;
1466 inBuff.pos = 0;
1467 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1468 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1469 cSize = outBuff.pos;
1470 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
1471 DISPLAYLEVEL(3, "OK \n");
1472
1473 /* Basic multithreading compression test */
1474 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
1475 { int jobSize;
1476 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1477 CHECK(jobSize != 0, "job size non-zero");
1478 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1479 CHECK(jobSize != 0, "job size non-zero");
1480 }
1481 outBuff.dst = compressedBuffer;
1482 outBuff.size = compressedBufferSize;
1483 outBuff.pos = 0;
1484 inBuff.src = CNBuffer;
1485 inBuff.size = CNBufferSize;
1486 inBuff.pos = 0;
1487 { size_t const compressResult = ZSTD_compressStream2(mtctx, &outBuff, &inBuff, ZSTD_e_end);
1488 if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */
1489 }
1490 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1491 { size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos);
1492 if (compressedSize != outBuff.pos) goto _output_error; /* must be a full valid frame */
1493 }
1494 DISPLAYLEVEL(3, "OK \n");
1495
1496 /* Complex multithreading + dictionary test */
1497 { U32 const nbWorkers = 2;
1498 size_t const jobSize = 4 * 1 MB;
1499 size_t const srcSize = jobSize * nbWorkers; /* we want each job to have predictable size */
1500 size_t const segLength = 2 KB;
1501 size_t const offset = 600 KB; /* must be larger than window defined in cdict */
1502 size_t const start = jobSize + (offset-1);
1503 const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
1504 BYTE* const dst = (BYTE*)CNBuffer + start - offset;
1505 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize);
1506 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
1507 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
1508 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, (int)jobSize) );
1509 assert(start > offset);
1510 assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
1511 memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
1512 outBuff.dst = compressedBuffer;
1513 outBuff.size = compressedBufferSize;
1514 outBuff.pos = 0;
1515 inBuff.src = CNBuffer;
1516 inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
1517 inBuff.pos = 0;
1518 }
1519 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
1520 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1521 DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
1522 CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
1523 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1524 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
1525 ZSTD_freeCDict(cdict);
1526 }
1527 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1528 cSize = outBuff.pos;
1529 DISPLAYLEVEL(3, "OK \n");
1530
1531 DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
1532 { ZSTD_DStream* const dstream = ZSTD_createDCtx();
1533 ZSTD_FrameHeader zfh;
1534 ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
1535 DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize);
1536 outBuff.dst = decodedBuffer;
1537 outBuff.size = CNBufferSize;
1538 outBuff.pos = 0;
1539 inBuff.src = compressedBuffer;
1540 inBuff.pos = 0;
1541 CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
1542 inBuff.size = 1; /* avoid shortcut to single-pass mode */
1543 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1544 inBuff.size = cSize;
1545 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1546 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1547 ZSTD_freeDStream(dstream);
1548 }
1549 DISPLAYLEVEL(3, "OK \n");
1550
1551 DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
1552 { unsigned const kMaxWindowLog = 24;
1553 unsigned value;
1554 ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1ULL << kMaxWindowLog, 1024);
1555 ZSTD_CDict* cdict;
1556 ZSTD_DDict* ddict;
1557 SEQ_stream seq = SEQ_initStream(0x87654321);
1558 SEQ_gen_type type;
1559 XXH64_state_t xxh;
1560
1561 XXH64_reset(&xxh, 0);
1562 cParams.windowLog = kMaxWindowLog;
1563 cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1564 ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1565
1566 if (!cdict || !ddict) goto _output_error;
1567
1568 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1569 ZSTD_resetDStream(zd);
1570 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1571 CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
1572 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
1573 /* Test all values < 300 */
1574 for (value = 0; value < 300; ++value) {
1575 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1576 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1577 }
1578 }
1579 /* Test values 2^8 to 2^17 */
1580 for (value = (1 << 8); value < (1 << 17); value <<= 1) {
1581 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1582 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1583 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
1584 }
1585 }
1586 /* Test offset values up to the max window log */
1587 for (value = 8; value <= kMaxWindowLog; ++value) {
1588 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
1589 }
1590
1591 CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
1592 CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
1593
1594 ZSTD_freeCDict(cdict);
1595 ZSTD_freeDDict(ddict);
1596 }
1597 DISPLAYLEVEL(3, "OK \n");
1598
1599 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
1600 { int level;
1601 CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
1602 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1603 CHECK(level != 11, "Compression level does not match");
1604 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1605 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1606 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1607 CHECK(level != 11, "Compression level does not match");
1608 }
1609 DISPLAYLEVEL(3, "OK \n");
1610
1611 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++);
1612 { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0);
1613 CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN));
1614 CHECK(badParameters(zc, params), "Compression parameters do not match");
1615 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1616 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1617 CHECK(badParameters(zc, params), "Compression parameters do not match");
1618 }
1619 DISPLAYLEVEL(3, "OK \n");
1620
1621 DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++);
1622 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1623 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX));
1624 { int srcSizeHint;
1625 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint));
1626 CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match");
1627 }
1628 CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
1629 DISPLAYLEVEL(3, "OK \n");
1630
1631 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy compress with hashLog = 29 and searchLog = 4 : ", testNb++);
1632 if (MEM_64bits()) {
1633 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
1634 ZSTD_inBuffer in = { CNBuffer, CNBufferSize, 0 };
1635 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1636 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_strategy, ZSTD_lazy));
1637 /* Force enable the row based match finder */
1638 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
1639 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_searchLog, 4));
1640 /* Set windowLog to 29 so the hashLog doesn't get sized down */
1641 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, 29));
1642 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_hashLog, 29));
1643 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1644 /* Compress with continue first so the hashLog doesn't get sized down */
1645 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_continue));
1646 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_end));
1647 cSize = out.pos;
1648 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
1649 }
1650 DISPLAYLEVEL(3, "OK \n");
1651
1652 DISPLAYLEVEL(3, "test%3i : Test offset == windowSize : ", testNb++);
1653 {
1654 int windowLog;
1655 int const kMaxWindowLog = bigTests ? 29 : 26;
1656 size_t const kNbSequences = 10000;
1657 size_t const kMaxSrcSize = ((size_t)1 << kMaxWindowLog) + 10 * kNbSequences;
1658 char* src = calloc(kMaxSrcSize, 1);
1659 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
1660 for (windowLog = ZSTD_WINDOWLOG_MIN; windowLog <= kMaxWindowLog; ++windowLog) {
1661 size_t const srcSize = ((size_t)1 << windowLog) + 10 * (kNbSequences - 1);
1662
1663 sequences[0].offset = 32;
1664 sequences[0].litLength = 32;
1665 sequences[0].matchLength = (1u << windowLog) - 32;
1666 sequences[0].rep = 0;
1667 {
1668 size_t i;
1669 for (i = 1; i < kNbSequences; ++i) {
1670 sequences[i].offset = (1u << windowLog) - (FUZ_rand(&seed) % 8);
1671 sequences[i].litLength = FUZ_rand(&seed) & 7;
1672 sequences[i].matchLength = 10 - sequences[i].litLength;
1673 sequences[i].rep = 0;
1674 }
1675 }
1676
1677 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1678 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1679 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_minMatch, 3));
1680 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, 1));
1681 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, windowLog));
1682 assert(srcSize <= kMaxSrcSize);
1683 cSize = ZSTD_compressSequences(zc, compressedBuffer, compressedBufferSize, sequences, kNbSequences, src, srcSize);
1684 CHECK_Z(cSize);
1685 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
1686 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, windowLog))
1687 {
1688 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1689 size_t decompressedBytes = 0;
1690 for (;;) {
1691 ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1692 size_t const ret = ZSTD_decompressStream(zd, &out, &in);
1693 CHECK_Z(ret);
1694 CHECK(decompressedBytes + out.pos > srcSize, "Output too large");
1695 CHECK(memcmp(out.dst, src + decompressedBytes, out.pos), "Corrupted");
1696 decompressedBytes += out.pos;
1697 if (ret == 0) {
1698 break;
1699 }
1700 }
1701 CHECK(decompressedBytes != srcSize, "Output wrong size");
1702 }
1703 }
1704 free(sequences);
1705 free(src);
1706 }
1707 DISPLAYLEVEL(3, "OK \n");
1708
1709 /* Overlen overwriting window data bug */
1710 DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
1711 { /* This test has a window size of 1024 bytes and consists of 3 blocks:
1712 1. 'a' repeated 517 times
1713 2. 'b' repeated 516 times
1714 3. a compressed block with no literals and 3 sequence commands:
1715 litlength = 0, offset = 24, match length = 24
1716 litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
1717 litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
1718
1719 const char* testCase =
1720 "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
1721 "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
1722 "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
1723 ZSTD_DStream* const zds = ZSTD_createDStream();
1724 if (zds==NULL) goto _output_error;
1725
1726 CHECK_Z( ZSTD_initDStream(zds) );
1727 inBuff.src = testCase;
1728 inBuff.size = 47;
1729 inBuff.pos = 0;
1730 outBuff.dst = decodedBuffer;
1731 outBuff.size = CNBufferSize;
1732 outBuff.pos = 0;
1733
1734 while (inBuff.pos < inBuff.size) {
1735 CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) );
1736 }
1737
1738 ZSTD_freeDStream(zds);
1739 }
1740 DISPLAYLEVEL(3, "OK \n");
1741
1742 /* Small Sequence Section bug */
1743 DISPLAYLEVEL(3, "test%3i : decompress blocks with small sequences section : ", testNb++);
1744 { /* This test consists of 3 blocks. Each block has one sequence.
1745 The sequence has literal length of 10, match length of 10 and offset of 10.
1746 The sequence value and compression mode for the blocks are following:
1747 The order of values are ll, ml, of.
1748 - First block : (10, 7, 13) (rle, rle, rle)
1749 - size of sequences section: 6 bytes (1 byte for nbSeq, 1 byte for encoding mode, 3 bytes for rle, 1 byte bitstream)
1750 - Second block : (10, 7, 1) (repeat, repeat, rle)
1751 - size of sequences section: 4 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 bytes for rle, 1 byte bitstream)
1752 - Third block : (10, 7, 1) (repeat, repeat, repeat)
1753 - size of sequences section: 3 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 byte bitstream) */
1754
1755 unsigned char compressed[] = {
1756 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x3c, 0x35, 0x01, 0x00, 0xf0, 0x85, 0x08,
1757 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1758 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac,
1759 0x69, 0x94, 0x89, 0x1c, 0x03, 0x44, 0x0a, 0x07, 0x00, 0xb4, 0x04, 0x80,
1760 0x40, 0x0a, 0xa4
1761 };
1762 unsigned int compressedSize = 51;
1763 unsigned char decompressed[] = {
1764 0x85, 0x08, 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x85, 0x08,
1765 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1766 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0x4c, 0x6b, 0xa9, 0x8b, 0xbc, 0xc5,
1767 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94,
1768 0x89, 0x1c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, 0x89, 0x1c
1769 };
1770 unsigned int decompressedSize = 60;
1771
1772 ZSTD_DStream* const zds = ZSTD_createDStream();
1773 if (zds==NULL) goto _output_error;
1774
1775 CHECK_Z( ZSTD_initDStream(zds) );
1776 inBuff.src = compressed;
1777 inBuff.size = compressedSize;
1778 inBuff.pos = 0;
1779 outBuff.dst = decodedBuffer;
1780 outBuff.size = CNBufferSize;
1781 outBuff.pos = 0;
1782
1783 CHECK(ZSTD_decompressStream(zds, &outBuff, &inBuff) != 0,
1784 "Decompress did not reach the end of frame");
1785 CHECK(inBuff.pos != inBuff.size, "Decompress did not fully consume input");
1786 CHECK(outBuff.pos != decompressedSize, "Decompressed size does not match");
1787 CHECK(memcmp(outBuff.dst, decompressed, decompressedSize) != 0,
1788 "Decompressed data does not match");
1789
1790 ZSTD_freeDStream(zds);
1791 }
1792 DISPLAYLEVEL(3, "OK \n");
1793
1794 DISPLAYLEVEL(3, "test%3i : raw block can be streamed: ", testNb++);
1795 { size_t const inputSize = 10000;
1796 size_t const compCapacity = ZSTD_compressBound(inputSize);
1797 BYTE* const input = (BYTE*)malloc(inputSize);
1798 BYTE* const comp = (BYTE*)malloc(compCapacity);
1799 BYTE* const decomp = (BYTE*)malloc(inputSize);
1800
1801 CHECK(input == NULL || comp == NULL || decomp == NULL, "failed to alloc buffers");
1802
1803 RDG_genBuffer(input, inputSize, 0.0, 0.0, seed);
1804 { size_t const compSize = ZSTD_compress(comp, compCapacity, input, inputSize, -(int)inputSize);
1805 ZSTD_inBuffer in = { comp, 0, 0 };
1806 ZSTD_outBuffer out = { decomp, 0, 0 };
1807 CHECK_Z(compSize);
1808 CHECK_Z( ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters) );
1809 while (in.size < compSize) {
1810 in.size = MIN(in.size + 100, compSize);
1811 while (in.pos < in.size) {
1812 size_t const outPos = out.pos;
1813 if (out.pos == out.size) {
1814 out.size = MIN(out.size + 10, inputSize);
1815 }
1816 CHECK_Z( ZSTD_decompressStream(zd, &out, &in) );
1817 CHECK(!(out.pos > outPos), "We are not streaming (no output generated)");
1818 }
1819 }
1820 CHECK(in.pos != compSize, "Not all input consumed!");
1821 CHECK(out.pos != inputSize, "Not all output produced!");
1822 }
1823 CHECK(memcmp(input, decomp, inputSize), "round trip failed!");
1824
1825 free(input);
1826 free(comp);
1827 free(decomp);
1828 }
1829 DISPLAYLEVEL(3, "OK \n");
1830
1831 DISPLAYLEVEL(3, "test%3i : dictionary + uncompressible block + reusing tables checks offset table validity: ", testNb++);
1832 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1833 dictionary.start, dictionary.filled,
1834 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1835 ZSTD_getCParams(3, 0, dictionary.filled),
1836 ZSTD_defaultCMem);
1837 const size_t inbufsize = 2 * 128 * 1024; /* 2 blocks */
1838 const size_t outbufsize = ZSTD_compressBound(inbufsize);
1839 size_t inbufpos = 0;
1840 size_t cursegmentlen;
1841 BYTE *inbuf = (BYTE *)malloc(inbufsize);
1842 BYTE *outbuf = (BYTE *)malloc(outbufsize);
1843 BYTE *checkbuf = (BYTE *)malloc(inbufsize);
1844 size_t ret;
1845
1846 CHECK(cdict == NULL, "failed to alloc cdict");
1847 CHECK(inbuf == NULL, "failed to alloc input buffer");
1848
1849 /* first block is uncompressible */
1850 cursegmentlen = 128 * 1024;
1851 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0., 0., seed);
1852 inbufpos += cursegmentlen;
1853
1854 /* second block is compressible */
1855 cursegmentlen = 128 * 1024 - 256;
1856 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0.05, 0., seed);
1857 inbufpos += cursegmentlen;
1858
1859 /* and includes a very long backref */
1860 cursegmentlen = 128;
1861 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 256, cursegmentlen);
1862 inbufpos += cursegmentlen;
1863
1864 /* and includes a very long backref */
1865 cursegmentlen = 128;
1866 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 128, cursegmentlen);
1867 inbufpos += cursegmentlen;
1868
1869 ret = ZSTD_compress_usingCDict(zc, outbuf, outbufsize, inbuf, inbufpos, cdict);
1870 CHECK_Z(ret);
1871
1872 ret = ZSTD_decompress_usingDict(zd, checkbuf, inbufsize, outbuf, ret, dictionary.start, dictionary.filled);
1873 CHECK_Z(ret);
1874
1875 CHECK(memcmp(inbuf, checkbuf, inbufpos), "start and finish buffers don't match");
1876
1877 ZSTD_freeCDict(cdict);
1878 free(inbuf);
1879 free(outbuf);
1880 free(checkbuf);
1881 }
1882 DISPLAYLEVEL(3, "OK \n");
1883
1884 DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++);
1885 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1886 dictionary.start, dictionary.filled,
1887 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1888 ZSTD_getCParams(3, 0, dictionary.filled),
1889 ZSTD_defaultCMem);
1890 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1891 int remainingInput = 256 * 1024;
1892 int offset;
1893
1894 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1895 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1896 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1897 /* Write a bunch of 6 byte blocks */
1898 while (remainingInput > 0) {
1899 char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
1900 const size_t kSmallBlockSize = sizeof(testBuffer);
1901 ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
1902
1903 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
1904 CHECK(in.pos != in.size, "input not fully consumed");
1905 remainingInput -= (int)kSmallBlockSize;
1906 }
1907 /* Write several very long offset matches into the dictionary */
1908 for (offset = 1024; offset >= 0; offset -= 128) {
1909 ZSTD_inBuffer in = {(BYTE*)dictionary.start + offset, 128, 0};
1910 ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
1911 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
1912 CHECK(in.pos != in.size, "input not fully consumed");
1913 }
1914 /* Ensure decompression works */
1915 CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled));
1916
1917 ZSTD_freeCDict(cdict);
1918 }
1919 DISPLAYLEVEL(3, "OK \n");
1920
1921 DISPLAYLEVEL(3, "test%3i : Block-Level External Sequence Producer API: ", testNb++);
1922 {
1923 size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
1924 BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
1925 size_t const checkBufSize = CNBufferSize;
1926 BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
1927 int enableFallback;
1928 EMF_testCase sequenceProducerState;
1929
1930 CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
1931
1932 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1933
1934 /* Reference external matchfinder outside the test loop to
1935 * check that the reference is preserved across compressions */
1936 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1937
1938 for (enableFallback = 0; enableFallback <= 1; enableFallback++) {
1939 size_t testCaseId;
1940 size_t const numTestCases = 9;
1941
1942 EMF_testCase const testCases[] = {
1943 EMF_ONE_BIG_SEQ,
1944 EMF_LOTS_OF_SEQS,
1945 EMF_ZERO_SEQS,
1946 EMF_BIG_ERROR,
1947 EMF_SMALL_ERROR,
1948 EMF_INVALID_OFFSET,
1949 EMF_INVALID_MATCHLEN,
1950 EMF_INVALID_LITLEN,
1951 EMF_INVALID_LAST_LITS
1952 };
1953
1954 ZSTD_ErrorCode const errorCodes[] = {
1955 ZSTD_error_no_error,
1956 ZSTD_error_no_error,
1957 ZSTD_error_sequenceProducer_failed,
1958 ZSTD_error_sequenceProducer_failed,
1959 ZSTD_error_sequenceProducer_failed,
1960 ZSTD_error_externalSequences_invalid,
1961 ZSTD_error_externalSequences_invalid,
1962 ZSTD_error_externalSequences_invalid,
1963 ZSTD_error_externalSequences_invalid
1964 };
1965
1966 for (testCaseId = 0; testCaseId < numTestCases; testCaseId++) {
1967 size_t res;
1968
1969 int const compressionShouldSucceed = (
1970 (errorCodes[testCaseId] == ZSTD_error_no_error) ||
1971 (enableFallback && errorCodes[testCaseId] == ZSTD_error_sequenceProducer_failed)
1972 );
1973
1974 int const testWithSequenceValidation = (
1975 testCases[testCaseId] == EMF_INVALID_OFFSET
1976 );
1977
1978 sequenceProducerState = testCases[testCaseId];
1979
1980 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1981 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, testWithSequenceValidation));
1982 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
1983 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
1984
1985 if (compressionShouldSucceed) {
1986 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
1987 CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
1988 CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
1989 } else {
1990 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
1991 CHECK(
1992 ZSTD_getErrorCode(res) != errorCodes[testCaseId],
1993 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
1994 );
1995 }
1996 }
1997
1998 /* Test compression with external matchfinder + empty src buffer */
1999 {
2000 size_t res;
2001 sequenceProducerState = EMF_ZERO_SEQS;
2002 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
2003 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
2004 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
2005 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
2006 CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
2007 }
2008 }
2009
2010 /* Test that reset clears the external matchfinder */
2011 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
2012 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
2013 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
2014 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
2015
2016 /* Test that registering mFinder == NULL clears the external matchfinder */
2017 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
2018 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2019 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
2020 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
2021 ZSTD_registerSequenceProducer(zc, NULL, NULL); /* clear the external matchfinder */
2022 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
2023
2024 /* Test that external matchfinder doesn't interact with older APIs */
2025 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
2026 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2027 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
2028 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
2029 CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
2030
2031 /* Test that compression returns the correct error with LDM */
2032 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
2033 {
2034 size_t res;
2035 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2036 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
2037 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2038 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
2039 CHECK(
2040 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
2041 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
2042 );
2043 }
2044
2045 #ifdef ZSTD_MULTITHREAD
2046 /* Test that compression returns the correct error with nbWorkers > 0 */
2047 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
2048 {
2049 size_t res;
2050 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2051 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
2052 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2053 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
2054 CHECK(
2055 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
2056 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
2057 );
2058 }
2059 #endif
2060
2061 free(dstBuf);
2062 free(checkBuf);
2063 }
2064 DISPLAYLEVEL(3, "OK \n");
2065
2066
2067 /* Test maxBlockSize cctx param functionality */
2068 DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
2069 {
2070 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2071
2072 /* Quick test to make sure maxBlockSize bounds are enforced */
2073 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
2074 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
2075
2076 /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
2077 {
2078 size_t srcSize = 2 << 10;
2079 void* const src = CNBuffer;
2080 size_t dstSize = ZSTD_compressBound(srcSize);
2081 void* const dst1 = compressedBuffer;
2082 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2083 size_t size1, size2;
2084 void* const checkBuf = malloc(srcSize);
2085 memset(src, 'x', srcSize);
2086
2087 /* maxBlockSize = 1KB */
2088 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2089 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2090
2091 if (ZSTD_isError(size1)) goto _output_error;
2092 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2093 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2094
2095 /* maxBlockSize = 3KB */
2096 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2097 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2098
2099 if (ZSTD_isError(size2)) goto _output_error;
2100 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2101 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2102
2103 assert(size1 - size2 == 4); /* We add another RLE block with header + character */
2104 assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
2105
2106 /* maxBlockSize = 1KB, windowLog = 10 */
2107 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2108 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2109 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2110
2111 if (ZSTD_isError(size1)) goto _output_error;
2112 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2113 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2114
2115 /* maxBlockSize = 3KB, windowLog = 10 */
2116 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2117 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2118 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2119
2120 if (ZSTD_isError(size2)) goto _output_error;
2121 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2122 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2123
2124 assert(size1 == size2);
2125 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2126
2127 free(checkBuf);
2128 }
2129
2130 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2131
2132 /* Test maxBlockSize = 0 is valid */
2133 { size_t srcSize = 256 << 10;
2134 void* const src = CNBuffer;
2135 size_t dstSize = ZSTD_compressBound(srcSize);
2136 void* const dst1 = compressedBuffer;
2137 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2138 size_t size1, size2;
2139 void* const checkBuf = malloc(srcSize);
2140
2141 /* maxBlockSize = 0 */
2142 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
2143 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2144
2145 if (ZSTD_isError(size1)) goto _output_error;
2146 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2147 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2148
2149 /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
2150 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
2151 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2152
2153 if (ZSTD_isError(size2)) goto _output_error;
2154 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2155 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2156
2157 assert(size1 == size2);
2158 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2159 free(checkBuf);
2160 }
2161 ZSTD_freeCCtx(cctx);
2162 }
2163 DISPLAYLEVEL(3, "OK \n");
2164
2165 /* Test Sequence Validation */
2166 DISPLAYLEVEL(3, "test%3i : Testing sequence validation: ", testNb++);
2167 {
2168 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2169
2170 /* Test minMatch >= 4, matchLength < 4 */
2171 {
2172 size_t srcSize = 11;
2173 void* const src = CNBuffer;
2174 size_t dstSize = ZSTD_compressBound(srcSize);
2175 void* const dst = compressedBuffer;
2176 size_t const kNbSequences = 4;
2177 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2178
2179 memset(src, 'x', srcSize);
2180
2181 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2182 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2183 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2184 sequences[3] = (ZSTD_Sequence) {0, 1, 0, 0};
2185
2186 /* Test with sequence validation */
2187 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2188 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2189 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2190
2191 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2192 sequences, kNbSequences,
2193 src, srcSize);
2194
2195 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2196 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2197
2198 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2199
2200 /* Test without sequence validation */
2201 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2202 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2203 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2204
2205 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2206 sequences, kNbSequences,
2207 src, srcSize);
2208
2209 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2210 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2211
2212 free(sequences);
2213 }
2214
2215 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2216
2217
2218 /* Test with no block delim */
2219 {
2220 size_t srcSize = 4;
2221 void* const src = CNBuffer;
2222 size_t dstSize = ZSTD_compressBound(srcSize);
2223 void* const dst = compressedBuffer;
2224 size_t const kNbSequences = 1;
2225 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2226 void* const checkBuf = malloc(srcSize);
2227
2228 memset(src, 'x', srcSize);
2229
2230 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2231
2232 /* Test with sequence validation */
2233 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 3));
2234 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters));
2235 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2236
2237 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2238 sequences, kNbSequences,
2239 src, srcSize);
2240
2241 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2242 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst, cSize));
2243 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2244
2245 free(sequences);
2246 free(checkBuf);
2247 }
2248
2249 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2250
2251 { /* Test case with two additional sequences */
2252 size_t srcSize = 19;
2253 void* const src = CNBuffer;
2254 size_t dstSize = ZSTD_compressBound(srcSize);
2255 void* const dst = compressedBuffer;
2256 size_t const kNbSequences = 7;
2257 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2258
2259 memset(src, 'x', srcSize);
2260
2261 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2262 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2263 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2264 sequences[3] = (ZSTD_Sequence) {1, 0, 3, 0};
2265 sequences[4] = (ZSTD_Sequence) {1, 0, 3, 0};
2266 sequences[5] = (ZSTD_Sequence) {1, 0, 3, 0};
2267 sequences[6] = (ZSTD_Sequence) {0, 0, 0, 0};
2268
2269 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2270 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2271 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2272
2273 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2274 sequences, kNbSequences,
2275 src, srcSize);
2276
2277 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2278 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2279
2280 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2281
2282 /* Test without sequence validation */
2283 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2284 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2285 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2286
2287 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2288 sequences, kNbSequences,
2289 src, srcSize);
2290
2291 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2292 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2293
2294 free(sequences);
2295 }
2296 ZSTD_freeCCtx(cctx);
2297 }
2298 DISPLAYLEVEL(3, "OK \n");
2299
2300
2301 DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
2302 {
2303 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2304 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2305
2306 /* Test large offset, small window size*/
2307 {
2308 size_t srcSize = 21;
2309 void* const src = CNBuffer;
2310 size_t dstSize = ZSTD_compressBound(srcSize);
2311 void* const dst = compressedBuffer;
2312 size_t const kNbSequences = 4;
2313 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2314 void* const checkBuf = malloc(srcSize);
2315 const size_t largeDictSize = 1 << 25;
2316 ZSTD_CDict* cdict = NULL;
2317 ZSTD_DDict* ddict = NULL;
2318
2319 /* Generate large dictionary */
2320 void* dictBuffer = calloc(largeDictSize, 1);
2321 ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, largeDictSize);
2322 cParams.minMatch = ZSTD_MINMATCH_MIN;
2323 cParams.hashLog = ZSTD_HASHLOG_MIN;
2324 cParams.chainLog = ZSTD_CHAINLOG_MIN;
2325
2326 cdict = ZSTD_createCDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
2327 ddict = ZSTD_createDDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
2328
2329 ZSTD_CCtx_refCDict(cctx, cdict);
2330 ZSTD_DCtx_refDDict(dctx, ddict);
2331
2332 sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0};
2333 sequences[1] = (ZSTD_Sequence) {1 << 25, 0, 3, 0};
2334 sequences[2] = (ZSTD_Sequence) {1 << 25, 0, 9, 0};
2335 sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0};
2336
2337 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2338 sequences, kNbSequences,
2339 src, srcSize);
2340
2341 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2342
2343 {
2344 size_t dSize = ZSTD_decompressDCtx(dctx, checkBuf, srcSize, dst, cSize);
2345 CHECK(ZSTD_isError(dSize), "Should not throw an error");
2346 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2347 }
2348
2349 free(sequences);
2350 free(checkBuf);
2351 free(dictBuffer);
2352 ZSTD_freeCDict(cdict);
2353 ZSTD_freeDDict(ddict);
2354 }
2355 ZSTD_freeCCtx(cctx);
2356 ZSTD_freeDCtx(dctx);
2357 }
2358 DISPLAYLEVEL(3, "OK \n");
2359
2360 DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx (one-shot): ", testNb++);
2361 {
2362 size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
2363 BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
2364 size_t const checkBufSize = CNBufferSize;
2365 BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
2366 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2367 ZSTD_CCtx* staticCCtx;
2368 void* cctxBuf;
2369 EMF_testCase seqProdState;
2370
2371 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_validateSequences, 1));
2372 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableSeqProducerFallback, 0));
2373 ZSTD_CCtxParams_registerSequenceProducer(params, &seqProdState, zstreamSequenceProducer);
2374
2375 {
2376 size_t const cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2377 cctxBuf = malloc(cctxSize);
2378 staticCCtx = ZSTD_initStaticCCtx(cctxBuf, cctxSize);
2379 CHECK_Z(ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params));
2380 }
2381
2382 // Check that compression with external sequence producer succeeds when expected
2383 seqProdState = EMF_LOTS_OF_SEQS;
2384 {
2385 size_t dResult;
2386 size_t const cResult = ZSTD_compress2(staticCCtx, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2387 CHECK(ZSTD_isError(cResult), "EMF: Compression error: %s", ZSTD_getErrorName(cResult));
2388 dResult = ZSTD_decompress(checkBuf, checkBufSize, dstBuf, cResult);
2389 CHECK(ZSTD_isError(dResult), "EMF: Decompression error: %s", ZSTD_getErrorName(dResult));
2390 CHECK(dResult != CNBufferSize, "EMF: Corruption!");
2391 CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
2392 }
2393
2394 // Check that compression with external sequence producer fails when expected
2395 seqProdState = EMF_BIG_ERROR;
2396 {
2397 size_t const cResult = ZSTD_compress2(staticCCtx, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2398 CHECK(!ZSTD_isError(cResult), "EMF: Should have raised an error!");
2399 CHECK(
2400 ZSTD_getErrorCode(cResult) != ZSTD_error_sequenceProducer_failed,
2401 "EMF: Wrong error code: %s", ZSTD_getErrorName(cResult)
2402 );
2403 }
2404
2405 free(dstBuf);
2406 free(checkBuf);
2407 free(cctxBuf);
2408 ZSTD_freeCCtxParams(params);
2409 }
2410 DISPLAYLEVEL(3, "OK \n");
2411
2412 DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx (streaming): ", testNb++);
2413 {
2414 size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
2415 BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
2416 size_t const checkBufSize = CNBufferSize;
2417 BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
2418 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2419 ZSTD_CCtx* staticCCtx;
2420 void* cctxBuf;
2421 EMF_testCase seqProdState;
2422
2423 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_validateSequences, 1));
2424 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableSeqProducerFallback, 0));
2425 ZSTD_CCtxParams_registerSequenceProducer(params, &seqProdState, zstreamSequenceProducer);
2426
2427 {
2428 size_t const cctxSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2429 cctxBuf = malloc(cctxSize);
2430 staticCCtx = ZSTD_initStaticCCtx(cctxBuf, cctxSize);
2431 CHECK_Z(ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params));
2432 }
2433
2434 // Check that compression with external sequence producer succeeds when expected
2435 seqProdState = EMF_LOTS_OF_SEQS;
2436 {
2437 ZSTD_inBuffer inBuf = { CNBuffer, CNBufferSize, 0 };
2438 ZSTD_outBuffer outBuf = { dstBuf, dstBufSize, 0 };
2439 size_t dResult;
2440 CHECK_Z(ZSTD_compressStream(staticCCtx, &outBuf, &inBuf));
2441 CHECK_Z(ZSTD_endStream(staticCCtx, &outBuf));
2442 CHECK(inBuf.pos != inBuf.size, "EMF: inBuf.pos != inBuf.size");
2443 dResult = ZSTD_decompress(checkBuf, checkBufSize, outBuf.dst, outBuf.pos);
2444 CHECK(ZSTD_isError(dResult), "EMF: Decompression error: %s", ZSTD_getErrorName(dResult));
2445 CHECK(dResult != CNBufferSize, "EMF: Corruption!");
2446 CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
2447 }
2448
2449 CHECK_Z(ZSTD_CCtx_reset(staticCCtx, ZSTD_reset_session_only));
2450
2451 // Check that compression with external sequence producer fails when expected
2452 seqProdState = EMF_BIG_ERROR;
2453 {
2454 ZSTD_inBuffer inBuf = { CNBuffer, CNBufferSize, 0 };
2455 ZSTD_outBuffer outBuf = { dstBuf, dstBufSize, 0 };
2456 size_t const cResult = ZSTD_compressStream(staticCCtx, &outBuf, &inBuf);
2457 CHECK(!ZSTD_isError(cResult), "EMF: Should have raised an error!");
2458 CHECK(
2459 ZSTD_getErrorCode(cResult) != ZSTD_error_sequenceProducer_failed,
2460 "EMF: Wrong error code: %s", ZSTD_getErrorName(cResult)
2461 );
2462 }
2463
2464 free(dstBuf);
2465 free(checkBuf);
2466 free(cctxBuf);
2467 ZSTD_freeCCtxParams(params);
2468 }
2469 DISPLAYLEVEL(3, "OK \n");
2470
2471 DISPLAYLEVEL(3, "test%3i : Decoder should reject invalid frame header on legacy frames: ", testNb++);
2472 {
2473 const unsigned char compressed[] = { 0x26,0xb5,0x2f,0xfd,0x50,0x91,0xfd,0xd8,0xb5 };
2474 const size_t compressedSize = 9;
2475 size_t const dSize = ZSTD_decompress(NULL, 0, compressed, compressedSize);
2476 CHECK(!ZSTD_isError(dSize), "must reject when legacy frame header is invalid");
2477 }
2478 DISPLAYLEVEL(3, "OK \n");
2479
2480 DISPLAYLEVEL(3, "test%3i : Test single-shot fallback for magicless mode: ", testNb++);
2481 {
2482 // Acquire resources
2483 size_t const srcSize = COMPRESSIBLE_NOISE_LENGTH;
2484 void* src = malloc(srcSize);
2485 size_t const dstSize = ZSTD_compressBound(srcSize);
2486 void* dst = malloc(dstSize);
2487 size_t const valSize = srcSize;
2488 void* val = malloc(valSize);
2489 ZSTD_inBuffer inBuf = { dst, dstSize, 0 };
2490 ZSTD_outBuffer outBuf = { val, valSize, 0 };
2491 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2492 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2493 CHECK(!src || !dst || !val || !dctx || !cctx, "memory allocation failure");
2494
2495 // Write test data for decompression to dst
2496 RDG_genBuffer(src, srcSize, compressibility, 0.0, 0xdeadbeef);
2497 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless));
2498 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
2499
2500 // Run decompression
2501 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless));
2502 CHECK_Z(ZSTD_decompressStream(dctx, &outBuf, &inBuf));
2503
2504 // Validate
2505 CHECK(outBuf.pos != srcSize, "decompressed size must match");
2506 CHECK(memcmp(src, val, srcSize) != 0, "decompressed data must match");
2507
2508 // Cleanup
2509 free(src); free(dst); free(val);
2510 ZSTD_freeCCtx(cctx);
2511 ZSTD_freeDCtx(dctx);
2512 }
2513 DISPLAYLEVEL(3, "OK \n");
2514
2515 _end:
2516 FUZ_freeDictionary(dictionary);
2517 ZSTD_freeCStream(zc);
2518 ZSTD_freeDStream(zd);
2519 ZSTD_freeCCtx(mtctx);
2520 free(CNBuffer);
2521 free(compressedBuffer);
2522 free(decodedBuffer);
2523 return testResult;
2524
2525 _output_error:
2526 testResult = 1;
2527 DISPLAY("Error detected in Unit tests ! \n");
2528 goto _end;
2529 }
2530
2531
2532 /* ====== Fuzzer tests ====== */
2533
findDiff(const void * buf1,const void * buf2,size_t max)2534 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
2535 {
2536 const BYTE* b1 = (const BYTE*)buf1;
2537 const BYTE* b2 = (const BYTE*)buf2;
2538 size_t u;
2539 for (u=0; u<max; u++) {
2540 if (b1[u] != b2[u]) break;
2541 }
2542 if (u==max) {
2543 DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max);
2544 return u;
2545 }
2546 DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max);
2547 if (u>=3)
2548 DISPLAY(" %02X %02X %02X ",
2549 b1[u-3], b1[u-2], b1[u-1]);
2550 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2551 b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]);
2552 if (u>=3)
2553 DISPLAY(" %02X %02X %02X ",
2554 b2[u-3], b2[u-2], b2[u-1]);
2555 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2556 b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]);
2557 return u;
2558 }
2559
FUZ_rLogLength(U32 * seed,U32 logLength)2560 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
2561 {
2562 size_t const lengthMask = ((size_t)1 << logLength) - 1;
2563 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
2564 }
2565
FUZ_randomLength(U32 * seed,U32 maxLog)2566 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
2567 {
2568 U32 const logLength = FUZ_rand(seed) % maxLog;
2569 return FUZ_rLogLength(seed, logLength);
2570 }
2571
2572 /* Return value in range minVal <= v <= maxVal */
FUZ_randomClampedLength(U32 * seed,U32 minVal,U32 maxVal)2573 static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
2574 {
2575 U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal;
2576 return (U32)((FUZ_rand(seed) % mod) + minVal);
2577 }
2578
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,double compressibility,int bigTests)2579 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests)
2580 {
2581 U32 const maxSrcLog = bigTests ? 24 : 22;
2582 static const U32 maxSampleLog = 19;
2583 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2584 BYTE* cNoiseBuffer[5];
2585 size_t const copyBufferSize = srcBufferSize + ((size_t)1 << maxSampleLog);
2586 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2587 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2588 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2589 size_t const dstBufferSize = srcBufferSize;
2590 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2591 int result = 0;
2592 unsigned testNb = 0;
2593 U32 coreSeed = seed;
2594 ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
2595 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
2596 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2597 UTIL_time_t const startClock = UTIL_getTime();
2598 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2599 size_t dictSize = 0;
2600 U32 oldTestLog = 0;
2601 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
2602
2603 /* allocations */
2604 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2605 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2606 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2607 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2608 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2609 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2610 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2611 "Not enough memory, fuzzer tests cancelled");
2612
2613 /* Create initial samples */
2614 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2615 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2616 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2617 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2618 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2619 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2620 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2621
2622 /* catch up testNb */
2623 for (testNb=1; testNb < startTest; testNb++)
2624 FUZ_rand(&coreSeed);
2625
2626 /* test loop */
2627 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2628 U32 lseed;
2629 const BYTE* srcBuffer;
2630 size_t totalTestSize, totalGenSize, cSize;
2631 XXH64_state_t xxhState;
2632 U64 crcOrig;
2633 U32 resetAllowed = 1;
2634 size_t maxTestSize;
2635
2636 /* init */
2637 FUZ_rand(&coreSeed);
2638 lseed = coreSeed ^ prime32;
2639 if (nbTests >= testNb) {
2640 DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
2641 } else {
2642 DISPLAYUPDATE(2, "\r%6u ", testNb);
2643 }
2644
2645 /* states full reset (deliberately not synchronized) */
2646 /* some issues can only happen when reusing states */
2647 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2648 ZSTD_freeCStream(zc);
2649 zc = ZSTD_createCStream();
2650 CHECK(zc==NULL, "ZSTD_createCStream : allocation error");
2651 resetAllowed=0;
2652 }
2653 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2654 ZSTD_freeDStream(zd);
2655 zd = ZSTD_createDStream();
2656 CHECK(zd==NULL, "ZSTD_createDStream : allocation error");
2657 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2658 }
2659
2660 /* srcBuffer selection [0-4] */
2661 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2662 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2663 else {
2664 buffNb >>= 3;
2665 if (buffNb & 7) {
2666 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2667 buffNb = tnb[buffNb >> 3];
2668 } else {
2669 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2670 buffNb = tnb[buffNb >> 3];
2671 } }
2672 srcBuffer = cNoiseBuffer[buffNb];
2673 }
2674
2675 /* compression init */
2676 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2677 && oldTestLog /* at least one test happened */ && resetAllowed) {
2678 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2679 maxTestSize = MIN(maxTestSize, srcBufferSize-16);
2680 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2681 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2682 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2683 }
2684 } else {
2685 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2686 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2687 U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
2688 ((unsigned)ZSTD_maxCLevel() -
2689 (MAX(testLog, dictLog) / 3)))
2690 + 1;
2691 U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
2692 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2693 oldTestLog = testLog;
2694 /* random dictionary selection */
2695 dictSize = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2696 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2697 dict = srcBuffer + dictStart;
2698 }
2699 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2700 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2701 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, (int)cLevel) );
2702 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) );
2703 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) );
2704 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) );
2705 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2706 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2707 } }
2708
2709 /* multi-segments compression test */
2710 XXH64_reset(&xxhState, 0);
2711 { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2712 cSize=0;
2713 totalTestSize=0;
2714 while(totalTestSize < maxTestSize) {
2715 /* compress random chunks into randomly sized dst buffers */
2716 { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2717 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2718 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2719 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2720 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
2721 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2722 outBuff.size = outBuff.pos + dstBuffSize;
2723
2724 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
2725
2726 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2727 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2728 totalTestSize += inBuff.pos;
2729 }
2730
2731 /* random flush operation, to mess around */
2732 if ((FUZ_rand(&lseed) & 15) == 0) {
2733 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2734 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2735 outBuff.size = outBuff.pos + adjustedDstSize;
2736 CHECK_Z( ZSTD_flushStream(zc, &outBuff) );
2737 } }
2738
2739 /* final frame epilogue */
2740 { size_t remainingToFlush = (size_t)(-1);
2741 while (remainingToFlush) {
2742 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2743 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2744 outBuff.size = outBuff.pos + adjustedDstSize;
2745 remainingToFlush = ZSTD_endStream(zc, &outBuff);
2746 CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
2747 } }
2748 crcOrig = XXH64_digest(&xxhState);
2749 cSize = outBuff.pos;
2750 }
2751
2752 /* multi - fragments decompression test */
2753 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2754 CHECK_Z ( ZSTD_resetDStream(zd) );
2755 } else {
2756 CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2757 }
2758 { size_t decompressionResult = 1;
2759 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2760 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2761 for (totalGenSize = 0 ; decompressionResult ; ) {
2762 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2763 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2764 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2765 inBuff.size = inBuff.pos + readCSrcSize;
2766 outBuff.size = outBuff.pos + dstBuffSize;
2767 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2768 if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) {
2769 DISPLAY("checksum error : \n");
2770 findDiff(copyBuffer, dstBuffer, totalTestSize);
2771 }
2772 CHECK( ZSTD_isError(decompressionResult), "decompression error : %s",
2773 ZSTD_getErrorName(decompressionResult) );
2774 }
2775 CHECK (decompressionResult != 0, "frame not fully decoded");
2776 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)",
2777 (unsigned)outBuff.pos, (unsigned)totalTestSize);
2778 CHECK (inBuff.pos != cSize, "compressed data should be fully read")
2779 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
2780 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
2781 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
2782 } }
2783
2784 /*===== noisy/erroneous src decompression test =====*/
2785
2786 /* add some noise */
2787 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
2788 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
2789 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
2790 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
2791 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
2792 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
2793 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
2794 } }
2795
2796 /* try decompression on noisy data */
2797 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
2798 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2799 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2800 while (outBuff.pos < dstBufferSize) {
2801 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2802 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2803 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
2804 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
2805 outBuff.size = outBuff.pos + adjustedDstSize;
2806 inBuff.size = inBuff.pos + adjustedCSrcSize;
2807 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2808 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
2809 /* No forward progress possible */
2810 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
2811 } } } }
2812 DISPLAY("\r%u fuzzer tests completed \n", testNb);
2813
2814 _cleanup:
2815 ZSTD_freeCStream(zc);
2816 ZSTD_freeDStream(zd);
2817 ZSTD_freeDStream(zd_noise);
2818 free(cNoiseBuffer[0]);
2819 free(cNoiseBuffer[1]);
2820 free(cNoiseBuffer[2]);
2821 free(cNoiseBuffer[3]);
2822 free(cNoiseBuffer[4]);
2823 free(copyBuffer);
2824 free(cBuffer);
2825 free(dstBuffer);
2826 return result;
2827
2828 _output_error:
2829 result = 1;
2830 goto _cleanup;
2831 }
2832
2833 /** If useOpaqueAPI, sets param in cctxParams.
2834 * Otherwise, sets the param in zc. */
setCCtxParameter(ZSTD_CCtx * zc,ZSTD_CCtx_params * cctxParams,ZSTD_cParameter param,unsigned value,int useOpaqueAPI)2835 static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
2836 ZSTD_cParameter param, unsigned value,
2837 int useOpaqueAPI)
2838 {
2839 if (useOpaqueAPI) {
2840 return ZSTD_CCtxParams_setParameter(cctxParams, param, (int)value);
2841 } else {
2842 return ZSTD_CCtx_setParameter(zc, param, (int)value);
2843 }
2844 }
2845
2846 /* Tests for ZSTD_compress_generic() API */
fuzzerTests_newAPI(U32 seed,int nbTests,int startTest,double compressibility,int bigTests)2847 static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
2848 double compressibility, int bigTests)
2849 {
2850 U32 const maxSrcLog = bigTests ? 24 : 22;
2851 static const U32 maxSampleLog = 19;
2852 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2853 BYTE* cNoiseBuffer[5];
2854 size_t const copyBufferSize= srcBufferSize + ((size_t)1 << maxSampleLog);
2855 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2856 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2857 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2858 size_t const dstBufferSize = srcBufferSize;
2859 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2860 U32 result = 0;
2861 int testNb = 0;
2862 U32 coreSeed = seed;
2863 ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
2864 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
2865 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2866 UTIL_time_t const startClock = UTIL_getTime();
2867 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2868 size_t dictSize = 0;
2869 U32 oldTestLog = 0;
2870 U32 windowLogMalus = 0; /* can survive between 2 loops */
2871 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
2872 U32 const nbThreadsMax = bigTests ? 4 : 2;
2873 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
2874
2875 /* allocations */
2876 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2877 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2878 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2879 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2880 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2881 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2882 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2883 "Not enough memory, fuzzer tests cancelled");
2884
2885 /* Create initial samples */
2886 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2887 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2888 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2889 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2890 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2891 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2892 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2893
2894 /* catch up testNb */
2895 for (testNb=1; testNb < startTest; testNb++)
2896 FUZ_rand(&coreSeed);
2897
2898 /* test loop */
2899 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2900 U32 lseed;
2901 int opaqueAPI;
2902 const BYTE* srcBuffer;
2903 size_t totalTestSize, totalGenSize, cSize;
2904 XXH64_state_t xxhState;
2905 U64 crcOrig;
2906 U32 resetAllowed = 1;
2907 size_t maxTestSize;
2908 ZSTD_parameters savedParams;
2909 int isRefPrefix = 0;
2910 U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2911
2912 /* init */
2913 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
2914 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
2915 FUZ_rand(&coreSeed);
2916 lseed = coreSeed ^ prime32;
2917 DISPLAYLEVEL(5, " *** Test %u *** \n", testNb);
2918 opaqueAPI = FUZ_rand(&lseed) & 1;
2919
2920 /* states full reset (deliberately not synchronized) */
2921 /* some issues can only happen when reusing states */
2922 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2923 DISPLAYLEVEL(5, "Creating new context \n");
2924 ZSTD_freeCCtx(zc);
2925 zc = ZSTD_createCCtx();
2926 CHECK(zc == NULL, "ZSTD_createCCtx allocation error");
2927 resetAllowed = 0;
2928 }
2929 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2930 ZSTD_freeDStream(zd);
2931 zd = ZSTD_createDStream();
2932 CHECK(zd == NULL, "ZSTD_createDStream allocation error");
2933 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2934 }
2935
2936 /* srcBuffer selection [0-4] */
2937 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2938 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2939 else {
2940 buffNb >>= 3;
2941 if (buffNb & 7) {
2942 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2943 buffNb = tnb[buffNb >> 3];
2944 } else {
2945 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2946 buffNb = tnb[buffNb >> 3];
2947 } }
2948 srcBuffer = cNoiseBuffer[buffNb];
2949 }
2950
2951 /* compression init */
2952 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
2953 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2954 && oldTestLog /* at least one test happened */
2955 && resetAllowed) {
2956 /* just set a compression level */
2957 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2958 if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
2959 { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
2960 DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
2961 CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
2962 }
2963 } else {
2964 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2965 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2966 U32 const cLevelCandidate = (FUZ_rand(&lseed) %
2967 (ZSTD_maxCLevel() -
2968 (MAX(testLog, dictLog) / 2))) +
2969 1;
2970 int const cLevel = MIN(cLevelCandidate, cLevelMax);
2971 DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel);
2972 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2973 DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize);
2974 oldTestLog = testLog;
2975 /* random dictionary selection */
2976 dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2977 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2978 dict = srcBuffer + dictStart;
2979 if (!dictSize) dict=NULL;
2980 }
2981 pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2982 { ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
2983 const U32 windowLogMax = bigTests ? 24 : 20;
2984 const U32 searchLogMax = bigTests ? 15 : 13;
2985 if (dictSize)
2986 DISPLAYLEVEL(5, "t%u: with dictionary of size : %u \n", testNb, (unsigned)dictSize);
2987
2988 /* mess with compression parameters */
2989 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
2990 cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
2991 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
2992 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
2993 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
2994 cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
2995 cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
2996 cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
2997 cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
2998
2999 if (FUZ_rand(&lseed) & 1) {
3000 DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
3001 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
3002 assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
3003 windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
3004 }
3005 if (FUZ_rand(&lseed) & 1) {
3006 DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
3007 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
3008 }
3009 if (FUZ_rand(&lseed) & 1) {
3010 DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
3011 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
3012 }
3013 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
3014 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
3015 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
3016
3017 /* mess with long distance matching parameters */
3018 if (bigTests) {
3019 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) );
3020 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
3021 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
3022 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
3023 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
3024 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_srcSizeHint, FUZ_randomClampedLength(&lseed, ZSTD_SRCSIZEHINT_MIN, ZSTD_SRCSIZEHINT_MAX), opaqueAPI) );
3025 }
3026
3027 /* mess with frame parameters */
3028 if (FUZ_rand(&lseed) & 1) {
3029 int const checksumFlag = FUZ_rand(&lseed) & 1;
3030 DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
3031 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
3032 }
3033 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
3034 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
3035 if (FUZ_rand(&lseed) & 1) {
3036 DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize);
3037 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
3038 } else {
3039 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
3040 }
3041
3042 /* multi-threading parameters. Only adjust occasionally for small tests. */
3043 if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
3044 U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
3045 U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
3046 int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
3047 DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads);
3048 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
3049 if (nbThreads > 1) {
3050 U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
3051 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
3052 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
3053 }
3054 }
3055 /* Enable rsyncable mode 1 in 4 times. */
3056 {
3057 int const rsyncable = (FUZ_rand(&lseed) % 4 == 0);
3058 DISPLAYLEVEL(5, "t%u: rsyncable : %d \n", testNb, rsyncable);
3059 setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, rsyncable, opaqueAPI);
3060 }
3061
3062 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
3063 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) );
3064
3065 /* Set max block size parameters */
3066 if (FUZ_rand(&lseed) & 1) {
3067 int maxBlockSize = (int)(FUZ_rand(&lseed) % ZSTD_BLOCKSIZE_MAX);
3068 maxBlockSize = MAX(1024, maxBlockSize);
3069 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_maxBlockSize, maxBlockSize, opaqueAPI) );
3070 }
3071
3072 /* Apply parameters */
3073 if (opaqueAPI) {
3074 DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb);
3075 CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
3076 }
3077
3078 if (FUZ_rand(&lseed) & 1) {
3079 if (FUZ_rand(&lseed) & 1) {
3080 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
3081 } else {
3082 CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
3083 }
3084 } else {
3085 isRefPrefix = 1;
3086 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
3087 }
3088 } }
3089
3090 CHECK_Z(getCCtxParams(zc, &savedParams));
3091
3092 /* multi-segments compression test */
3093 { int iter;
3094 int const startSeed = lseed;
3095 XXH64_hash_t compressedCrcs[2];
3096 for (iter = 0; iter < 2; ++iter, lseed = startSeed) {
3097 ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
3098 int const singlePass = (FUZ_rand(&lseed) & 3) == 0;
3099 int nbWorkers;
3100
3101 XXH64_reset(&xxhState, 0);
3102
3103 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
3104 if (isRefPrefix) {
3105 DISPLAYLEVEL(6, "t%u: Reloading prefix\n", testNb);
3106 /* Need to reload the prefix because it gets dropped after one compression */
3107 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
3108 }
3109
3110 /* Adjust number of workers occasionally - result must be deterministic independent of nbWorkers */
3111 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers));
3112 if (nbWorkers > 0 && (FUZ_rand(&lseed) & 7) == 0) {
3113 DISPLAYLEVEL(6, "t%u: Modify nbWorkers: %d -> %d \n", testNb, nbWorkers, nbWorkers + iter);
3114 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers + iter));
3115 }
3116
3117 if (singlePass) {
3118 ZSTD_inBuffer inBuff = { srcBuffer, maxTestSize, 0 };
3119 CHECK_Z(ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end));
3120 DISPLAYLEVEL(6, "t%u: Single pass compression: consumed %u bytes ; produced %u bytes \n",
3121 testNb, (unsigned)inBuff.pos, (unsigned)outBuff.pos);
3122 CHECK(inBuff.pos != inBuff.size, "Input not consumed!");
3123 crcOrig = XXH64(srcBuffer, maxTestSize, 0);
3124 totalTestSize = maxTestSize;
3125 } else {
3126 outBuff.size = 0;
3127 for (totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
3128 /* compress random chunks into randomly sized dst buffers */
3129 size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3130 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
3131 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
3132 ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
3133 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
3134 int forwardProgress;
3135 do {
3136 size_t const ipos = inBuff.pos;
3137 size_t const opos = outBuff.pos;
3138 size_t ret;
3139 if (outBuff.pos == outBuff.size) {
3140 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
3141 size_t const dstBuffSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
3142 outBuff.size = outBuff.pos + dstBuffSize;
3143 }
3144 CHECK_Z( ret = ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
3145 DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
3146 testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos);
3147
3148 /* We've completed the flush */
3149 if (flush == ZSTD_e_flush && ret == 0)
3150 break;
3151
3152 /* Ensure maximal forward progress for determinism */
3153 forwardProgress = (inBuff.pos != ipos) || (outBuff.pos != opos);
3154 } while (forwardProgress);
3155 assert(inBuff.pos == inBuff.size);
3156
3157 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
3158 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
3159 totalTestSize += inBuff.pos;
3160 }
3161
3162 /* final frame epilogue */
3163 { size_t remainingToFlush = 1;
3164 while (remainingToFlush) {
3165 ZSTD_inBuffer inBuff = { NULL, 0, 0 };
3166 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
3167 size_t const adjustedDstSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
3168 outBuff.size = outBuff.pos + adjustedDstSize;
3169 DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize);
3170 /* ZSTD_e_end guarantees maximal forward progress */
3171 remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
3172 DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos);
3173 CHECK( ZSTD_isError(remainingToFlush),
3174 "ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
3175 ZSTD_getErrorName(remainingToFlush) );
3176 } }
3177 crcOrig = XXH64_digest(&xxhState);
3178 }
3179 cSize = outBuff.pos;
3180 compressedCrcs[iter] = XXH64(cBuffer, cSize, 0);
3181 DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (unsigned)cSize);
3182 }
3183 CHECK(!(compressedCrcs[0] == compressedCrcs[1]), "Compression is not deterministic!");
3184 }
3185
3186 CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
3187
3188 /* multi - fragments decompression test */
3189 if (FUZ_rand(&lseed) & 1) {
3190 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
3191 }
3192 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
3193 DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", (void const*)dict);
3194 CHECK_Z( ZSTD_resetDStream(zd) );
3195 } else {
3196 if (dictSize)
3197 DISPLAYLEVEL(5, "using dictionary of size %u \n", (unsigned)dictSize);
3198 CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
3199 }
3200 if (FUZ_rand(&lseed) & 1) {
3201 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
3202 }
3203 if (FUZ_rand(&lseed) & 1) {
3204 int maxBlockSize;
3205 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_maxBlockSize, &maxBlockSize));
3206 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_maxBlockSize, maxBlockSize));
3207 } else {
3208 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_maxBlockSize, 0));
3209 }
3210 { size_t decompressionResult = 1;
3211 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
3212 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
3213 for (totalGenSize = 0 ; decompressionResult ; ) {
3214 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3215 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
3216 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
3217 inBuff.size = inBuff.pos + readCSrcSize;
3218 outBuff.size = outBuff.pos + dstBuffSize;
3219 DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n",
3220 (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize);
3221 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
3222 DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n",
3223 (unsigned)inBuff.pos, (unsigned)outBuff.pos);
3224 if (ZSTD_isError(decompressionResult)) {
3225 DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult));
3226 findDiff(copyBuffer, dstBuffer, totalTestSize);
3227 }
3228 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
3229 CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize);
3230 }
3231 CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize);
3232 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize);
3233 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
3234 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
3235 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
3236 } }
3237
3238 /*===== noisy/erroneous src decompression test =====*/
3239
3240 /* add some noise */
3241 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
3242 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
3243 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
3244 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
3245 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
3246 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
3247 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
3248 } }
3249
3250 /* try decompression on noisy data */
3251 if (FUZ_rand(&lseed) & 1) {
3252 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_and_parameters));
3253 } else {
3254 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_only));
3255 }
3256 if (FUZ_rand(&lseed) & 1) {
3257 CHECK_Z(ZSTD_DCtx_setParameter(zd_noise, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
3258 }
3259 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
3260 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
3261 while (outBuff.pos < dstBufferSize) {
3262 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3263 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
3264 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
3265 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
3266 outBuff.size = outBuff.pos + adjustedDstSize;
3267 inBuff.size = inBuff.pos + adjustedCSrcSize;
3268 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
3269 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
3270 /* Good so far, but no more progress possible */
3271 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
3272 } } } }
3273 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
3274
3275 _cleanup:
3276 ZSTD_freeCCtx(zc);
3277 ZSTD_freeDStream(zd);
3278 ZSTD_freeDStream(zd_noise);
3279 ZSTD_freeCCtxParams(cctxParams);
3280 free(cNoiseBuffer[0]);
3281 free(cNoiseBuffer[1]);
3282 free(cNoiseBuffer[2]);
3283 free(cNoiseBuffer[3]);
3284 free(cNoiseBuffer[4]);
3285 free(copyBuffer);
3286 free(cBuffer);
3287 free(dstBuffer);
3288 return result;
3289
3290 _output_error:
3291 result = 1;
3292 goto _cleanup;
3293 }
3294
3295 /*-*******************************************************
3296 * Command line
3297 *********************************************************/
FUZ_usage(const char * programName)3298 static int FUZ_usage(const char* programName)
3299 {
3300 DISPLAY( "Usage :\n");
3301 DISPLAY( " %s [args]\n", programName);
3302 DISPLAY( "\n");
3303 DISPLAY( "Arguments :\n");
3304 DISPLAY( " -i# : Number of tests (default:%u)\n", nbTestsDefault);
3305 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
3306 DISPLAY( " -s# : Select seed (default:prompt user)\n");
3307 DISPLAY( " -t# : Select starting test number (default:0)\n");
3308 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
3309 DISPLAY( " -v : verbose\n");
3310 DISPLAY( " -p : pause at the end\n");
3311 DISPLAY( " -h : display help and exit\n");
3312 return 0;
3313 }
3314
3315 typedef enum { simple_api, advanced_api } e_api;
3316
main(int argc,const char ** argv)3317 int main(int argc, const char** argv)
3318 {
3319 U32 seed = 0;
3320 int seedset = 0;
3321 int nbTests = nbTestsDefault;
3322 int testNb = 0;
3323 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
3324 int result = 0;
3325 int mainPause = 0;
3326 int bigTests = (sizeof(size_t) == 8);
3327 e_api selected_api = simple_api;
3328 const char* const programName = argv[0];
3329 int argNb;
3330
3331 /* Check command line */
3332 for(argNb=1; argNb<argc; argNb++) {
3333 const char* argument = argv[argNb];
3334 assert(argument != NULL);
3335
3336 /* Parsing commands. Aggregated commands are allowed */
3337 if (argument[0]=='-') {
3338
3339 if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
3340 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
3341 if (!strcmp(argument, "--big-tests")) { bigTests=1; continue; }
3342
3343 argument++;
3344 while (*argument!=0) {
3345 switch(*argument)
3346 {
3347 case 'h':
3348 return FUZ_usage(programName);
3349
3350 case 'v':
3351 argument++;
3352 g_displayLevel++;
3353 break;
3354
3355 case 'q':
3356 argument++;
3357 g_displayLevel--;
3358 break;
3359
3360 case 'p': /* pause at the end */
3361 argument++;
3362 mainPause = 1;
3363 break;
3364
3365 case 'i': /* limit tests by nb of iterations (default) */
3366 argument++;
3367 nbTests=0; g_clockTime=0;
3368 while ((*argument>='0') && (*argument<='9')) {
3369 nbTests *= 10;
3370 nbTests += *argument - '0';
3371 argument++;
3372 }
3373 break;
3374
3375 case 'T': /* limit tests by time */
3376 argument++;
3377 nbTests=0; g_clockTime=0;
3378 while ((*argument>='0') && (*argument<='9')) {
3379 g_clockTime *= 10;
3380 g_clockTime += *argument - '0';
3381 argument++;
3382 }
3383 if (*argument=='m') { /* -T1m == -T60 */
3384 g_clockTime *=60, argument++;
3385 if (*argument=='n') argument++; /* -T1mn == -T60 */
3386 } else if (*argument=='s') argument++; /* -T10s == -T10 */
3387 g_clockTime *= SEC_TO_MICRO;
3388 break;
3389
3390 case 's': /* manually select seed */
3391 argument++;
3392 seedset=1;
3393 seed=0;
3394 while ((*argument>='0') && (*argument<='9')) {
3395 seed *= 10;
3396 seed += *argument - '0';
3397 argument++;
3398 }
3399 break;
3400
3401 case 't': /* select starting test number */
3402 argument++;
3403 testNb=0;
3404 while ((*argument>='0') && (*argument<='9')) {
3405 testNb *= 10;
3406 testNb += *argument - '0';
3407 argument++;
3408 }
3409 break;
3410
3411 case 'P': /* compressibility % */
3412 argument++;
3413 proba=0;
3414 while ((*argument>='0') && (*argument<='9')) {
3415 proba *= 10;
3416 proba += *argument - '0';
3417 argument++;
3418 }
3419 if (proba<0) proba=0;
3420 if (proba>100) proba=100;
3421 break;
3422
3423 default:
3424 return FUZ_usage(programName);
3425 }
3426 } } } /* for(argNb=1; argNb<argc; argNb++) */
3427
3428 /* Get Seed */
3429 DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
3430
3431 if (!seedset) {
3432 time_t const t = time(NULL);
3433 U32 const h = XXH32(&t, sizeof(t), 1);
3434 seed = h % 10000;
3435 }
3436
3437 DISPLAY("Seed = %u\n", (unsigned)seed);
3438 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
3439
3440 if (nbTests<=0) nbTests=1;
3441
3442 if (testNb==0) {
3443 result = basicUnitTests(0, ((double)proba) / 100, bigTests); /* constant seed for predictability */
3444 }
3445
3446 if (!result) {
3447 switch(selected_api)
3448 {
3449 case simple_api :
3450 result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3451 break;
3452 case advanced_api :
3453 result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3454 break;
3455 default :
3456 assert(0); /* impossible */
3457 }
3458 }
3459
3460 if (mainPause) {
3461 int unused;
3462 DISPLAY("Press Enter \n");
3463 unused = getchar();
3464 (void)unused;
3465 }
3466 return result;
3467 }
3468