1 /*
2 * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12 /*-************************************
13 * Compiler specific
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
19 #endif
20
21
22 /*-************************************
23 * Includes
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #undef NDEBUG
29 #include <assert.h>
30 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
31 #include "debug.h" /* DEBUG_STATIC_ASSERT */
32 #include "fse.h"
33 #include "zstd.h" /* ZSTD_VERSION_STRING */
34 #include "zstd_errors.h" /* ZSTD_getErrorCode */
35 #define ZDICT_STATIC_LINKING_ONLY
36 #include "zdict.h" /* ZDICT_trainFromBuffer */
37 #include "mem.h"
38 #include "datagen.h" /* RDG_genBuffer */
39 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
40 #include "xxhash.h" /* XXH64 */
41 #include "util.h"
42 #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
43 /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
44 #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
45
46
47 /*-************************************
48 * Constants
49 **************************************/
50 #define GB *(1U<<30)
51
52 static const int FUZ_compressibility_default = 50;
53 static const int nbTestsDefault = 30000;
54
55
56 /*-************************************
57 * Display Macros
58 **************************************/
59 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
60 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
61 static U32 g_displayLevel = 2;
62
63 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
64 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
65
66 #define DISPLAYUPDATE(l, ...) \
67 if (g_displayLevel>=l) { \
68 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
69 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
70 if (g_displayLevel>=4) fflush(stderr); } \
71 }
72
73
74 /*-*******************************************************
75 * Compile time test
76 *********************************************************/
77 #undef MIN
78 #undef MAX
79 /* Declaring the function, to avoid -Wmissing-prototype */
80 void FUZ_bug976(void);
FUZ_bug976(void)81 void FUZ_bug976(void)
82 { /* these constants shall not depend on MIN() macro */
83 assert(ZSTD_HASHLOG_MAX < 31);
84 assert(ZSTD_CHAINLOG_MAX < 31);
85 }
86
87
88 /*-*******************************************************
89 * Internal functions
90 *********************************************************/
91 #define MIN(a,b) ((a)<(b)?(a):(b))
92 #define MAX(a,b) ((a)>(b)?(a):(b))
93
94 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(U32 * src)95 static U32 FUZ_rand(U32* src)
96 {
97 static const U32 prime1 = 2654435761U;
98 static const U32 prime2 = 2246822519U;
99 U32 rand32 = *src;
100 rand32 *= prime1;
101 rand32 += prime2;
102 rand32 = FUZ_rotl32(rand32, 13);
103 *src = rand32;
104 return rand32 >> 5;
105 }
106
FUZ_highbit32(U32 v32)107 static U32 FUZ_highbit32(U32 v32)
108 {
109 unsigned nbBits = 0;
110 if (v32==0) return 0;
111 while (v32) v32 >>= 1, nbBits++;
112 return nbBits;
113 }
114
115
116 /*=============================================
117 * Test macros
118 =============================================*/
119 #define CHECK_Z(f) { \
120 size_t const err = f; \
121 if (ZSTD_isError(err)) { \
122 DISPLAY("Error => %s : %s ", \
123 #f, ZSTD_getErrorName(err)); \
124 exit(1); \
125 } }
126
127 #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; }
128 #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
129 #define CHECK(fn) { CHECK_NEWV(err, fn); }
130 #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
131
132 #define CHECK_OP(op, lhs, rhs) { \
133 if (!((lhs) op (rhs))) { \
134 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
135 goto _output_error; \
136 } \
137 }
138 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
139 #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
140
141
142 /*=============================================
143 * Memory Tests
144 =============================================*/
145 #if defined(__APPLE__) && defined(__MACH__)
146
147 #include <malloc/malloc.h> /* malloc_size */
148
149 typedef struct {
150 unsigned long long totalMalloc;
151 size_t currentMalloc;
152 size_t peakMalloc;
153 unsigned nbMalloc;
154 unsigned nbFree;
155 } mallocCounter_t;
156
157 static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };
158
FUZ_mallocDebug(void * counter,size_t size)159 static void* FUZ_mallocDebug(void* counter, size_t size)
160 {
161 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
162 void* const ptr = malloc(size);
163 if (ptr==NULL) return NULL;
164 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
165 (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */
166 mcPtr->totalMalloc += size;
167 mcPtr->currentMalloc += size;
168 if (mcPtr->currentMalloc > mcPtr->peakMalloc)
169 mcPtr->peakMalloc = mcPtr->currentMalloc;
170 mcPtr->nbMalloc += 1;
171 return ptr;
172 }
173
FUZ_freeDebug(void * counter,void * address)174 static void FUZ_freeDebug(void* counter, void* address)
175 {
176 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
177 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10));
178 mcPtr->nbFree += 1;
179 mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */
180 free(address);
181 }
182
FUZ_displayMallocStats(mallocCounter_t count)183 static void FUZ_displayMallocStats(mallocCounter_t count)
184 {
185 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n",
186 (unsigned)(count.peakMalloc >> 10),
187 count.nbMalloc,
188 (unsigned)(count.totalMalloc >> 10));
189 }
190
FUZ_mallocTests_internal(unsigned seed,double compressibility,unsigned part,void * inBuffer,size_t inSize,void * outBuffer,size_t outSize)191 static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
192 void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
193 {
194 /* test only played in verbose mode, as they are long */
195 if (g_displayLevel<3) return 0;
196
197 /* Create compressible noise */
198 if (!inBuffer || !outBuffer) {
199 DISPLAY("Not enough memory, aborting\n");
200 exit(1);
201 }
202 RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
203
204 /* simple compression tests */
205 if (part <= 1)
206 { int compressionLevel;
207 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
208 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
209 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
210 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
211 CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
212 ZSTD_freeCCtx(cctx);
213 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
214 FUZ_displayMallocStats(malcount);
215 } }
216
217 /* streaming compression tests */
218 if (part <= 2)
219 { int compressionLevel;
220 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
221 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
222 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
223 ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
224 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
225 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
226 CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
227 CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
228 CHECK_Z( ZSTD_endStream(cstream, &out) );
229 ZSTD_freeCStream(cstream);
230 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
231 FUZ_displayMallocStats(malcount);
232 } }
233
234 /* advanced MT API test */
235 if (part <= 3)
236 { int nbThreads;
237 for (nbThreads=1; nbThreads<=4; nbThreads++) {
238 int compressionLevel;
239 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
240 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
241 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
242 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
243 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
244 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
245 CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
246 ZSTD_freeCCtx(cctx);
247 DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
248 nbThreads, compressionLevel);
249 FUZ_displayMallocStats(malcount);
250 } } }
251
252 /* advanced MT streaming API test */
253 if (part <= 4)
254 { int nbThreads;
255 for (nbThreads=1; nbThreads<=4; nbThreads++) {
256 int compressionLevel;
257 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
258 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
259 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
260 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
261 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
262 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
263 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
264 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
265 CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
266 while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
267 ZSTD_freeCCtx(cctx);
268 DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ",
269 nbThreads, compressionLevel);
270 FUZ_displayMallocStats(malcount);
271 } } }
272
273 return 0;
274 }
275
FUZ_mallocTests(unsigned seed,double compressibility,unsigned part)276 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
277 {
278 size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
279 size_t const outSize = ZSTD_compressBound(inSize);
280 void* const inBuffer = malloc(inSize);
281 void* const outBuffer = malloc(outSize);
282 int result;
283
284 /* Create compressible noise */
285 if (!inBuffer || !outBuffer) {
286 DISPLAY("Not enough memory, aborting \n");
287 exit(1);
288 }
289
290 result = FUZ_mallocTests_internal(seed, compressibility, part,
291 inBuffer, inSize, outBuffer, outSize);
292
293 free(inBuffer);
294 free(outBuffer);
295 return result;
296 }
297
298 #else
299
FUZ_mallocTests(unsigned seed,double compressibility,unsigned part)300 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
301 {
302 (void)seed; (void)compressibility; (void)part;
303 return 0;
304 }
305
306 #endif
307
FUZ_decodeSequences(BYTE * dst,ZSTD_Sequence * seqs,size_t seqsSize,BYTE * src,size_t size,ZSTD_sequenceFormat_e format)308 static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
309 BYTE* src, size_t size, ZSTD_sequenceFormat_e format)
310 {
311 size_t i;
312 size_t j;
313 for(i = 0; i < seqsSize; ++i) {
314 assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size);
315 assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size);
316 if (format == ZSTD_sf_noBlockDelimiters) {
317 assert(seqs[i].matchLength != 0 || seqs[i].offset != 0);
318 }
319
320 memcpy(dst, src, seqs[i].litLength);
321 dst += seqs[i].litLength;
322 src += seqs[i].litLength;
323 size -= seqs[i].litLength;
324
325 if (seqs[i].offset != 0) {
326 for (j = 0; j < seqs[i].matchLength; ++j)
327 dst[j] = dst[j - seqs[i].offset];
328 dst += seqs[i].matchLength;
329 src += seqs[i].matchLength;
330 size -= seqs[i].matchLength;
331 }
332 }
333 if (format == ZSTD_sf_noBlockDelimiters) {
334 memcpy(dst, src, size);
335 }
336 }
337
338 /*=============================================
339 * Unit tests
340 =============================================*/
341
basicUnitTests(U32 const seed,double compressibility)342 static int basicUnitTests(U32 const seed, double compressibility)
343 {
344 size_t const CNBuffSize = 5 MB;
345 void* const CNBuffer = malloc(CNBuffSize);
346 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
347 void* const compressedBuffer = malloc(compressedBufferSize);
348 void* const decodedBuffer = malloc(CNBuffSize);
349 int testResult = 0;
350 unsigned testNb=0;
351 size_t cSize;
352
353 /* Create compressible noise */
354 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
355 DISPLAY("Not enough memory, aborting\n");
356 testResult = 1;
357 goto _end;
358 }
359 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
360
361 /* Basic tests */
362 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++);
363 { const char* errorString = ZSTD_getErrorName(0);
364 DISPLAYLEVEL(3, "OK : %s \n", errorString);
365 }
366
367 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++);
368 { const char* errorString = ZSTD_getErrorName(499);
369 DISPLAYLEVEL(3, "OK : %s \n", errorString);
370 }
371
372 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++);
373 { int const mcl = ZSTD_minCLevel();
374 DISPLAYLEVEL(3, "%i (OK) \n", mcl);
375 }
376
377 DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
378 { unsigned const vn = ZSTD_versionNumber();
379 DISPLAYLEVEL(3, "%u (OK) \n", vn);
380 }
381
382 DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
383 {
384 ZSTD_compressionParameters params;
385 memset(¶ms, 0, sizeof(params));
386 params.windowLog = 10;
387 params.hashLog = 19;
388 params.chainLog = 19;
389 params = ZSTD_adjustCParams(params, 1000, 100000);
390 if (params.hashLog != 18) goto _output_error;
391 if (params.chainLog != 17) goto _output_error;
392 }
393 DISPLAYLEVEL(3, "OK \n");
394
395 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
396 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
397 if (cctx==NULL) goto _output_error;
398 CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
399 compressedBuffer, compressedBufferSize,
400 CNBuffer, CNBuffSize, 1) );
401 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
402
403 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
404 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
405 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize);
406 }
407 ZSTD_freeCCtx(cctx);
408 }
409
410 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++);
411 {
412 char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
413 size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8);
414 if (!ZSTD_isError(size)) goto _output_error;
415 }
416 DISPLAYLEVEL(3, "OK \n");
417
418 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
419 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
420 if (rSize != CNBuffSize) goto _output_error;
421 }
422 DISPLAYLEVEL(3, "OK \n");
423
424 DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++);
425 { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
426 if (rSize != CNBuffSize) goto _output_error;
427 }
428 DISPLAYLEVEL(3, "OK \n");
429
430 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
431 { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
432 if (rSize != CNBuffSize) goto _output_error;
433 }
434 DISPLAYLEVEL(3, "OK \n");
435
436 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
437 {
438 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
439 if (bound != CNBuffSize) goto _output_error;
440 }
441 DISPLAYLEVEL(3, "OK \n");
442
443 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
444 {
445 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
446 if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
447 }
448 DISPLAYLEVEL(3, "OK \n");
449
450 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
451 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
452 if (r != CNBuffSize) goto _output_error; }
453 DISPLAYLEVEL(3, "OK \n");
454
455 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
456 { size_t u;
457 for (u=0; u<CNBuffSize; u++) {
458 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
459 } }
460 DISPLAYLEVEL(3, "OK \n");
461
462 DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++);
463 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
464 ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 };
465 ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 };
466 if (cctx==NULL) goto _output_error;
467 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */
468 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */
469 ZSTD_freeCCtx(cctx);
470 }
471 DISPLAYLEVEL(3, "OK \n");
472
473 DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++);
474 {
475 ZSTD_parameters params = ZSTD_getParams(3, 0, 0);
476 assert(!ZSTD_checkCParams(params.cParams));
477 }
478 DISPLAYLEVEL(3, "OK \n");
479
480 DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++);
481 {
482 ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
483 assert(dctx != NULL);
484 assert(ZSTD_sizeof_DCtx(dctx) != 0);
485 ZSTD_freeDCtx(dctx);
486 }
487 DISPLAYLEVEL(3, "OK \n");
488
489 DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++);
490 {
491 /* %p takes a void*. In ISO C, it's illegal to cast a function pointer
492 * to a data pointer. (Although in POSIX you're required to be allowed
493 * to do it...) So we have to fall back to our trusty friend memcpy. */
494 unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) =
495 ZSTD_getDictID_fromDDict;
496 ZSTD_DStream* (* const funcptr_createDStream)(
497 ZSTD_customMem customMem) = ZSTD_createDStream_advanced;
498 void (* const funcptr_copyDCtx)(
499 ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx;
500 ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) =
501 ZSTD_nextInputType;
502 const void *voidptr_getDictID;
503 const void *voidptr_createDStream;
504 const void *voidptr_copyDCtx;
505 const void *voidptr_nextInputType;
506 DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID));
507 memcpy(
508 (void*)&voidptr_getDictID,
509 (const void*)&funcptr_getDictID,
510 sizeof(void*));
511 memcpy(
512 (void*)&voidptr_createDStream,
513 (const void*)&funcptr_createDStream,
514 sizeof(void*));
515 memcpy(
516 (void*)&voidptr_copyDCtx,
517 (const void*)&funcptr_copyDCtx,
518 sizeof(void*));
519 memcpy(
520 (void*)&voidptr_nextInputType,
521 (const void*)&funcptr_nextInputType,
522 sizeof(void*));
523 DISPLAYLEVEL(3, "%p ", voidptr_getDictID);
524 DISPLAYLEVEL(3, "%p ", voidptr_createDStream);
525 DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx);
526 DISPLAYLEVEL(3, "%p ", voidptr_nextInputType);
527 }
528 DISPLAYLEVEL(3, ": OK \n");
529
530 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
531 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
532 { size_t const r = ZSTD_decompress_usingDict(dctx,
533 decodedBuffer, CNBuffSize,
534 compressedBuffer, cSize,
535 NULL, 0);
536 if (r != CNBuffSize) goto _output_error;
537 }
538 ZSTD_freeDCtx(dctx);
539 }
540 DISPLAYLEVEL(3, "OK \n");
541
542 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
543 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
544 { size_t const r = ZSTD_decompress_usingDDict(dctx,
545 decodedBuffer, CNBuffSize,
546 compressedBuffer, cSize,
547 NULL);
548 if (r != CNBuffSize) goto _output_error;
549 }
550 ZSTD_freeDCtx(dctx);
551 }
552 DISPLAYLEVEL(3, "OK \n");
553
554 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
555 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
556 if (!ZSTD_isError(r)) goto _output_error;
557 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
558 DISPLAYLEVEL(3, "OK \n");
559
560 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
561 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
562 if (!ZSTD_isError(r)) goto _output_error;
563 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
564 DISPLAYLEVEL(3, "OK \n");
565
566 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++);
567 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize);
568 if (!ZSTD_isError(r)) goto _output_error;
569 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
570 DISPLAYLEVEL(3, "OK \n");
571
572 DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++);
573 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize);
574 if (!ZSTD_isError(r)) goto _output_error;
575 if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; }
576 DISPLAYLEVEL(3, "OK \n");
577
578 DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++);
579 { /* create compressed buffer with checksumming enabled */
580 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
581 if (!cctx) {
582 DISPLAY("Not enough memory, aborting\n");
583 testResult = 1;
584 goto _end;
585 }
586 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
587 CHECK_VAR(cSize, ZSTD_compress2(cctx,
588 compressedBuffer, compressedBufferSize,
589 CNBuffer, CNBuffSize) );
590 ZSTD_freeCCtx(cctx);
591 }
592 { /* copy the compressed buffer and corrupt the checksum */
593 size_t r;
594 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
595 if (!dctx) {
596 DISPLAY("Not enough memory, aborting\n");
597 testResult = 1;
598 goto _end;
599 }
600
601 ((char*)compressedBuffer)[cSize-1] += 1;
602 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
603 if (!ZSTD_isError(r)) goto _output_error;
604 if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error;
605
606 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum));
607 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
608 if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */
609 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
610 if (ZSTD_isError(r)) goto _output_error;
611
612 ZSTD_freeDCtx(dctx);
613 }
614 DISPLAYLEVEL(3, "OK \n");
615
616
617 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
618 { /* create compressed buffer with content size missing */
619 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
620 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
621 CHECK_VAR(cSize, ZSTD_compress2(cctx,
622 compressedBuffer, compressedBufferSize,
623 CNBuffer, CNBuffSize) );
624 ZSTD_freeCCtx(cctx);
625 }
626 { /* ensure frame content size is missing */
627 ZSTD_frameHeader zfh;
628 size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
629 if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
630 }
631 { /* ensure CNBuffSize <= decompressBound */
632 unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
633 if (CNBuffSize > bound) goto _output_error;
634 }
635 DISPLAYLEVEL(3, "OK \n");
636
637 DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++);
638 {
639 size_t const largeFrameSrcSize = 200;
640 size_t const smallFrameSrcSize = 10;
641 size_t const nbFrames = 256;
642
643 size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0;
644 int sizeReduced = 0;
645
646 BYTE* const dst = (BYTE*)compressedBuffer;
647 ZSTD_DCtx* dctx = ZSTD_createDCtx();
648
649 /* create a large frame and then a bunch of small frames */
650 size_t srcSize = ZSTD_compress((void*)dst,
651 compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
652 for (i = 0; i < nbFrames; i++)
653 srcSize += ZSTD_compress((void*)(dst + srcSize),
654 compressedBufferSize - srcSize, CNBuffer,
655 smallFrameSrcSize, 3);
656
657 /* decompressStream and make sure that dctx size was reduced at least once */
658 while (consumed < srcSize) {
659 ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
660 ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0};
661 ZSTD_decompressStream(dctx, &out, &in);
662 consumed += in.pos;
663 produced += out.pos;
664
665 /* success! size was reduced from the previous frame */
666 if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx))
667 sizeReduced = 1;
668
669 prevDCtxSize = ZSTD_sizeof_DCtx(dctx);
670 }
671
672 assert(sizeReduced);
673
674 ZSTD_freeDCtx(dctx);
675 }
676 DISPLAYLEVEL(3, "OK \n");
677
678 DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
679 {
680 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
681
682 size_t const size = (1U << 10);
683 size_t const dstCapacity = ZSTD_compressBound(size);
684 void* dict = (void*)malloc(size);
685 void* src = (void*)malloc(size);
686 void* dst = (void*)malloc(dstCapacity);
687
688 RDG_genBuffer(dict, size, 0.5, 0.5, seed);
689 RDG_genBuffer(src, size, 0.5, 0.5, seed);
690
691 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
692 assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
693
694 ZSTD_freeCCtx(cctx);
695 free(dict);
696 free(src);
697 free(dst);
698 }
699 DISPLAYLEVEL(3, "OK \n");
700
701 DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
702 {
703 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
704 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
705 void* dict = (void*)malloc(CNBuffSize);
706 int nbWorkers;
707
708 for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) {
709 RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
710 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
711
712 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
713 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
714 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
715 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
716 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
717 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
718 CHECK_Z(cSize);
719 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize));
720 }
721
722 ZSTD_freeCCtx(cctx);
723 ZSTD_freeDCtx(dctx);
724 free(dict);
725 }
726 DISPLAYLEVEL(3, "OK \n");
727
728 DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
729 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
730 ZSTD_DCtx* dctx = ZSTD_createDCtx();
731 size_t const srcSize = 300 KB;
732 size_t const flushSize = 128 KB + 5;
733 size_t const dstSize = ZSTD_compressBound(srcSize);
734 char* src = (char*)CNBuffer;
735 char* dst = (char*)compressedBuffer;
736
737 ZSTD_outBuffer out = { dst, dstSize, 0 };
738 ZSTD_inBuffer in = { src, flushSize, 0 };
739
740 if (!cctx || !dctx) {
741 DISPLAY("Not enough memory, aborting\n");
742 testResult = 1;
743 goto _end;
744 }
745
746 RDG_genBuffer(src, srcSize, 0.5, 0.5, seed);
747 /* Force an LDM to exist that crosses block boundary into uncompressible block */
748 memcpy(src + 125 KB, src, 3 KB + 5);
749
750 /* Enable MT, LDM, and opt parser */
751 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
752 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
753 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
754 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
755
756 /* Flushes a block of 128 KB and block of 5 bytes */
757 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
758
759 /* Compress the rest */
760 in.size = 300 KB;
761 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
762
763 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos));
764
765 ZSTD_freeCCtx(cctx);
766 ZSTD_freeDCtx(dctx);
767 }
768 DISPLAYLEVEL(3, "OK \n");
769
770 DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++);
771 {
772 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
773 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
774 void* dict = (void*)malloc(CNBuffSize);
775 size_t const kWindowLog = 10;
776 size_t const kWindowSize = (size_t)1 << kWindowLog;
777 size_t const dictSize = kWindowSize * 10;
778 size_t const srcSize1 = kWindowSize / 2;
779 size_t const srcSize2 = kWindowSize * 10;
780
781 if (CNBuffSize < dictSize) goto _output_error;
782
783 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
784 RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed);
785
786 /* Enable checksum to verify round trip. */
787 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
788 /* Disable content size to skip single-pass decompression. */
789 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
790 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
791 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
792 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
793 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
794 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
795 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3));
796
797 /* Round trip once with a dictionary. */
798 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
799 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
800 CHECK_Z(cSize);
801 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
802 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
803 /* Streaming decompression to catch out of bounds offsets. */
804 {
805 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
806 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
807 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
808 CHECK_Z(dSize);
809 if (dSize != 0) goto _output_error;
810 }
811
812 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
813 /* Round trip once with a dictionary. */
814 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
815 {
816 ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
817 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
818 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
819 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
820 cSize = out.pos;
821 }
822 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
823 {
824 ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
825 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
826 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
827 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
828 cSize = out.pos;
829 }
830 /* Streaming decompression to catch out of bounds offsets. */
831 {
832 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
833 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
834 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
835 CHECK_Z(dSize);
836 if (dSize != 0) goto _output_error;
837 }
838
839 ZSTD_freeCCtx(cctx);
840 ZSTD_freeDCtx(dctx);
841 free(dict);
842 }
843 DISPLAYLEVEL(3, "OK \n");
844
845 /* Note: this test takes 0.5 seconds to run */
846 DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
847 {
848 /* test a big buffer so that ldm can take effect */
849 size_t const size = 100 MB;
850 int const windowLog = 27;
851 size_t const dstSize = ZSTD_compressBound(size);
852
853 void* dict = (void*)malloc(size);
854 void* src = (void*)malloc(size);
855 void* dst = (void*)malloc(dstSize);
856 void* recon = (void*)malloc(size);
857
858 size_t refPrefixCompressedSize = 0;
859 size_t refPrefixLdmComrpessedSize = 0;
860 size_t reconSize = 0;
861
862 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
863 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
864
865 /* make dict and src the same uncompressible data */
866 RDG_genBuffer(src, size, 0, 0, seed);
867 memcpy(dict, src, size);
868 assert(!memcmp(dict, src, size));
869
870 /* set level 1 and windowLog to cover src */
871 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1));
872 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog));
873
874 /* compress on level 1 using just refPrefix and no ldm */
875 ZSTD_CCtx_refPrefix(cctx, dict, size);
876 refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
877 assert(!ZSTD_isError(refPrefixCompressedSize));
878
879 /* test round trip just refPrefix */
880 ZSTD_DCtx_refPrefix(dctx, dict, size);
881 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize);
882 assert(!ZSTD_isError(reconSize));
883 assert(reconSize == size);
884 assert(!memcmp(recon, src, size));
885
886 /* compress on level 1 using refPrefix and ldm */
887 ZSTD_CCtx_refPrefix(cctx, dict, size);;
888 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1))
889 refPrefixLdmComrpessedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
890 assert(!ZSTD_isError(refPrefixLdmComrpessedSize));
891
892 /* test round trip refPrefix + ldm*/
893 ZSTD_DCtx_refPrefix(dctx, dict, size);
894 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmComrpessedSize);
895 assert(!ZSTD_isError(reconSize));
896 assert(reconSize == size);
897 assert(!memcmp(recon, src, size));
898
899 /* make sure that refPrefixCompressedSize is significantly greater */
900 assert(refPrefixCompressedSize > 10 * refPrefixLdmComrpessedSize);
901 /* make sure the ldm comrpessed size is less than 1% of original */
902 assert((double)refPrefixLdmComrpessedSize / (double)size < 0.01);
903
904 ZSTD_freeDCtx(dctx);
905 ZSTD_freeCCtx(cctx);
906 free(recon);
907 free(dict);
908 free(src);
909 free(dst);
910 }
911 DISPLAYLEVEL(3, "OK \n");
912
913 DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
914 {
915 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
916 const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;
917 size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize);
918
919 /* This is the number of bytes to stream before ending. This value
920 * was obtained by trial and error :/. */
921
922 const size_t streamCompressThreshold = 161792;
923 const size_t streamCompressDelta = 1024;
924
925 /* The first 1/5 of the buffer is compressible and the last 4/5 is
926 * uncompressible. This is an approximation of the type of data
927 * the fuzzer generated to catch this bug. Streams like this were making
928 * zstd generate noCompress superblocks (which are larger than the src
929 * they come from). Do this enough times, and we'll run out of room
930 * and throw a dstSize_tooSmall error. */
931
932 const size_t compressiblePartSize = srcSize/5;
933 const size_t uncompressiblePartSize = srcSize-compressiblePartSize;
934 RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed);
935 RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed);
936
937 /* Setting target block size so that superblock is used */
938
939 assert(cctx != NULL);
940 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81);
941
942 { size_t read;
943 for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) {
944 ZSTD_inBuffer in = {src, streamCompressDelta, 0};
945 ZSTD_outBuffer out = {dst, dstCapacity, 0};
946 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
947 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
948 src += streamCompressDelta; srcSize -= streamCompressDelta;
949 dst += out.pos; dstCapacity -= out.pos;
950 } }
951
952 /* This is trying to catch a dstSize_tooSmall error */
953
954 { ZSTD_inBuffer in = {src, srcSize, 0};
955 ZSTD_outBuffer out = {dst, dstCapacity, 0};
956 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
957 }
958 ZSTD_freeCCtx(cctx);
959 }
960 DISPLAYLEVEL(3, "OK \n");
961
962 DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++);
963 /* Generate the same data 20 times over */
964 { size_t const avgChunkSize = CNBuffSize / 20;
965 size_t b;
966 for (b = 0; b < CNBuffSize; b += avgChunkSize) {
967 size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize);
968 RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed);
969 } }
970 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
971 size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
972 size_t const allowedExpansion = (CNBuffSize * 3 / 1000);
973 size_t superCSize;
974 CHECK_Z(normalCSize);
975 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
976 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000);
977 superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
978 CHECK_Z(superCSize);
979 if (superCSize > normalCSize + allowedExpansion) {
980 DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion);
981 goto _output_error;
982 }
983 ZSTD_freeCCtx(cctx);
984 }
985 DISPLAYLEVEL(3, "OK \n");
986
987 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed);
988 DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++)
989 /* This tests whether or not we leave enough room for the checksum at the end
990 * of the dst buffer. The bug that motivated this test was found by the
991 * stream_round_trip fuzzer but this crashes for the same reason and is
992 * far more compact than re-creating the stream_round_trip fuzzer's code path */
993 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
994 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64);
995 assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278)));
996 ZSTD_freeCCtx(cctx);
997 }
998 DISPLAYLEVEL(3, "OK \n");
999
1000 DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
1001 { int level = -1;
1002 ZSTD_CCtx* cctx = ZSTD_createCCtx();
1003 if (!cctx) goto _output_error;
1004 for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
1005 CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
1006 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
1007 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
1008 }
1009 ZSTD_freeCCtx(cctx);
1010 }
1011 DISPLAYLEVEL(3, "OK \n");
1012
1013 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
1014 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1015 size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
1016 if (ZSTD_isError(r)) goto _output_error;
1017 if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
1018 ZSTD_freeCCtx(cctx);
1019 cSize = r;
1020 }
1021 DISPLAYLEVEL(3, "OK \n");
1022
1023 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++);
1024 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize);
1025 if (ZSTD_isError(r)) goto _output_error;
1026 if (r != 0) goto _output_error;
1027 }
1028 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1029 ZSTD_outBuffer output;
1030 if (cctx==NULL) goto _output_error;
1031 output.dst = compressedBuffer;
1032 output.size = compressedBufferSize;
1033 output.pos = 0;
1034 CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */
1035 CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */
1036 CHECK_Z( ZSTD_endStream(cctx, &output) );
1037 ZSTD_freeCCtx(cctx);
1038 /* single scan decompression */
1039 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos);
1040 if (ZSTD_isError(r)) goto _output_error;
1041 if (r != 0) goto _output_error;
1042 }
1043 /* streaming decompression */
1044 { ZSTD_DCtx* const dstream = ZSTD_createDStream();
1045 ZSTD_inBuffer dinput;
1046 ZSTD_outBuffer doutput;
1047 size_t ipos;
1048 if (dstream==NULL) goto _output_error;
1049 dinput.src = compressedBuffer;
1050 dinput.size = 0;
1051 dinput.pos = 0;
1052 doutput.dst = NULL;
1053 doutput.size = 0;
1054 doutput.pos = 0;
1055 CHECK_Z ( ZSTD_initDStream(dstream) );
1056 for (ipos=1; ipos<=output.pos; ipos++) {
1057 dinput.size = ipos;
1058 CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) );
1059 }
1060 if (doutput.pos != 0) goto _output_error;
1061 ZSTD_freeDStream(dstream);
1062 }
1063 }
1064 DISPLAYLEVEL(3, "OK \n");
1065
1066 DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++);
1067 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1068 ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1069 assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
1070 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
1071 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
1072
1073 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1074 { size_t const inSize = 2* 128 KB;
1075 size_t const outSize = ZSTD_compressBound(inSize);
1076 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
1077 /* will fail if blockSize is not resized */
1078 }
1079 ZSTD_freeCCtx(cctx);
1080 }
1081 DISPLAYLEVEL(3, "OK \n");
1082
1083 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++);
1084 { size_t const sampleSize = 30;
1085 int i;
1086 for (i=0; i<20; i++)
1087 ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */
1088 memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
1089 for (i=1; i<=19; i++) {
1090 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1091 size_t size1, size2;
1092 DISPLAYLEVEL(5, "l%i ", i);
1093 size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1094 CHECK_Z(size1);
1095
1096 size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1097 CHECK_Z(size2);
1098 CHECK_EQ(size1, size2);
1099
1100 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) );
1101 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize);
1102 CHECK_Z(size2);
1103 CHECK_EQ(size1, size2);
1104
1105 size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */
1106 CHECK_Z(size2);
1107 CHECK_EQ(size1, size2);
1108
1109 { ZSTD_inBuffer inb;
1110 ZSTD_outBuffer outb;
1111 inb.src = CNBuffer;
1112 inb.pos = 0;
1113 inb.size = sampleSize;
1114 outb.dst = compressedBuffer;
1115 outb.pos = 0;
1116 outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */
1117 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );
1118 assert(inb.pos == inb.size);
1119 CHECK_EQ(size1, outb.pos);
1120 }
1121
1122 ZSTD_freeCCtx(cctx);
1123 }
1124 }
1125 DISPLAYLEVEL(3, "OK \n");
1126
1127 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++);
1128 { size_t const sampleSize = 1024;
1129 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1130 ZSTD_inBuffer inb;
1131 ZSTD_outBuffer outb;
1132 inb.src = CNBuffer;
1133 inb.pos = 0;
1134 inb.size = 0;
1135 outb.dst = compressedBuffer;
1136 outb.pos = 0;
1137 outb.size = compressedBufferSize;
1138 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) );
1139
1140 inb.size = sampleSize; /* start with something, so that context is already used */
1141 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1142 assert(inb.pos == inb.size);
1143 outb.pos = 0; /* cancel output */
1144
1145 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) );
1146 inb.size = 4; /* too small size : compression will be skipped */
1147 inb.pos = 0;
1148 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1149 assert(inb.pos == inb.size);
1150
1151 inb.size += 5; /* too small size : compression will be skipped */
1152 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1153 assert(inb.pos == inb.size);
1154
1155 inb.size += 11; /* small enough to attempt compression */
1156 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1157 assert(inb.pos == inb.size);
1158
1159 assert(inb.pos < sampleSize);
1160 inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */
1161 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1162 assert(inb.pos == inb.size);
1163 ZSTD_freeCCtx(cctx);
1164 }
1165 DISPLAYLEVEL(3, "OK \n");
1166
1167 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
1168 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1169 ZSTD_outBuffer out = {NULL, 0, 0};
1170 ZSTD_inBuffer in = {NULL, 0, 0};
1171 int value;
1172
1173 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1174 CHECK_EQ(value, 3);
1175 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1176 CHECK_EQ(value, 0);
1177 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
1178 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1179 CHECK_EQ(value, 3);
1180 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1181 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1182 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
1183 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1184 CHECK_EQ(value, 7);
1185 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1186 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1187 /* Start a compression job */
1188 ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1189 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1190 CHECK_EQ(value, 7);
1191 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1192 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1193 /* Reset the CCtx */
1194 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
1195 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1196 CHECK_EQ(value, 7);
1197 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1198 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1199 /* Reset the parameters */
1200 ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
1201 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1202 CHECK_EQ(value, 3);
1203 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1204 CHECK_EQ(value, 0);
1205
1206 ZSTD_freeCCtx(cctx);
1207 }
1208 DISPLAYLEVEL(3, "OK \n");
1209
1210 DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
1211 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1212 ZSTD_outBuffer out = {NULL, 0, 0};
1213 ZSTD_inBuffer in = {NULL, 0, 0};
1214 int value;
1215
1216 /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt),
1217 * we should not modify the actual parameter specified by the user within the CCtx
1218 */
1219 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27));
1220 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt));
1221
1222 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1223 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value));
1224 CHECK_EQ(value, 0);
1225
1226 ZSTD_freeCCtx(cctx);
1227 }
1228 DISPLAYLEVEL(3, "OK \n");
1229
1230 /* this test is really too long, and should be made faster */
1231 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
1232 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1233 ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1234 size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
1235 size_t cnb;
1236 assert(cctx != NULL);
1237 params.fParams.contentSizeFlag = 0;
1238 params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
1239 for (cnb = 0; cnb < nbCompressions; ++cnb) {
1240 DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions);
1241 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1242 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
1243 }
1244 ZSTD_freeCCtx(cctx);
1245 }
1246 DISPLAYLEVEL(3, "OK \n");
1247
1248 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
1249 { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
1250 assert(largeCCtx != NULL);
1251 CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
1252 CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
1253 { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
1254 { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
1255 assert(smallCCtx != NULL);
1256 CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1257 { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
1258 DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
1259 largeCCtxSize>>10, smallCCtxSize>>10);
1260 assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
1261 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
1262 }
1263 ZSTD_freeCCtx(smallCCtx);
1264 }
1265 { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
1266 * currently defined as 128x, but could be adjusted in the future.
1267 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
1268 unsigned u;
1269 for (u=0; u<maxNbAttempts; u++) {
1270 CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1271 if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
1272 }
1273 DISPLAYLEVEL(5, "size down after %u attempts : ", u);
1274 if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
1275 }
1276 }
1277 ZSTD_freeCCtx(largeCCtx);
1278 }
1279 DISPLAYLEVEL(3, "OK \n");
1280
1281 /* Static CCtx tests */
1282 #define STATIC_CCTX_LEVEL 4
1283 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1284 { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
1285 void* const staticCCtxBuffer = malloc(staticCStreamSize);
1286 size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
1287 void* const staticDCtxBuffer = malloc(staticDCtxSize);
1288 DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize);
1289 if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
1290 free(staticCCtxBuffer);
1291 free(staticDCtxBuffer);
1292 DISPLAY("Not enough memory, aborting\n");
1293 testResult = 1;
1294 goto _end;
1295 }
1296 { size_t const smallInSize = 32 KB;
1297 ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0);
1298 size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small);
1299 size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL);
1300 ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize);
1301 ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
1302 DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize);
1303 DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize);
1304 if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
1305 DISPLAYLEVEL(3, "OK \n");
1306
1307 DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++);
1308 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1309 compressedBuffer, compressedBufferSize,
1310 CNBuffer, smallInSize, STATIC_CCTX_LEVEL) );
1311 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1312 (unsigned)cSize, (double)cSize/smallInSize*100);
1313
1314 DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++);
1315 { size_t const r = ZSTD_compressCCtx(staticCCtx,
1316 compressedBuffer, compressedBufferSize,
1317 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL);
1318 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error;
1319 }
1320 DISPLAYLEVEL(3, "OK \n");
1321
1322 DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
1323 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
1324 DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, staticCCtx);
1325 if (staticCCtx == NULL) goto _output_error;
1326 DISPLAYLEVEL(3, "OK \n");
1327
1328 DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++);
1329 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1330 compressedBuffer, compressedBufferSize,
1331 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1332 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1333 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1334
1335 DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++);
1336 { int nbc;
1337 assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */
1338 assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3);
1339 for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) {
1340 CHECK_Z(ZSTD_compressCCtx(staticCCtx,
1341 compressedBuffer, compressedBufferSize,
1342 (char*)CNBuffer + nbc, smallInSize,
1343 STATIC_CCTX_LEVEL) );
1344 } }
1345 DISPLAYLEVEL(3, "OK \n")
1346
1347 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1348 CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) );
1349 DISPLAYLEVEL(3, "OK \n");
1350
1351 DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++);
1352 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1353 compressedBuffer, compressedBufferSize,
1354 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1355 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1356 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1357
1358 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
1359 { size_t const r = ZSTD_decompressDCtx(staticDCtx,
1360 decodedBuffer, CNBuffSize,
1361 compressedBuffer, cSize);
1362 if (r != CNBuffSize) goto _output_error; }
1363 DISPLAYLEVEL(3, "OK \n");
1364
1365 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1366 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
1367 DISPLAYLEVEL(3, "OK \n");
1368
1369 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
1370 { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
1371 if (!ZSTD_isError(r)) goto _output_error; }
1372 DISPLAYLEVEL(3, "OK \n");
1373
1374 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
1375 CHECK( ZSTD_compressBegin(staticCCtx, 1) );
1376 DISPLAYLEVEL(3, "OK \n");
1377
1378 DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
1379 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1380 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1381 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1382 size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */
1383 if (!ZSTD_isError(r)) goto _output_error;
1384 }
1385 DISPLAYLEVEL(3, "OK \n");
1386
1387 DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++);
1388 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize);
1389 assert(staticCCtx != NULL);
1390 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1391 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1392 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1393 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1394 }
1395 DISPLAYLEVEL(3, "OK \n");
1396
1397 DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1);
1398 CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */
1399 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1400 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1401 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1402 }
1403 DISPLAYLEVEL(3, "OK \n");
1404
1405 DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++);
1406 { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
1407 if (!ZSTD_isError(r)) goto _output_error; }
1408 DISPLAYLEVEL(3, "OK \n");
1409
1410 DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++);
1411 CHECK_Z( ZSTD_initDStream(staticDCtx) );
1412 { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
1413 ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
1414 size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
1415 if (!ZSTD_isError(r)) goto _output_error;
1416 }
1417 DISPLAYLEVEL(3, "OK \n");
1418 }
1419 free(staticCCtxBuffer);
1420 free(staticDCtxBuffer);
1421 }
1422
1423 DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++);
1424 { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1);
1425 size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1);
1426 size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1);
1427 size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1);
1428
1429 if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error;
1430 if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error;
1431 }
1432 DISPLAYLEVEL(3, "OK \n");
1433
1434
1435 /* ZSTDMT simple MT compression test */
1436 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
1437 { ZSTD_CCtx* const mtctx = ZSTD_createCCtx();
1438 if (mtctx==NULL) {
1439 DISPLAY("mtctx : not enough memory, aborting \n");
1440 testResult = 1;
1441 goto _end;
1442 }
1443 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
1444 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
1445 DISPLAYLEVEL(3, "OK \n");
1446
1447 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
1448 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
1449 compressedBuffer, compressedBufferSize,
1450 CNBuffer, CNBuffSize) );
1451 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1452
1453 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
1454 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
1455 if (rSize != CNBuffSize) {
1456 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize);
1457 goto _output_error;
1458 } }
1459 DISPLAYLEVEL(3, "OK \n");
1460
1461 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
1462 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1463 if (r != CNBuffSize) goto _output_error; }
1464 DISPLAYLEVEL(3, "OK \n");
1465
1466 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1467 { size_t u;
1468 for (u=0; u<CNBuffSize; u++) {
1469 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
1470 } }
1471 DISPLAYLEVEL(3, "OK \n");
1472
1473 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
1474 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
1475 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
1476 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
1477 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
1478 compressedBuffer, compressedBufferSize,
1479 CNBuffer, CNBuffSize) );
1480 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1481
1482 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
1483 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1484 if (r != CNBuffSize) goto _output_error; }
1485 DISPLAYLEVEL(3, "OK \n");
1486
1487 ZSTD_freeCCtx(mtctx);
1488 }
1489
1490 DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++);
1491 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1492 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1493 char out[32];
1494 if (cctx == NULL || dctx == NULL) goto _output_error;
1495 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
1496 CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
1497 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
1498
1499 CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
1500 { char const* outPtr = out;
1501 ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
1502 ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
1503 size_t dSize;
1504 CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) );
1505 if (dSize != 0) goto _output_error;
1506 }
1507
1508 ZSTD_freeDCtx(dctx);
1509 ZSTD_freeCCtx(cctx);
1510 }
1511
1512 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
1513 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1514 size_t cSize1, cSize2;
1515 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
1516 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
1517 cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1518 CHECK(cSize1);
1519 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_lcm_uncompressed) );
1520 cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1521 CHECK(cSize2);
1522 CHECK_LT(cSize1, cSize2);
1523 ZSTD_freeCCtx(cctx);
1524 }
1525 DISPLAYLEVEL(3, "OK \n");
1526
1527 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++)
1528 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1529 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
1530 * ZSTDMT is forced to not take the shortcut.
1531 */
1532 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
1533 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
1534 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
1535 CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
1536 ZSTD_freeCCtx(cctx);
1537 }
1538 DISPLAYLEVEL(3, "OK \n");
1539
1540 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
1541 { ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
1542 int value;
1543 /* Check that the overlap log and job size are unset. */
1544 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
1545 CHECK_EQ(value, 0);
1546 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
1547 CHECK_EQ(value, 0);
1548 /* Set and check the overlap log and job size. */
1549 CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
1550 CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, 2 MB) );
1551 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
1552 CHECK_EQ(value, 5);
1553 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
1554 CHECK_EQ(value, 2 MB);
1555 /* Set the number of workers and check the overlap log and job size. */
1556 CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
1557 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
1558 CHECK_EQ(value, 5);
1559 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
1560 CHECK_EQ(value, 2 MB);
1561 ZSTD_freeCCtxParams(params);
1562
1563 }
1564 DISPLAYLEVEL(3, "OK \n");
1565
1566 /* Simple API multiframe test */
1567 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);
1568 { size_t off = 0;
1569 int i;
1570 int const segs = 4;
1571 /* only use the first half so we don't push against size limit of compressedBuffer */
1572 size_t const segSize = (CNBuffSize / 2) / segs;
1573 for (i = 0; i < segs; i++) {
1574 CHECK_NEWV(r, ZSTD_compress(
1575 (BYTE*)compressedBuffer + off, CNBuffSize - off,
1576 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
1577 5) );
1578 off += r;
1579 if (i == segs/2) {
1580 /* insert skippable frame */
1581 const U32 skipLen = 129 KB;
1582 MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START);
1583 MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen);
1584 off += skipLen + ZSTD_SKIPPABLEHEADERSIZE;
1585 }
1586 }
1587 cSize = off;
1588 }
1589 DISPLAYLEVEL(3, "OK \n");
1590
1591 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++);
1592 { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
1593 if (r != CNBuffSize / 2) goto _output_error; }
1594 DISPLAYLEVEL(3, "OK \n");
1595
1596 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
1597 { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize);
1598 if (bound != CNBuffSize / 2) goto _output_error; }
1599 DISPLAYLEVEL(3, "OK \n");
1600
1601 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
1602 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
1603 if (r != CNBuffSize / 2) goto _output_error; }
1604 DISPLAYLEVEL(3, "OK \n");
1605
1606 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1607 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
1608 DISPLAYLEVEL(3, "OK \n");
1609
1610 /* Dictionary and CCtx Duplication tests */
1611 { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
1612 ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
1613 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1614 static const size_t dictSize = 551;
1615 assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
1616
1617 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
1618 { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
1619 if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
1620 DISPLAYLEVEL(3, "OK \n");
1621
1622 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
1623 CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
1624 CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
1625 DISPLAYLEVEL(3, "OK \n");
1626
1627 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
1628 cSize = 0;
1629 CHECKPLUS(r, ZSTD_compressEnd(ctxOrig,
1630 compressedBuffer, compressedBufferSize,
1631 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
1632 cSize += r);
1633 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1634
1635 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
1636 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
1637 decodedBuffer, CNBuffSize,
1638 compressedBuffer, cSize,
1639 CNBuffer, dictSize),
1640 if (r != CNBuffSize - dictSize) goto _output_error);
1641 DISPLAYLEVEL(3, "OK \n");
1642
1643 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
1644 { size_t const cSizeOrig = cSize;
1645 cSize = 0;
1646 CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated,
1647 compressedBuffer, compressedBufferSize,
1648 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
1649 cSize += r);
1650 if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
1651 }
1652 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1653
1654 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
1655 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
1656 decodedBuffer, CNBuffSize,
1657 compressedBuffer, cSize,
1658 CNBuffer, dictSize),
1659 if (r != CNBuffSize - dictSize) goto _output_error);
1660 DISPLAYLEVEL(3, "OK \n");
1661
1662 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++);
1663 { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
1664 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
1665 if (r != CNBuffSize - dictSize) goto _output_error;
1666 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict));
1667 ZSTD_freeDDict(ddict);
1668 }
1669
1670 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
1671 { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
1672 void* const ddictBuffer = malloc(ddictBufferSize);
1673 if (ddictBuffer == NULL) goto _output_error;
1674 { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
1675 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
1676 if (r != CNBuffSize - dictSize) goto _output_error;
1677 }
1678 free(ddictBuffer);
1679 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize);
1680 }
1681
1682 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
1683 { size_t const testSize = CNBuffSize / 3;
1684 { ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize);
1685 p.fParams.contentSizeFlag = 1;
1686 CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
1687 }
1688 CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
1689
1690 CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
1691 (const char*)CNBuffer + dictSize, testSize) );
1692 { ZSTD_frameHeader zfh;
1693 if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
1694 if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
1695 } }
1696 DISPLAYLEVEL(3, "OK \n");
1697
1698 if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
1699 size_t const flatdictSize = 22 KB;
1700 size_t const contentSize = 9 KB;
1701 const void* const dict = (const char*)CNBuffer;
1702 const void* const contentStart = (const char*)dict + flatdictSize;
1703 size_t const target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
1704 3770, 3770, 3770, 3750, 3750,
1705 3742, 3670, 3670, 3660, 3660,
1706 3660, 3660, 3660, 3660, 3660,
1707 3660, 3660, 3660 };
1708 size_t const target_wdict_cSize[22+1] = { 2830, 2890, 2890, 2820, 2940,
1709 2950, 2950, 2921, 2900, 2891,
1710 2910, 2910, 2910, 2770, 2760,
1711 2750, 2750, 2750, 2750, 2750,
1712 2750, 2750, 2750 };
1713 int l = 1;
1714 int const maxLevel = ZSTD_maxCLevel();
1715
1716 DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
1717 assert(maxLevel == 22);
1718 RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
1719 DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n", XXH64(contentStart, contentSize, 0), XXH64(dict, flatdictSize, 0));
1720
1721 for ( ; l <= maxLevel; l++) {
1722 size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize,
1723 contentStart, contentSize, l);
1724 if (nodict_cSize > target_nodict_cSize[l]) {
1725 DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n",
1726 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
1727 goto _output_error;
1728 }
1729 DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n",
1730 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
1731 }
1732 for ( l=1 ; l <= maxLevel; l++) {
1733 size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig,
1734 compressedBuffer, compressedBufferSize,
1735 contentStart, contentSize,
1736 dict, flatdictSize,
1737 l);
1738 if (wdict_cSize > target_wdict_cSize[l]) {
1739 DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n",
1740 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
1741 goto _output_error;
1742 }
1743 DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
1744 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
1745 }
1746
1747 DISPLAYLEVEL(4, "compression efficiency tests OK \n");
1748 }
1749
1750 ZSTD_freeCCtx(ctxOrig);
1751 ZSTD_freeCCtx(ctxDuplicated);
1752 ZSTD_freeDCtx(dctx);
1753 }
1754
1755 /* Dictionary and dictBuilder tests */
1756 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1757 size_t const dictBufferCapacity = 16 KB;
1758 void* const dictBuffer = malloc(dictBufferCapacity);
1759 size_t const totalSampleSize = 1 MB;
1760 size_t const sampleUnitSize = 8 KB;
1761 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
1762 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
1763 size_t dictSize;
1764 U32 dictID;
1765 size_t dictHeaderSize;
1766
1767 if (dictBuffer==NULL || samplesSizes==NULL) {
1768 free(dictBuffer);
1769 free(samplesSizes);
1770 goto _output_error;
1771 }
1772
1773 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++);
1774 assert(compressedBufferSize >= totalSampleSize);
1775 { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; }
1776 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
1777 { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
1778 decodedBuffer, samplesSizes, nbSamples);
1779 if (ZDICT_isError(sDictSize)) goto _output_error;
1780 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize);
1781 }
1782
1783 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++);
1784 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
1785 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
1786 CNBuffer, samplesSizes, nbSamples);
1787 if (ZDICT_isError(dictSize)) goto _output_error;
1788 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
1789
1790 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++);
1791 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
1792 { ZDICT_cover_params_t coverParams;
1793 memset(&coverParams, 0, sizeof(coverParams));
1794 coverParams.steps = 8;
1795 coverParams.nbThreads = 4;
1796 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
1797 dictBuffer, dictBufferCapacity,
1798 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
1799 &coverParams);
1800 if (ZDICT_isError(dictSize)) goto _output_error;
1801 }
1802 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
1803
1804 DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++);
1805 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
1806 { ZDICT_cover_params_t coverParams;
1807 memset(&coverParams, 0, sizeof(coverParams));
1808 coverParams.steps = 8;
1809 coverParams.nbThreads = 4;
1810 coverParams.shrinkDict = 1;
1811 coverParams.shrinkDictMaxRegression = 1;
1812 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
1813 dictBuffer, dictBufferCapacity,
1814 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
1815 &coverParams);
1816 if (ZDICT_isError(dictSize)) goto _output_error;
1817 }
1818 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
1819
1820 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++);
1821 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
1822 { ZDICT_fastCover_params_t fastCoverParams;
1823 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
1824 fastCoverParams.steps = 8;
1825 fastCoverParams.nbThreads = 4;
1826 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
1827 dictBuffer, dictBufferCapacity,
1828 CNBuffer, samplesSizes, nbSamples,
1829 &fastCoverParams);
1830 if (ZDICT_isError(dictSize)) goto _output_error;
1831 }
1832 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
1833
1834 DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++);
1835 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
1836 { ZDICT_fastCover_params_t fastCoverParams;
1837 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
1838 fastCoverParams.steps = 8;
1839 fastCoverParams.nbThreads = 4;
1840 fastCoverParams.shrinkDict = 1;
1841 fastCoverParams.shrinkDictMaxRegression = 1;
1842 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
1843 dictBuffer, dictBufferCapacity,
1844 CNBuffer, samplesSizes, nbSamples,
1845 &fastCoverParams);
1846 if (ZDICT_isError(dictSize)) goto _output_error;
1847 }
1848 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
1849
1850 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
1851 dictID = ZDICT_getDictID(dictBuffer, dictSize);
1852 if (dictID==0) goto _output_error;
1853 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
1854
1855 DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++);
1856 dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize);
1857 if (dictHeaderSize==0) goto _output_error;
1858 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
1859
1860 DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
1861 { unsigned char const dictBufferFixed[144] = { 0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
1862 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1863 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
1864 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
1865 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
1866 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
1867 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
1868 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
1869 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
1870 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
1871 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
1872 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69 };
1873 dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, 144);
1874 if (dictHeaderSize != 115) goto _output_error;
1875 }
1876 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
1877
1878 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
1879 cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
1880 CNBuffer, CNBuffSize,
1881 dictBuffer, dictSize, 4);
1882 if (ZSTD_isError(cSize)) goto _output_error;
1883 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1884
1885 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++);
1886 { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
1887 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
1888 }
1889 DISPLAYLEVEL(3, "OK \n");
1890
1891 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
1892 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1893 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
1894 }
1895 DISPLAYLEVEL(3, "OK \n");
1896
1897 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
1898 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
1899 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
1900 decodedBuffer, CNBuffSize,
1901 compressedBuffer, cSize,
1902 dictBuffer, dictSize),
1903 if (r != CNBuffSize) goto _output_error);
1904 ZSTD_freeDCtx(dctx);
1905 }
1906 DISPLAYLEVEL(3, "OK \n");
1907
1908 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
1909 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
1910 size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef);
1911 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize);
1912 }
1913
1914 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
1915 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
1916 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
1917 ZSTD_dlm_byRef, ZSTD_dct_auto,
1918 cParams, ZSTD_defaultCMem);
1919 assert(cdict != NULL);
1920 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict));
1921 assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict));
1922 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
1923 CNBuffer, CNBuffSize, cdict);
1924 ZSTD_freeCDict(cdict);
1925 if (ZSTD_isError(cSize)) goto _output_error;
1926 }
1927 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1928
1929 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
1930 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1931 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
1932 }
1933 DISPLAYLEVEL(3, "OK \n");
1934
1935 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
1936 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
1937 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
1938 decodedBuffer, CNBuffSize,
1939 compressedBuffer, cSize,
1940 dictBuffer, dictSize),
1941 if (r != CNBuffSize) goto _output_error);
1942 ZSTD_freeDCtx(dctx);
1943 }
1944 DISPLAYLEVEL(3, "OK \n");
1945
1946 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
1947 { int const maxLevel = ZSTD_maxCLevel();
1948 int level;
1949 for (level = 1; level <= maxLevel; ++level) {
1950 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize);
1951 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
1952 void* const cdictBuffer = malloc(cdictSize);
1953 if (cdictBuffer==NULL) goto _output_error;
1954 { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict(
1955 cdictBuffer, cdictSize,
1956 dictBuffer, dictSize,
1957 ZSTD_dlm_byCopy, ZSTD_dct_auto,
1958 cParams);
1959 if (cdict == NULL) {
1960 DISPLAY("ZSTD_initStaticCDict failed ");
1961 goto _output_error;
1962 }
1963 cSize = ZSTD_compress_usingCDict(cctx,
1964 compressedBuffer, compressedBufferSize,
1965 CNBuffer, MIN(10 KB, CNBuffSize), cdict);
1966 if (ZSTD_isError(cSize)) {
1967 DISPLAY("ZSTD_compress_usingCDict failed ");
1968 goto _output_error;
1969 } }
1970 free(cdictBuffer);
1971 } }
1972 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1973
1974 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
1975 { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
1976 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
1977 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
1978 assert(cdict != NULL);
1979 cSize = ZSTD_compress_usingCDict_advanced(cctx,
1980 compressedBuffer, compressedBufferSize,
1981 CNBuffer, CNBuffSize,
1982 cdict, fParams);
1983 ZSTD_freeCDict(cdict);
1984 if (ZSTD_isError(cSize)) goto _output_error;
1985 }
1986 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1987
1988 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++);
1989 { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
1990 if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
1991 }
1992 DISPLAYLEVEL(3, "OK (unknown)\n");
1993
1994 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
1995 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1996 assert(dctx != NULL);
1997 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
1998 decodedBuffer, CNBuffSize,
1999 compressedBuffer, cSize,
2000 dictBuffer, dictSize),
2001 if (r != CNBuffSize) goto _output_error);
2002 ZSTD_freeDCtx(dctx);
2003 }
2004 DISPLAYLEVEL(3, "OK \n");
2005
2006 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
2007 { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
2008 p.fParams.noDictIDFlag = 1;
2009 cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
2010 CNBuffer, CNBuffSize,
2011 dictBuffer, dictSize, p);
2012 if (ZSTD_isError(cSize)) goto _output_error;
2013 }
2014 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2015
2016 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2017 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2018 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2019 decodedBuffer, CNBuffSize,
2020 compressedBuffer, cSize,
2021 dictBuffer, dictSize),
2022 if (r != CNBuffSize) goto _output_error);
2023 ZSTD_freeDCtx(dctx);
2024 }
2025 DISPLAYLEVEL(3, "OK \n");
2026
2027 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
2028 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2029 assert(dctx != NULL);
2030 { const size_t ret = ZSTD_decompress_usingDict(
2031 dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
2032 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
2033 if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
2034 goto _output_error;
2035 }
2036 ZSTD_freeDCtx(dctx);
2037 }
2038 DISPLAYLEVEL(3, "OK \n");
2039
2040 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
2041 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2042 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2043 if (cdict==NULL) goto _output_error;
2044 ZSTD_freeCDict(cdict);
2045 }
2046 DISPLAYLEVEL(3, "OK \n");
2047
2048 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
2049 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2050 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2051 if (cdict!=NULL) goto _output_error;
2052 ZSTD_freeCDict(cdict);
2053 }
2054 DISPLAYLEVEL(3, "OK \n");
2055
2056 { char* rawDictBuffer = (char*)malloc(dictSize);
2057 assert(rawDictBuffer);
2058 memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2);
2059 memset(rawDictBuffer + dictSize - 2, 0, 2);
2060 MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY);
2061
2062 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
2063 {
2064 size_t ret;
2065 /* Either operation is allowed to fail, but one must fail. */
2066 ret = ZSTD_CCtx_loadDictionary_advanced(
2067 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
2068 if (!ZSTD_isError(ret)) {
2069 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2070 if (!ZSTD_isError(ret)) goto _output_error;
2071 }
2072 }
2073 DISPLAYLEVEL(3, "OK \n");
2074
2075 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
2076 {
2077 size_t ret;
2078 ret = ZSTD_CCtx_loadDictionary_advanced(
2079 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
2080 if (ZSTD_isError(ret)) goto _output_error;
2081 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2082 if (ZSTD_isError(ret)) goto _output_error;
2083 }
2084 DISPLAYLEVEL(3, "OK \n");
2085
2086 DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++);
2087 { size_t const srcSize = MIN(CNBuffSize, 100);
2088 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2089 /* Force the dictionary to be reloaded in raw content mode */
2090 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad));
2091 CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent));
2092 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
2093 CHECK_Z(cSize);
2094 }
2095 DISPLAYLEVEL(3, "OK \n");
2096
2097 free(rawDictBuffer);
2098 }
2099
2100 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
2101 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
2102 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2103 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2104 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2105 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2106 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2107 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2108 ZSTD_freeCDict(cdict);
2109 }
2110 DISPLAYLEVEL(3, "OK \n");
2111
2112 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++);
2113 {
2114 size_t size1, size2;
2115 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2116 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2117 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2118 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2119 if (ZSTD_isError(size1)) goto _output_error;
2120
2121 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2122 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2123 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2124 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2125 if (ZSTD_isError(size2)) goto _output_error;
2126
2127 if (size1 != size2) goto _output_error;
2128 }
2129 DISPLAYLEVEL(3, "OK \n");
2130
2131 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++);
2132 {
2133 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2134 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2135 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2136 }
2137 DISPLAYLEVEL(3, "OK \n");
2138
2139 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++);
2140 {
2141 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2142 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2143 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2144 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2145 ZSTD_freeCDict(cdict);
2146 }
2147 DISPLAYLEVEL(3, "OK \n");
2148
2149 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++);
2150 {
2151 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2152 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2153 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2154 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2155 ZSTD_freeCDict(cdict);
2156 }
2157 DISPLAYLEVEL(3, "OK \n");
2158
2159 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++);
2160 {
2161 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2162 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2163 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2164 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2165 ZSTD_freeCDict(cdict);
2166 }
2167 DISPLAYLEVEL(3, "OK \n");
2168
2169 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++);
2170 {
2171 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2172 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2173 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2174 }
2175 DISPLAYLEVEL(3, "OK \n");
2176
2177 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++);
2178 {
2179 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2180 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2181 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2182 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2183 ZSTD_freeCDict(cdict);
2184 }
2185 DISPLAYLEVEL(3, "OK \n");
2186
2187 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++);
2188 {
2189 size_t size1, size2;
2190 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2191 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2192 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2193 if (ZSTD_isError(size1)) goto _output_error;
2194
2195 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
2196 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2197 if (ZSTD_isError(size2)) goto _output_error;
2198
2199 if (size1 != size2) goto _output_error;
2200 }
2201 DISPLAYLEVEL(3, "OK \n");
2202
2203 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++);
2204 {
2205 size_t size1, size2;
2206 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2207 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2208 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2209 if (ZSTD_isError(size1)) goto _output_error;
2210
2211 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2212 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2213 if (ZSTD_isError(size2)) goto _output_error;
2214
2215 if (size1 == size2) goto _output_error;
2216 }
2217 DISPLAYLEVEL(3, "OK \n");
2218
2219 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2220 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) );
2221 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2222 CHECK_Z(cSize);
2223 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++);
2224 {
2225 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2226 size_t ret;
2227 /* We should fail to decompress without a dictionary. */
2228 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2229 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2230 if (!ZSTD_isError(ret)) goto _output_error;
2231 /* We should succeed to decompress with the dictionary. */
2232 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2233 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) );
2234 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2235 /* The dictionary should presist across calls. */
2236 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2237 /* When we reset the context the dictionary is cleared. */
2238 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2239 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2240 if (!ZSTD_isError(ret)) goto _output_error;
2241 ZSTD_freeDCtx(dctx);
2242 }
2243 DISPLAYLEVEL(3, "OK \n");
2244
2245 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
2246 {
2247 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2248 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
2249 size_t ret;
2250 /* We should succeed to decompress with the ddict. */
2251 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2252 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
2253 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2254 /* The ddict should presist across calls. */
2255 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2256 /* When we reset the context the ddict is cleared. */
2257 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2258 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2259 if (!ZSTD_isError(ret)) goto _output_error;
2260 ZSTD_freeDCtx(dctx);
2261 ZSTD_freeDDict(ddict);
2262 }
2263 DISPLAYLEVEL(3, "OK \n");
2264
2265 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
2266 {
2267 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2268 size_t ret;
2269 /* We should succeed to decompress with the prefix. */
2270 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2271 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
2272 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2273 /* The prefix should be cleared after the first compression. */
2274 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2275 if (!ZSTD_isError(ret)) goto _output_error;
2276 ZSTD_freeDCtx(dctx);
2277 }
2278 DISPLAYLEVEL(3, "OK \n");
2279
2280 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
2281 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2282 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
2283 CNBuffer, samplesSizes, nbSamples);
2284 if (ZDICT_isError(dictSize)) goto _output_error;
2285 /* Set all the repcodes to non-default */
2286 {
2287 BYTE* dictPtr = (BYTE*)dictBuffer;
2288 BYTE* dictLimit = dictPtr + dictSize - 12;
2289 /* Find the repcodes */
2290 while (dictPtr < dictLimit &&
2291 (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
2292 MEM_readLE32(dictPtr + 8) != 8)) {
2293 ++dictPtr;
2294 }
2295 if (dictPtr >= dictLimit) goto _output_error;
2296 MEM_writeLE32(dictPtr + 0, 10);
2297 MEM_writeLE32(dictPtr + 4, 10);
2298 MEM_writeLE32(dictPtr + 8, 10);
2299 /* Set the last 8 bytes to 'x' */
2300 memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
2301 }
2302 /* The optimal parser checks all the repcodes.
2303 * Make sure at least one is a match >= targetLength so that it is
2304 * immediately chosen. This will make sure that the compressor and
2305 * decompressor agree on at least one of the repcodes.
2306 */
2307 { size_t dSize;
2308 BYTE data[1024];
2309 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2310 ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
2311 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
2312 ZSTD_dlm_byRef, ZSTD_dct_auto,
2313 cParams, ZSTD_defaultCMem);
2314 assert(dctx != NULL); assert(cdict != NULL);
2315 memset(data, 'x', sizeof(data));
2316 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
2317 data, sizeof(data), cdict);
2318 ZSTD_freeCDict(cdict);
2319 if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
2320 dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
2321 if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
2322 if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
2323 ZSTD_freeDCtx(dctx);
2324 }
2325 DISPLAYLEVEL(3, "OK \n");
2326
2327 ZSTD_freeCCtx(cctx);
2328 free(dictBuffer);
2329 free(samplesSizes);
2330 }
2331
2332 /* COVER dictionary builder tests */
2333 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2334 size_t dictSize = 16 KB;
2335 size_t optDictSize = dictSize;
2336 void* dictBuffer = malloc(dictSize);
2337 size_t const totalSampleSize = 1 MB;
2338 size_t const sampleUnitSize = 8 KB;
2339 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
2340 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
2341 U32 seed32 = seed;
2342 ZDICT_cover_params_t params;
2343 U32 dictID;
2344
2345 if (dictBuffer==NULL || samplesSizes==NULL) {
2346 free(dictBuffer);
2347 free(samplesSizes);
2348 goto _output_error;
2349 }
2350
2351 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
2352 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2353 memset(¶ms, 0, sizeof(params));
2354 params.d = 1 + (FUZ_rand(&seed32) % 16);
2355 params.k = params.d + (FUZ_rand(&seed32) % 256);
2356 dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize,
2357 CNBuffer, samplesSizes, nbSamples,
2358 params);
2359 if (ZDICT_isError(dictSize)) goto _output_error;
2360 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2361
2362 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2363 dictID = ZDICT_getDictID(dictBuffer, dictSize);
2364 if (dictID==0) goto _output_error;
2365 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2366
2367 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
2368 memset(¶ms, 0, sizeof(params));
2369 params.steps = 4;
2370 optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize,
2371 CNBuffer, samplesSizes,
2372 nbSamples / 4, ¶ms);
2373 if (ZDICT_isError(optDictSize)) goto _output_error;
2374 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize);
2375
2376 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2377 dictID = ZDICT_getDictID(dictBuffer, optDictSize);
2378 if (dictID==0) goto _output_error;
2379 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2380
2381 ZSTD_freeCCtx(cctx);
2382 free(dictBuffer);
2383 free(samplesSizes);
2384 }
2385
2386 /* Decompression defense tests */
2387 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++);
2388 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
2389 if (!ZSTD_isError(r)) goto _output_error;
2390 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
2391 DISPLAYLEVEL(3, "OK \n");
2392
2393 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++);
2394 ((char*)(CNBuffer))[0] = 1;
2395 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
2396 if (!ZSTD_isError(r)) goto _output_error; }
2397 DISPLAYLEVEL(3, "OK \n");
2398
2399 /* content size verification test */
2400 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++);
2401 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2402 size_t const srcSize = 5000;
2403 size_t const wrongSrcSize = (srcSize + 1000);
2404 ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
2405 params.fParams.contentSizeFlag = 1;
2406 CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
2407 { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
2408 if (!ZSTD_isError(result)) goto _output_error;
2409 if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
2410 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result));
2411 }
2412 ZSTD_freeCCtx(cctx);
2413 }
2414
2415 /* negative compression level test : ensure simple API and advanced API produce same result */
2416 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++);
2417 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2418 size_t const srcSize = CNBuffSize / 5;
2419 int const compressionLevel = -1;
2420
2421 assert(cctx != NULL);
2422 { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize, 0);
2423 size_t const cSize_1pass = ZSTD_compress_advanced(cctx,
2424 compressedBuffer, compressedBufferSize,
2425 CNBuffer, srcSize,
2426 NULL, 0,
2427 params);
2428 if (ZSTD_isError(cSize_1pass)) goto _output_error;
2429
2430 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
2431 { size_t const compressionResult = ZSTD_compress2(cctx,
2432 compressedBuffer, compressedBufferSize,
2433 CNBuffer, srcSize);
2434 DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
2435 if (ZSTD_isError(compressionResult)) goto _output_error;
2436 if (compressionResult != cSize_1pass) goto _output_error;
2437 } }
2438 ZSTD_freeCCtx(cctx);
2439 }
2440 DISPLAYLEVEL(3, "OK \n");
2441
2442 /* parameters order test */
2443 { size_t const inputSize = CNBuffSize / 2;
2444 U64 xxh64;
2445
2446 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2447 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
2448 assert(cctx != NULL);
2449 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
2450 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
2451 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
2452 { size_t const compressedSize = ZSTD_compress2(cctx,
2453 compressedBuffer, ZSTD_compressBound(inputSize),
2454 CNBuffer, inputSize);
2455 CHECK(compressedSize);
2456 cSize = compressedSize;
2457 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
2458 }
2459 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
2460 ZSTD_freeCCtx(cctx);
2461 }
2462
2463 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2464 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
2465 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
2466 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
2467 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
2468 { size_t const result = ZSTD_compress2(cctx,
2469 compressedBuffer, ZSTD_compressBound(inputSize),
2470 CNBuffer, inputSize);
2471 CHECK(result);
2472 if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
2473 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
2474 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
2475 }
2476 ZSTD_freeCCtx(cctx);
2477 }
2478 }
2479
2480 /* advanced parameters for decompression */
2481 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2482 assert(dctx != NULL);
2483
2484 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
2485 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
2486 CHECK(bounds.error);
2487 }
2488 DISPLAYLEVEL(3, "OK \n");
2489
2490 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
2491 { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
2492 if (!ZSTD_isError(sr)) goto _output_error;
2493 }
2494 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
2495 if (!ZSTD_isError(bounds.error)) goto _output_error;
2496 }
2497 DISPLAYLEVEL(3, "OK \n");
2498
2499 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
2500 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
2501 if (!ZSTD_isError(sr)) goto _output_error;
2502 }
2503 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
2504 if (!ZSTD_isError(sr)) goto _output_error;
2505 }
2506 DISPLAYLEVEL(3, "OK \n");
2507
2508 ZSTD_freeDCtx(dctx);
2509 }
2510
2511
2512 /* custom formats tests */
2513 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2514 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2515 size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
2516 assert(dctx != NULL); assert(cctx != NULL);
2517
2518 /* basic block compression */
2519 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
2520 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
2521 { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
2522 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
2523 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
2524 if (result != 0) goto _output_error;
2525 if (in.pos != in.size) goto _output_error;
2526 cSize = out.pos;
2527 }
2528 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
2529
2530 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++);
2531 { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2532 if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
2533 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult));
2534 }
2535
2536 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
2537 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2538 CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
2539 { ZSTD_frameHeader zfh;
2540 size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
2541 if (zfhrt != 0) goto _output_error;
2542 }
2543 /* one shot */
2544 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2545 if (result != inputSize) goto _output_error;
2546 DISPLAYLEVEL(3, "one-shot OK, ");
2547 }
2548 /* streaming */
2549 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
2550 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
2551 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
2552 if (result != 0) goto _output_error;
2553 if (in.pos != in.size) goto _output_error;
2554 if (out.pos != inputSize) goto _output_error;
2555 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
2556 }
2557
2558 /* basic block compression */
2559 DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
2560 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
2561 { ZSTD_inBuffer in = { CNBuffer, 0, 0 };
2562 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
2563 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
2564 if (result != 0) goto _output_error;
2565 if (in.pos != in.size) goto _output_error;
2566 cSize = out.pos;
2567 }
2568 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
2569
2570 DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
2571 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2572 CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
2573 /* one shot */
2574 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2575 if (result != 0) goto _output_error;
2576 DISPLAYLEVEL(3, "one-shot OK, ");
2577 }
2578 /* streaming */
2579 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
2580 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
2581 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
2582 if (result != 0) goto _output_error;
2583 if (in.pos != in.size) goto _output_error;
2584 if (out.pos != 0) goto _output_error;
2585 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
2586 }
2587
2588 ZSTD_freeCCtx(cctx);
2589 ZSTD_freeDCtx(dctx);
2590 }
2591
2592 DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++);
2593 {
2594 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2595 /* Attempt to future proof this to new parameters. */
2596 int const maxParam = 2000;
2597 int param;
2598 if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error;
2599 for (param = 0; param < maxParam; ++param) {
2600 ZSTD_dParameter dParam = (ZSTD_dParameter)param;
2601 ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam);
2602 int value1;
2603 int value2;
2604 int check;
2605 if (ZSTD_isError(bounds.error))
2606 continue;
2607 CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
2608 value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
2609 CHECK(ZSTD_DCtx_setParameter(dctx, dParam, value2));
2610 CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
2611 if (check != value2) goto _output_error;
2612 CHECK(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
2613 CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
2614 if (check != value1) goto _output_error;
2615 }
2616 ZSTD_freeDCtx(dctx);
2617 }
2618 DISPLAYLEVEL(3, "OK \n");
2619
2620 /* block API tests */
2621 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2622 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2623 static const size_t dictSize = 65 KB;
2624 static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
2625 size_t cSize2;
2626 assert(cctx != NULL); assert(dctx != NULL);
2627
2628 /* basic block compression */
2629 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
2630 CHECK( ZSTD_compressBegin(cctx, 5) );
2631 CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
2632 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
2633 DISPLAYLEVEL(3, "OK \n");
2634
2635 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
2636 CHECK( ZSTD_decompressBegin(dctx) );
2637 { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2638 if (r != blockSize) goto _output_error; }
2639 DISPLAYLEVEL(3, "OK \n");
2640
2641 /* very long stream of block compression */
2642 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
2643 CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
2644 CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
2645 { U64 const toCompress = 5000000000ULL; /* > 4 GB */
2646 U64 compressed = 0;
2647 while (compressed < toCompress) {
2648 size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
2649 assert(blockCSize != 0);
2650 if (ZSTD_isError(blockCSize)) goto _output_error;
2651 compressed += blockCSize;
2652 } }
2653 DISPLAYLEVEL(3, "OK \n");
2654
2655 /* dictionary block compression */
2656 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
2657 CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
2658 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
2659 RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
2660 { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
2661 assert(r == 0); /* non-compressible block */ }
2662 memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */
2663 CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
2664 (char*)CNBuffer+dictSize+2*blockSize, blockSize));
2665 DISPLAYLEVEL(3, "OK \n");
2666
2667 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
2668 CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
2669 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
2670 if (r != blockSize) {
2671 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
2672 goto _output_error;
2673 } }
2674 memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize);
2675 ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
2676 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
2677 if (r != blockSize) {
2678 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
2679 goto _output_error;
2680 } }
2681 assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */
2682 DISPLAYLEVEL(3, "OK \n");
2683
2684 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
2685 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
2686 if (cdict==NULL) goto _output_error;
2687 CHECK( ZSTD_compressBegin_usingCDict(cctx, cdict) );
2688 CHECK( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
2689 ZSTD_freeCDict(cdict);
2690 }
2691 DISPLAYLEVEL(3, "OK \n");
2692
2693 ZSTD_freeCCtx(cctx);
2694 ZSTD_freeDCtx(dctx);
2695 }
2696
2697 /* long rle test */
2698 { size_t sampleSize = 0;
2699 size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
2700 DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
2701 memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
2702 sampleSize += 256 KB - 1;
2703 memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
2704 sampleSize += 96 KB;
2705 cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
2706 if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
2707 { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
2708 if (regenSize!=sampleSize) goto _output_error; }
2709 DISPLAYLEVEL(3, "OK \n");
2710 }
2711
2712 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++);
2713 {
2714 size_t srcSize = 150 KB;
2715 BYTE* src = (BYTE*)CNBuffer;
2716 BYTE* decoded = (BYTE*)compressedBuffer;
2717
2718 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2719 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
2720 size_t seqsSize;
2721
2722 if (seqs == NULL) goto _output_error;
2723 assert(cctx != NULL);
2724 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
2725 /* Populate src with random data */
2726 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
2727
2728 /* Test with block delimiters roundtrip */
2729 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
2730 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters);
2731 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
2732
2733 /* Test no block delimiters roundtrip */
2734 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
2735 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters);
2736 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
2737
2738 ZSTD_freeCCtx(cctx);
2739 free(seqs);
2740 }
2741 DISPLAYLEVEL(3, "OK \n");
2742
2743 DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
2744 {
2745 size_t srcSize = 500 KB;
2746 BYTE* src = (BYTE*)CNBuffer;
2747 BYTE* dst = (BYTE*)compressedBuffer;
2748 size_t dstSize = ZSTD_compressBound(srcSize);
2749 size_t decompressSize = srcSize;
2750 char* decompressBuffer = (char*)malloc(decompressSize);
2751 size_t compressedSize;
2752 size_t dSize;
2753
2754 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2755 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
2756 size_t seqsSize;
2757
2758 if (seqs == NULL) goto _output_error;
2759 assert(cctx != NULL);
2760
2761 /* Populate src with random data */
2762 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
2763
2764 /* Test with block delimiters roundtrip */
2765 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
2766 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2767 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
2768 compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize);
2769 if (ZSTD_isError(compressedSize)) {
2770 DISPLAY("Error in sequence compression with block delims\n");
2771 goto _output_error;
2772 }
2773 dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
2774 if (ZSTD_isError(dSize)) {
2775 DISPLAY("Error in sequence compression roundtrip with block delims\n");
2776 goto _output_error;
2777 }
2778 assert(!memcmp(decompressBuffer, src, srcSize));
2779
2780 /* Test with no block delimiters roundtrip */
2781 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
2782 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2783 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
2784 compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize);
2785 if (ZSTD_isError(compressedSize)) {
2786 DISPLAY("Error in sequence compression with no block delims\n");
2787 goto _output_error;
2788 }
2789 dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
2790 if (ZSTD_isError(dSize)) {
2791 DISPLAY("Error in sequence compression roundtrip with no block delims\n");
2792 goto _output_error;
2793 }
2794 assert(!memcmp(decompressBuffer, src, srcSize));
2795
2796 ZSTD_freeCCtx(cctx);
2797 free(decompressBuffer);
2798 free(seqs);
2799 }
2800 DISPLAYLEVEL(3, "OK \n");
2801
2802 /* Multiple blocks of zeros test */
2803 #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
2804 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
2805 memset(CNBuffer, 0, LONGZEROSLENGTH);
2806 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) );
2807 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100);
2808
2809 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH);
2810 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) );
2811 if (r != LONGZEROSLENGTH) goto _output_error; }
2812 DISPLAYLEVEL(3, "OK \n");
2813
2814 /* All zeroes test (test bug #137) */
2815 #define ZEROESLENGTH 100
2816 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
2817 memset(CNBuffer, 0, ZEROESLENGTH);
2818 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
2819 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100);
2820
2821 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
2822 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
2823 if (r != ZEROESLENGTH) goto _output_error; }
2824 DISPLAYLEVEL(3, "OK \n");
2825
2826 /* nbSeq limit test */
2827 #define _3BYTESTESTLENGTH 131000
2828 #define NB3BYTESSEQLOG 9
2829 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
2830 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
2831 /* creates a buffer full of 3-bytes sequences */
2832 { BYTE _3BytesSeqs[NB3BYTESSEQ][3];
2833 U32 rSeed = 1;
2834
2835 /* create batch of 3-bytes sequences */
2836 { int i;
2837 for (i=0; i < NB3BYTESSEQ; i++) {
2838 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
2839 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
2840 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
2841 } }
2842
2843 /* randomly fills CNBuffer with prepared 3-bytes sequences */
2844 { int i;
2845 for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
2846 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
2847 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
2848 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
2849 ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
2850 } } }
2851 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++);
2852 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2853 size_t const maxNbSeq = _3BYTESTESTLENGTH / 3;
2854 size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
2855 size_t nbSeq = 1;
2856 while (nbSeq <= maxNbSeq) {
2857 CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
2858 /* Check every sequence for the first 100, then skip more rapidly. */
2859 if (nbSeq < 100) {
2860 ++nbSeq;
2861 } else {
2862 nbSeq += (nbSeq >> 2);
2863 }
2864 }
2865 ZSTD_freeCCtx(cctx);
2866 }
2867 DISPLAYLEVEL(3, "OK \n");
2868
2869 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
2870 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
2871 CNBuffer, _3BYTESTESTLENGTH, 19) );
2872 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
2873
2874 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
2875 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
2876 if (r != _3BYTESTESTLENGTH) goto _output_error; }
2877 DISPLAYLEVEL(3, "OK \n");
2878
2879
2880 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++);
2881 RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
2882 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2883 size_t const bound = ZSTD_compressBound(CNBuffSize);
2884 size_t size = 1;
2885 while (size <= CNBuffSize) {
2886 CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
2887 /* Check every size for the first 100, then skip more rapidly. */
2888 if (size < 100) {
2889 ++size;
2890 } else {
2891 size += (size >> 2);
2892 }
2893 }
2894 ZSTD_freeCCtx(cctx);
2895 }
2896 DISPLAYLEVEL(3, "OK \n");
2897
2898 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
2899 { /* Train a dictionary on low characters */
2900 size_t dictSize = 16 KB;
2901 void* const dictBuffer = malloc(dictSize);
2902 size_t const totalSampleSize = 1 MB;
2903 size_t const sampleUnitSize = 8 KB;
2904 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
2905 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
2906 if (!dictBuffer || !samplesSizes) goto _output_error;
2907 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2908 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
2909 if (ZDICT_isError(dictSize)) goto _output_error;
2910 /* Reverse the characters to make the dictionary ill suited */
2911 { U32 u;
2912 for (u = 0; u < CNBuffSize; ++u) {
2913 ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
2914 }
2915 }
2916 { /* Compress the data */
2917 size_t const inputSize = 500;
2918 size_t const outputSize = ZSTD_compressBound(inputSize);
2919 void* const outputBuffer = malloc(outputSize);
2920 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2921 if (!outputBuffer || !cctx) goto _output_error;
2922 CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
2923 free(outputBuffer);
2924 ZSTD_freeCCtx(cctx);
2925 }
2926
2927 free(dictBuffer);
2928 free(samplesSizes);
2929 }
2930 DISPLAYLEVEL(3, "OK \n");
2931
2932
2933 /* findFrameCompressedSize on skippable frames */
2934 DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++);
2935 { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
2936 size_t const frameSrcSize = 13;
2937 if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
2938 DISPLAYLEVEL(3, "OK \n");
2939
2940 /* error string tests */
2941 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
2942 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
2943 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
2944 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
2945 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
2946 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
2947 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
2948 DISPLAYLEVEL(3, "OK \n");
2949
2950 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
2951 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
2952 {
2953 size_t const size = MIN(128 KB, CNBuffSize);
2954 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2955 ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
2956 ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
2957 ZSTD_frameHeader lgHeader;
2958 ZSTD_frameHeader smHeader;
2959
2960 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
2961 CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
2962 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
2963 CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
2964
2965 if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
2966
2967 ZSTD_freeCDict(smCDict);
2968 ZSTD_freeCDict(lgCDict);
2969 ZSTD_freeCCtx(cctx);
2970 }
2971 DISPLAYLEVEL(3, "OK \n");
2972
2973 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++);
2974 {
2975 short norm[32];
2976 unsigned count[32];
2977 unsigned const tableLog = 5;
2978 size_t const nbSeq = 32;
2979 unsigned const maxSymbolValue = 31;
2980 size_t i;
2981
2982 for (i = 0; i < 32; ++i)
2983 count[i] = 1;
2984 /* Calling FSE_normalizeCount() on a uniform distribution should not
2985 * cause a division by zero.
2986 */
2987 FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1);
2988 }
2989 DISPLAYLEVEL(3, "OK \n");
2990 #ifdef ZSTD_MULTITHREAD
2991 DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++);
2992 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2993 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
2994 size_t const dstSize = ZSTD_compressBound(srcSize);
2995 void* const src = CNBuffer;
2996 void* const dst = compressedBuffer;
2997 void* dict = (void*)malloc(srcSize);
2998
2999 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
3000 RDG_genBuffer(dict, srcSize, compressibility, 0., seed);
3001
3002 /* Make sure there is no ZSTD_MAGIC_NUMBER */
3003 memset(dict, 0, sizeof(U32));
3004
3005 /* something more than 1 */
3006 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
3007 /* lie and claim this is a full dict */
3008 CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict));
3009
3010 { ZSTD_outBuffer out = {dst, dstSize, 0};
3011 ZSTD_inBuffer in = {src, srcSize, 0};
3012 /* should fail because its not a full dict like we said it was */
3013 assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)));
3014 }
3015
3016 ZSTD_freeCCtx(cctx);
3017 free(dict);
3018 }
3019 DISPLAYLEVEL(3, "OK \n");
3020
3021 DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++);
3022 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
3023 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
3024 size_t const dictSize = 10;
3025 size_t const dstSize = ZSTD_compressBound(srcSize);
3026 void* const src = CNBuffer;
3027 void* const dst = compressedBuffer;
3028 void* dict = (void*)malloc(dictSize);
3029
3030 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
3031 RDG_genBuffer(dict, dictSize, compressibility, 0., seed);
3032
3033 /* Make sure there is no ZSTD_MAGIC_NUMBER */
3034 memset(dict, 0, sizeof(U32));
3035
3036 /* Enable MT, LDM, and use refPrefix() for a small dict */
3037 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
3038 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
3039 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
3040
3041 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
3042
3043 ZSTD_freeCCtx(cctx);
3044 free(dict);
3045 }
3046 DISPLAYLEVEL(3, "OK \n");
3047 #endif
3048
3049 _end:
3050 free(CNBuffer);
3051 free(compressedBuffer);
3052 free(decodedBuffer);
3053 return testResult;
3054
3055 _output_error:
3056 testResult = 1;
3057 DISPLAY("Error detected in Unit tests ! \n");
3058 goto _end;
3059 }
3060
longUnitTests(U32 const seed,double compressibility)3061 static int longUnitTests(U32 const seed, double compressibility)
3062 {
3063 size_t const CNBuffSize = 5 MB;
3064 void* const CNBuffer = malloc(CNBuffSize);
3065 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
3066 void* const compressedBuffer = malloc(compressedBufferSize);
3067 void* const decodedBuffer = malloc(CNBuffSize);
3068 int testResult = 0;
3069 unsigned testNb=0;
3070 size_t cSize;
3071
3072 /* Create compressible noise */
3073 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
3074 DISPLAY("Not enough memory, aborting\n");
3075 testResult = 1;
3076 goto _end;
3077 }
3078 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
3079
3080 /* note : this test is rather long, it would be great to find a way to speed up its execution */
3081 DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++);
3082 { int cLevel;
3083 size_t approxIndex = 0;
3084 size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */
3085
3086 /* Provision enough space in a static context so that we can do all
3087 * this without ever reallocating, which would reset the indices. */
3088 size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22);
3089 void* const staticCCtxBuffer = malloc(staticCCtxSize);
3090 ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
3091
3092 /* bump the indices so the following compressions happen at high
3093 * indices. */
3094 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
3095 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
3096 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3097 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
3098 while (approxIndex <= (maxIndex / 4) * 3) {
3099 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
3100 approxIndex += in.pos;
3101 CHECK(in.pos == in.size);
3102 in.pos = 0;
3103 out.pos = 0;
3104 }
3105 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
3106 }
3107
3108 /* spew a bunch of stuff into the table area */
3109 for (cLevel = 1; cLevel <= 22; cLevel++) {
3110 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
3111 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
3112 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3113 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
3114 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
3115 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
3116 approxIndex += in.pos;
3117 }
3118
3119 /* now crank the indices so we overflow */
3120 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
3121 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
3122 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3123 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
3124 while (approxIndex <= maxIndex) {
3125 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
3126 approxIndex += in.pos;
3127 CHECK(in.pos == in.size);
3128 in.pos = 0;
3129 out.pos = 0;
3130 }
3131 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
3132 }
3133
3134 /* do a bunch of compressions again in low indices and ensure we don't
3135 * hit untracked invalid indices */
3136 for (cLevel = 1; cLevel <= 22; cLevel++) {
3137 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
3138 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
3139 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3140 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
3141 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
3142 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
3143 approxIndex += in.pos;
3144 }
3145
3146 free(staticCCtxBuffer);
3147 }
3148 DISPLAYLEVEL(3, "OK \n");
3149
3150 DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++);
3151 {
3152 size_t cSizeLdm;
3153 size_t cSizeNoLdm;
3154 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3155
3156 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed);
3157
3158 /* Enable checksum to verify round trip. */
3159 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3160 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
3161 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
3162
3163 /* Round trip once with ldm. */
3164 cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3165 CHECK_Z(cSizeLdm);
3166 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm));
3167
3168 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3169 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3170 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0));
3171 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
3172
3173 /* Round trip once without ldm. */
3174 cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3175 CHECK_Z(cSizeNoLdm);
3176 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm));
3177
3178 if (cSizeLdm > cSizeNoLdm) {
3179 DISPLAY("Using long mode should not cause regressions for btopt+\n");
3180 testResult = 1;
3181 goto _end;
3182 }
3183
3184 ZSTD_freeCCtx(cctx);
3185 }
3186 DISPLAYLEVEL(3, "OK \n");
3187
3188 DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++);
3189 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3190 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3191 size_t dictSize = CNBuffSize;
3192 void* dict = (void*)malloc(dictSize);
3193 ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams();
3194 ZSTD_dictAttachPref_e const attachPrefs[] = {
3195 ZSTD_dictDefaultAttach,
3196 ZSTD_dictForceAttach,
3197 ZSTD_dictForceCopy,
3198 ZSTD_dictForceLoad,
3199 ZSTD_dictDefaultAttach,
3200 ZSTD_dictForceAttach,
3201 ZSTD_dictForceCopy,
3202 ZSTD_dictForceLoad
3203 };
3204 int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1};
3205 int cLevel;
3206 int i;
3207
3208 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
3209 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
3210
3211 CHECK(cctx_params != NULL);
3212
3213 for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
3214 DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
3215 for (cLevel = 4; cLevel < 13; cLevel++) {
3216 for (i = 0; i < 8; ++i) {
3217 ZSTD_dictAttachPref_e const attachPref = attachPrefs[i];
3218 int const enableDDS = enableDedicatedDictSearch[i];
3219 ZSTD_CDict* cdict;
3220
3221 DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i);
3222
3223 ZSTD_CCtxParams_init(cctx_params, cLevel);
3224 CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS));
3225
3226 cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem);
3227 CHECK(cdict != NULL);
3228
3229 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3230 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, attachPref));
3231
3232 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3233 CHECK_Z(cSize);
3234 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
3235
3236 DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize);
3237
3238 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3239 ZSTD_freeCDict(cdict);
3240 } } }
3241
3242 ZSTD_freeCCtx(cctx);
3243 ZSTD_freeDCtx(dctx);
3244 ZSTD_freeCCtxParams(cctx_params);
3245 free(dict);
3246 }
3247 DISPLAYLEVEL(3, "OK \n");
3248
3249 _end:
3250 free(CNBuffer);
3251 free(compressedBuffer);
3252 free(decodedBuffer);
3253 return testResult;
3254
3255 _output_error:
3256 testResult = 1;
3257 DISPLAY("Error detected in Unit tests ! \n");
3258 goto _end;
3259 }
3260
3261
findDiff(const void * buf1,const void * buf2,size_t max)3262 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
3263 {
3264 const BYTE* b1 = (const BYTE*)buf1;
3265 const BYTE* b2 = (const BYTE*)buf2;
3266 size_t u;
3267 for (u=0; u<max; u++) {
3268 if (b1[u] != b2[u]) break;
3269 }
3270 return u;
3271 }
3272
3273
FUZ_makeParams(ZSTD_compressionParameters cParams,ZSTD_frameParameters fParams)3274 static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
3275 {
3276 ZSTD_parameters params;
3277 params.cParams = cParams;
3278 params.fParams = fParams;
3279 return params;
3280 }
3281
FUZ_rLogLength(U32 * seed,U32 logLength)3282 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
3283 {
3284 size_t const lengthMask = ((size_t)1 << logLength) - 1;
3285 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
3286 }
3287
FUZ_randomLength(U32 * seed,U32 maxLog)3288 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
3289 {
3290 U32 const logLength = FUZ_rand(seed) % maxLog;
3291 return FUZ_rLogLength(seed, logLength);
3292 }
3293
3294 #undef CHECK
3295 #define CHECK(cond, ...) { \
3296 if (cond) { \
3297 DISPLAY("Error => "); \
3298 DISPLAY(__VA_ARGS__); \
3299 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
3300 goto _output_error; \
3301 } }
3302
3303 #undef CHECK_Z
3304 #define CHECK_Z(f) { \
3305 size_t const err = f; \
3306 if (ZSTD_isError(err)) { \
3307 DISPLAY("Error => %s : %s ", \
3308 #f, ZSTD_getErrorName(err)); \
3309 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
3310 goto _output_error; \
3311 } }
3312
3313
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,U32 const maxDurationS,double compressibility,int bigTests)3314 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
3315 {
3316 static const U32 maxSrcLog = 23;
3317 static const U32 maxSampleLog = 22;
3318 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
3319 size_t const dstBufferSize = (size_t)1<<maxSampleLog;
3320 size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
3321 BYTE* cNoiseBuffer[5];
3322 BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
3323 BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
3324 BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
3325 ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
3326 ZSTD_CCtx* const ctx = ZSTD_createCCtx();
3327 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3328 U32 result = 0;
3329 unsigned testNb = 0;
3330 U32 coreSeed = seed;
3331 UTIL_time_t const startClock = UTIL_getTime();
3332 U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
3333 int const cLevelLimiter = bigTests ? 3 : 2;
3334
3335 /* allocation */
3336 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
3337 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
3338 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
3339 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
3340 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
3341 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
3342 || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
3343 "Not enough memory, fuzzer tests cancelled");
3344
3345 /* Create initial samples */
3346 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
3347 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
3348 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
3349 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
3350 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
3351
3352 /* catch up testNb */
3353 for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
3354
3355 /* main test loop */
3356 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
3357 BYTE* srcBuffer; /* jumping pointer */
3358 U32 lseed;
3359 size_t sampleSize, maxTestSize, totalTestSize;
3360 size_t cSize, totalCSize, totalGenSize;
3361 U64 crcOrig;
3362 BYTE* sampleBuffer;
3363 const BYTE* dict;
3364 size_t dictSize;
3365
3366 /* notification */
3367 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
3368 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
3369
3370 FUZ_rand(&coreSeed);
3371 { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
3372
3373 /* srcBuffer selection [0-4] */
3374 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
3375 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
3376 else {
3377 buffNb >>= 3;
3378 if (buffNb & 7) {
3379 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
3380 buffNb = tnb[buffNb >> 3];
3381 } else {
3382 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
3383 buffNb = tnb[buffNb >> 3];
3384 } }
3385 srcBuffer = cNoiseBuffer[buffNb];
3386 }
3387
3388 /* select src segment */
3389 sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
3390
3391 /* create sample buffer (to catch read error with valgrind & sanitizers) */
3392 sampleBuffer = (BYTE*)malloc(sampleSize);
3393 CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
3394 { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
3395 memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
3396 crcOrig = XXH64(sampleBuffer, sampleSize, 0);
3397
3398 /* compression tests */
3399 { int const cLevelPositive = (int)
3400 ( FUZ_rand(&lseed) %
3401 ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) )
3402 + 1;
3403 int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ?
3404 - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */
3405 cLevelPositive;
3406 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel);
3407 cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
3408 CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
3409
3410 /* compression failure test : too small dest buffer */
3411 assert(cSize > 3);
3412 { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;
3413 const size_t tooSmallSize = cSize - missing;
3414 const unsigned endMark = 0x4DC2B1A9;
3415 memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark));
3416 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
3417 testNb, (unsigned)tooSmallSize, (unsigned)missing);
3418 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
3419 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
3420 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck));
3421 CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); }
3422 } }
3423
3424 /* frame header decompression test */
3425 { ZSTD_frameHeader zfh;
3426 CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
3427 CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
3428 }
3429
3430 /* Decompressed size test */
3431 { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
3432 CHECK(rSize != sampleSize, "decompressed size incorrect");
3433 }
3434
3435 /* successful decompression test */
3436 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb);
3437 { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
3438 size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
3439 CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize);
3440 { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
3441 CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize);
3442 } }
3443
3444 free(sampleBuffer); /* no longer useful after this point */
3445
3446 /* truncated src decompression test */
3447 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb);
3448 { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
3449 size_t const tooSmallSize = cSize - missing;
3450 void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
3451 CHECK(cBufferTooSmall == NULL, "not enough memory !");
3452 memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
3453 { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
3454 CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
3455 free(cBufferTooSmall);
3456 }
3457
3458 /* too small dst decompression test */
3459 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb);
3460 if (sampleSize > 3) {
3461 size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
3462 size_t const tooSmallSize = sampleSize - missing;
3463 static const BYTE token = 0xA9;
3464 dstBuffer[tooSmallSize] = token;
3465 { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
3466 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
3467 CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
3468 }
3469
3470 /* noisy src decompression test */
3471 if (cSize > 6) {
3472 /* insert noise into src */
3473 { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
3474 size_t pos = 4; /* preserve magic number (too easy to detect) */
3475 for (;;) {
3476 /* keep some original src */
3477 { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
3478 size_t const mask = (1<<nbBits) - 1;
3479 size_t const skipLength = FUZ_rand(&lseed) & mask;
3480 pos += skipLength;
3481 }
3482 if (pos >= cSize) break;
3483 /* add noise */
3484 { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
3485 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
3486 size_t const mask = (1<<nbBits) - 1;
3487 size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
3488 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
3489 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
3490 memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
3491 pos += noiseLength;
3492 } } }
3493
3494 /* decompress noisy source */
3495 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb);
3496 { U32 const endMark = 0xA9B1C3D6;
3497 memcpy(dstBuffer+sampleSize, &endMark, 4);
3498 { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
3499 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
3500 CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
3501 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize);
3502 }
3503 { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
3504 CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
3505 } } } /* noisy src decompression test */
3506
3507 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
3508 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
3509 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
3510 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
3511 int const cLevel = (FUZ_rand(&lseed) %
3512 (ZSTD_maxCLevel() -
3513 (MAX(testLog, dictLog) / cLevelLimiter))) +
3514 1;
3515 maxTestSize = FUZ_rLogLength(&lseed, testLog);
3516 if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
3517
3518 dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
3519 dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
3520
3521 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
3522 testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize);
3523
3524 if (FUZ_rand(&lseed) & 0xF) {
3525 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
3526 } else {
3527 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
3528 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
3529 !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
3530 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
3531 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
3532 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
3533 }
3534 CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
3535 }
3536
3537 { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
3538 U32 n;
3539 XXH64_state_t xxhState;
3540 XXH64_reset(&xxhState, 0);
3541 for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
3542 size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
3543 size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
3544
3545 if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
3546 if (totalTestSize+segmentSize > maxTestSize) break;
3547
3548 { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
3549 CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
3550 cSize += compressResult;
3551 }
3552 XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
3553 memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
3554 totalTestSize += segmentSize;
3555 }
3556
3557 { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
3558 CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
3559 cSize += flushResult;
3560 }
3561 crcOrig = XXH64_digest(&xxhState);
3562 }
3563
3564 /* streaming decompression test */
3565 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
3566 /* ensure memory requirement is good enough (should always be true) */
3567 { ZSTD_frameHeader zfh;
3568 CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
3569 "ZSTD_getFrameHeader(): error retrieving frame information");
3570 { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
3571 CHECK_Z(roundBuffSize);
3572 CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN),
3573 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)",
3574 (unsigned)roundBuffSize, (unsigned)totalTestSize );
3575 } }
3576 if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
3577 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
3578 totalCSize = 0;
3579 totalGenSize = 0;
3580 while (totalCSize < cSize) {
3581 size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
3582 size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
3583 CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize));
3584 totalGenSize += genSize;
3585 totalCSize += inSize;
3586 }
3587 CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
3588 CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
3589 CHECK (totalCSize != cSize, "compressed data should be fully read")
3590 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
3591 CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
3592 (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize);
3593 }
3594 } /* for ( ; (testNb <= nbTests) */
3595 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
3596
3597 _cleanup:
3598 ZSTD_freeCCtx(refCtx);
3599 ZSTD_freeCCtx(ctx);
3600 ZSTD_freeDCtx(dctx);
3601 free(cNoiseBuffer[0]);
3602 free(cNoiseBuffer[1]);
3603 free(cNoiseBuffer[2]);
3604 free(cNoiseBuffer[3]);
3605 free(cNoiseBuffer[4]);
3606 free(cBuffer);
3607 free(dstBuffer);
3608 free(mirrorBuffer);
3609 return result;
3610
3611 _output_error:
3612 result = 1;
3613 goto _cleanup;
3614 }
3615
3616
3617 /*_*******************************************************
3618 * Command line
3619 *********************************************************/
FUZ_usage(const char * programName)3620 static int FUZ_usage(const char* programName)
3621 {
3622 DISPLAY( "Usage :\n");
3623 DISPLAY( " %s [args]\n", programName);
3624 DISPLAY( "\n");
3625 DISPLAY( "Arguments :\n");
3626 DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault);
3627 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
3628 DISPLAY( " -s# : Select seed (default:prompt user)\n");
3629 DISPLAY( " -t# : Select starting test number (default:0)\n");
3630 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default);
3631 DISPLAY( " -v : verbose\n");
3632 DISPLAY( " -p : pause at the end\n");
3633 DISPLAY( " -h : display help and exit\n");
3634 return 0;
3635 }
3636
3637 /*! readU32FromChar() :
3638 @return : unsigned integer value read from input in `char` format
3639 allows and interprets K, KB, KiB, M, MB and MiB suffix.
3640 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
3641 Note : function result can overflow if digit string > MAX_UINT */
readU32FromChar(const char ** stringPtr)3642 static unsigned readU32FromChar(const char** stringPtr)
3643 {
3644 unsigned result = 0;
3645 while ((**stringPtr >='0') && (**stringPtr <='9'))
3646 result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
3647 if ((**stringPtr=='K') || (**stringPtr=='M')) {
3648 result <<= 10;
3649 if (**stringPtr=='M') result <<= 10;
3650 (*stringPtr)++ ;
3651 if (**stringPtr=='i') (*stringPtr)++;
3652 if (**stringPtr=='B') (*stringPtr)++;
3653 }
3654 return result;
3655 }
3656
3657 /** longCommandWArg() :
3658 * check if *stringPtr is the same as longCommand.
3659 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
3660 * @return 0 and doesn't modify *stringPtr otherwise.
3661 */
longCommandWArg(const char ** stringPtr,const char * longCommand)3662 static int longCommandWArg(const char** stringPtr, const char* longCommand)
3663 {
3664 size_t const comSize = strlen(longCommand);
3665 int const result = !strncmp(*stringPtr, longCommand, comSize);
3666 if (result) *stringPtr += comSize;
3667 return result;
3668 }
3669
main(int argc,const char ** argv)3670 int main(int argc, const char** argv)
3671 {
3672 U32 seed = 0;
3673 int seedset = 0;
3674 int argNb;
3675 int nbTests = nbTestsDefault;
3676 int testNb = 0;
3677 int proba = FUZ_compressibility_default;
3678 double probfloat;
3679 int result = 0;
3680 U32 mainPause = 0;
3681 U32 maxDuration = 0;
3682 int bigTests = 1;
3683 int longTests = 0;
3684 U32 memTestsOnly = 0;
3685 const char* const programName = argv[0];
3686
3687 /* Check command line */
3688 for (argNb=1; argNb<argc; argNb++) {
3689 const char* argument = argv[argNb];
3690 if(!argument) continue; /* Protection if argument empty */
3691
3692 /* Handle commands. Aggregated commands are allowed */
3693 if (argument[0]=='-') {
3694
3695 if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; }
3696
3697 if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
3698 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
3699 if (!strcmp(argument, "--long-tests")) { longTests=1; continue; }
3700 if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; }
3701
3702 argument++;
3703 while (*argument!=0) {
3704 switch(*argument)
3705 {
3706 case 'h':
3707 return FUZ_usage(programName);
3708
3709 case 'v':
3710 argument++;
3711 g_displayLevel++;
3712 break;
3713
3714 case 'q':
3715 argument++;
3716 g_displayLevel--;
3717 break;
3718
3719 case 'p': /* pause at the end */
3720 argument++;
3721 mainPause = 1;
3722 break;
3723
3724 case 'i':
3725 argument++; maxDuration = 0;
3726 nbTests = (int)readU32FromChar(&argument);
3727 break;
3728
3729 case 'T':
3730 argument++;
3731 nbTests = 0;
3732 maxDuration = readU32FromChar(&argument);
3733 if (*argument=='s') argument++; /* seconds */
3734 if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
3735 if (*argument=='n') argument++;
3736 break;
3737
3738 case 's':
3739 argument++;
3740 seedset = 1;
3741 seed = readU32FromChar(&argument);
3742 break;
3743
3744 case 't':
3745 argument++;
3746 testNb = (int)readU32FromChar(&argument);
3747 break;
3748
3749 case 'P': /* compressibility % */
3750 argument++;
3751 proba = (int)readU32FromChar(&argument);
3752 if (proba>100) proba = 100;
3753 break;
3754
3755 default:
3756 return (FUZ_usage(programName), 1);
3757 } } } } /* for (argNb=1; argNb<argc; argNb++) */
3758
3759 /* Get Seed */
3760 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
3761
3762 if (!seedset) {
3763 time_t const t = time(NULL);
3764 U32 const h = XXH32(&t, sizeof(t), 1);
3765 seed = h % 10000;
3766 }
3767
3768 DISPLAY("Seed = %u\n", (unsigned)seed);
3769 if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba);
3770
3771 probfloat = ((double)proba) / 100;
3772
3773 if (memTestsOnly) {
3774 g_displayLevel = MAX(3, g_displayLevel);
3775 return FUZ_mallocTests(seed, probfloat, memTestsOnly);
3776 }
3777
3778 if (nbTests < testNb) nbTests = testNb;
3779
3780 if (testNb==0) {
3781 result = basicUnitTests(0, probfloat); /* constant seed for predictability */
3782
3783 if (!result && longTests) {
3784 result = longUnitTests(0, probfloat);
3785 }
3786 }
3787 if (!result)
3788 result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100, bigTests);
3789 if (mainPause) {
3790 int unused;
3791 DISPLAY("Press Enter \n");
3792 unused = getchar();
3793 (void)unused;
3794 }
3795 return result;
3796 }
3797