1 /*
2 * Copyright (c) Yann Collet, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12 /*-************************************
13 * Compiler specific
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
19 #endif
20
21
22 /*-************************************
23 * Includes
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #undef NDEBUG
29 #include <assert.h>
30 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
31 #include "debug.h" /* DEBUG_STATIC_ASSERT */
32 #include "fse.h"
33 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
34 #include "zstd.h" /* ZSTD_VERSION_STRING */
35 #include "zstd_errors.h" /* ZSTD_getErrorCode */
36 #define ZDICT_STATIC_LINKING_ONLY
37 #include "zdict.h" /* ZDICT_trainFromBuffer */
38 #include "mem.h"
39 #include "datagen.h" /* RDG_genBuffer */
40 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
41 #include "xxhash.h" /* XXH64 */
42 #include "util.h"
43 #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
44 /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
45 #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
46 #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */
47
48
49 /*-************************************
50 * Constants
51 **************************************/
52 #define GB *(1U<<30)
53
54 static const int FUZ_compressibility_default = 50;
55 static const int nbTestsDefault = 30000;
56
57
58 /*-************************************
59 * Display Macros
60 **************************************/
61 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
62 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
63 static U32 g_displayLevel = 2;
64
65 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
66 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
67
68 #define DISPLAYUPDATE(l, ...) \
69 if (g_displayLevel>=l) { \
70 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
71 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
72 if (g_displayLevel>=4) fflush(stderr); } \
73 }
74
75
76 /*-*******************************************************
77 * Compile time test
78 *********************************************************/
79 #undef MIN
80 #undef MAX
81 /* Declaring the function, to avoid -Wmissing-prototype */
82 void FUZ_bug976(void);
FUZ_bug976(void)83 void FUZ_bug976(void)
84 { /* these constants shall not depend on MIN() macro */
85 assert(ZSTD_HASHLOG_MAX < 31);
86 assert(ZSTD_CHAINLOG_MAX < 31);
87 }
88
89
90 /*-*******************************************************
91 * Internal functions
92 *********************************************************/
93 #define MIN(a,b) ((a)<(b)?(a):(b))
94 #define MAX(a,b) ((a)>(b)?(a):(b))
95
96 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(U32 * src)97 static U32 FUZ_rand(U32* src)
98 {
99 static const U32 prime1 = 2654435761U;
100 static const U32 prime2 = 2246822519U;
101 U32 rand32 = *src;
102 rand32 *= prime1;
103 rand32 += prime2;
104 rand32 = FUZ_rotl32(rand32, 13);
105 *src = rand32;
106 return rand32 >> 5;
107 }
108
FUZ_highbit32(U32 v32)109 static U32 FUZ_highbit32(U32 v32)
110 {
111 unsigned nbBits = 0;
112 if (v32==0) return 0;
113 while (v32) v32 >>= 1, nbBits++;
114 return nbBits;
115 }
116
117
118 /*=============================================
119 * Test macros
120 =============================================*/
121 #define CHECK_Z(f) { \
122 size_t const err = f; \
123 if (ZSTD_isError(err)) { \
124 DISPLAY("Error => %s : %s ", \
125 #f, ZSTD_getErrorName(err)); \
126 exit(1); \
127 } }
128
129 #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; }
130 #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
131 #define CHECK(fn) { CHECK_NEWV(__err, fn); }
132 #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
133
134 #define CHECK_OP(op, lhs, rhs) { \
135 if (!((lhs) op (rhs))) { \
136 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
137 goto _output_error; \
138 } \
139 }
140 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
141 #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
142
143
144 /*=============================================
145 * Memory Tests
146 =============================================*/
147 #if defined(__APPLE__) && defined(__MACH__)
148
149 #include <malloc/malloc.h> /* malloc_size */
150
151 typedef struct {
152 unsigned long long totalMalloc;
153 size_t currentMalloc;
154 size_t peakMalloc;
155 unsigned nbMalloc;
156 unsigned nbFree;
157 } mallocCounter_t;
158
159 static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };
160
FUZ_mallocDebug(void * counter,size_t size)161 static void* FUZ_mallocDebug(void* counter, size_t size)
162 {
163 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
164 void* const ptr = malloc(size);
165 if (ptr==NULL) return NULL;
166 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
167 (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */
168 mcPtr->totalMalloc += size;
169 mcPtr->currentMalloc += size;
170 if (mcPtr->currentMalloc > mcPtr->peakMalloc)
171 mcPtr->peakMalloc = mcPtr->currentMalloc;
172 mcPtr->nbMalloc += 1;
173 return ptr;
174 }
175
FUZ_freeDebug(void * counter,void * address)176 static void FUZ_freeDebug(void* counter, void* address)
177 {
178 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
179 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10));
180 mcPtr->nbFree += 1;
181 mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */
182 free(address);
183 }
184
FUZ_displayMallocStats(mallocCounter_t count)185 static void FUZ_displayMallocStats(mallocCounter_t count)
186 {
187 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n",
188 (unsigned)(count.peakMalloc >> 10),
189 count.nbMalloc,
190 (unsigned)(count.totalMalloc >> 10));
191 }
192
FUZ_mallocTests_internal(unsigned seed,double compressibility,unsigned part,void * inBuffer,size_t inSize,void * outBuffer,size_t outSize)193 static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
194 void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
195 {
196 /* test only played in verbose mode, as they are long */
197 if (g_displayLevel<3) return 0;
198
199 /* Create compressible noise */
200 if (!inBuffer || !outBuffer) {
201 DISPLAY("Not enough memory, aborting\n");
202 exit(1);
203 }
204 RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
205
206 /* simple compression tests */
207 if (part <= 1)
208 { int compressionLevel;
209 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
210 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
211 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
212 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
213 CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
214 ZSTD_freeCCtx(cctx);
215 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
216 FUZ_displayMallocStats(malcount);
217 } }
218
219 /* streaming compression tests */
220 if (part <= 2)
221 { int compressionLevel;
222 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
223 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
224 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
225 ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
226 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
227 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
228 CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
229 CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
230 CHECK_Z( ZSTD_endStream(cstream, &out) );
231 ZSTD_freeCStream(cstream);
232 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
233 FUZ_displayMallocStats(malcount);
234 } }
235
236 /* advanced MT API test */
237 if (part <= 3)
238 { int nbThreads;
239 for (nbThreads=1; nbThreads<=4; nbThreads++) {
240 int compressionLevel;
241 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
242 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
243 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
244 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
245 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
246 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
247 CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
248 ZSTD_freeCCtx(cctx);
249 DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
250 nbThreads, compressionLevel);
251 FUZ_displayMallocStats(malcount);
252 } } }
253
254 /* advanced MT streaming API test */
255 if (part <= 4)
256 { int nbThreads;
257 for (nbThreads=1; nbThreads<=4; nbThreads++) {
258 int compressionLevel;
259 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
260 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
261 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
262 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
263 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
264 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
265 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
266 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
267 CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
268 while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
269 ZSTD_freeCCtx(cctx);
270 DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ",
271 nbThreads, compressionLevel);
272 FUZ_displayMallocStats(malcount);
273 } } }
274
275 return 0;
276 }
277
FUZ_mallocTests(unsigned seed,double compressibility,unsigned part)278 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
279 {
280 size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
281 size_t const outSize = ZSTD_compressBound(inSize);
282 void* const inBuffer = malloc(inSize);
283 void* const outBuffer = malloc(outSize);
284 int result;
285
286 /* Create compressible noise */
287 if (!inBuffer || !outBuffer) {
288 DISPLAY("Not enough memory, aborting \n");
289 exit(1);
290 }
291
292 result = FUZ_mallocTests_internal(seed, compressibility, part,
293 inBuffer, inSize, outBuffer, outSize);
294
295 free(inBuffer);
296 free(outBuffer);
297 return result;
298 }
299
300 #else
301
FUZ_mallocTests(unsigned seed,double compressibility,unsigned part)302 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
303 {
304 (void)seed; (void)compressibility; (void)part;
305 return 0;
306 }
307
308 #endif
309
FUZ_decodeSequences(BYTE * dst,ZSTD_Sequence * seqs,size_t seqsSize,BYTE * src,size_t size,ZSTD_sequenceFormat_e format)310 static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
311 BYTE* src, size_t size, ZSTD_sequenceFormat_e format)
312 {
313 size_t i;
314 size_t j;
315 for(i = 0; i < seqsSize; ++i) {
316 assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size);
317 assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size);
318 if (format == ZSTD_sf_noBlockDelimiters) {
319 assert(seqs[i].matchLength != 0 || seqs[i].offset != 0);
320 }
321
322 memcpy(dst, src, seqs[i].litLength);
323 dst += seqs[i].litLength;
324 src += seqs[i].litLength;
325 size -= seqs[i].litLength;
326
327 if (seqs[i].offset != 0) {
328 for (j = 0; j < seqs[i].matchLength; ++j)
329 dst[j] = dst[j - seqs[i].offset];
330 dst += seqs[i].matchLength;
331 src += seqs[i].matchLength;
332 size -= seqs[i].matchLength;
333 }
334 }
335 if (format == ZSTD_sf_noBlockDelimiters) {
336 memcpy(dst, src, size);
337 }
338 }
339
340 #ifdef ZSTD_MULTITHREAD
341 typedef struct {
342 ZSTD_CCtx* cctx;
343 ZSTD_threadPool* pool;
344 void* CNBuffer;
345 size_t CNBuffSize;
346 void* compressedBuffer;
347 size_t compressedBufferSize;
348 void* decodedBuffer;
349 int err;
350 } threadPoolTests_compressionJob_payload;
351
threadPoolTests_compressionJob(void * payload)352 static void* threadPoolTests_compressionJob(void* payload) {
353 threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload;
354 size_t cSize;
355 if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1;
356 cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize);
357 if (ZSTD_isError(cSize)) args->err = 1;
358 if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1;
359 return payload;
360 }
361
threadPoolTests(void)362 static int threadPoolTests(void) {
363 int testResult = 0;
364 size_t err;
365
366 size_t const CNBuffSize = 5 MB;
367 void* const CNBuffer = malloc(CNBuffSize);
368 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
369 void* const compressedBuffer = malloc(compressedBufferSize);
370 void* const decodedBuffer = malloc(CNBuffSize);
371
372 size_t const kPoolNumThreads = 8;
373
374 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0);
375
376 DISPLAYLEVEL(3, "thread pool test : threadPool re-use roundtrips: ");
377 {
378 ZSTD_CCtx* cctx = ZSTD_createCCtx();
379 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
380
381 size_t nbThreads = 1;
382 for (; nbThreads <= kPoolNumThreads; ++nbThreads) {
383 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
384 ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads);
385 err = ZSTD_CCtx_refThreadPool(cctx, pool);
386 if (ZSTD_isError(err)) {
387 DISPLAYLEVEL(3, "refThreadPool error!\n");
388 ZSTD_freeCCtx(cctx);
389 goto _output_error;
390 }
391 err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
392 if (ZSTD_isError(err)) {
393 DISPLAYLEVEL(3, "Compression error!\n");
394 ZSTD_freeCCtx(cctx);
395 goto _output_error;
396 }
397 err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err);
398 if (ZSTD_isError(err)) {
399 DISPLAYLEVEL(3, "Decompression error!\n");
400 ZSTD_freeCCtx(cctx);
401 goto _output_error;
402 }
403 }
404
405 ZSTD_freeCCtx(cctx);
406 ZSTD_freeThreadPool(pool);
407 }
408 DISPLAYLEVEL(3, "OK \n");
409
410 DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: ");
411 {
412 void* const decodedBuffer2 = malloc(CNBuffSize);
413 void* const compressedBuffer2 = malloc(compressedBufferSize);
414 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
415 ZSTD_CCtx* cctx1 = ZSTD_createCCtx();
416 ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
417
418 ZSTD_pthread_t t1;
419 ZSTD_pthread_t t2;
420 threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize,
421 compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */};
422 threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize,
423 compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */};
424
425 ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2);
426 ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2);
427 ZSTD_CCtx_refThreadPool(cctx1, pool);
428 ZSTD_CCtx_refThreadPool(cctx2, pool);
429
430 ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
431 ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
432 ZSTD_pthread_join(t1, NULL);
433 ZSTD_pthread_join(t2, NULL);
434
435 assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
436 free(decodedBuffer2);
437 free(compressedBuffer2);
438
439 ZSTD_freeThreadPool(pool);
440 ZSTD_freeCCtx(cctx1);
441 ZSTD_freeCCtx(cctx2);
442
443 if (p1.err || p2.err) goto _output_error;
444 }
445 DISPLAYLEVEL(3, "OK \n");
446
447 _end:
448 free(CNBuffer);
449 free(compressedBuffer);
450 free(decodedBuffer);
451 return testResult;
452
453 _output_error:
454 testResult = 1;
455 DISPLAY("Error detected in Unit tests ! \n");
456 goto _end;
457 }
458 #endif /* ZSTD_MULTITHREAD */
459
460 /*=============================================
461 * Unit tests
462 =============================================*/
463
basicUnitTests(U32 const seed,double compressibility)464 static int basicUnitTests(U32 const seed, double compressibility)
465 {
466 size_t const CNBuffSize = 5 MB;
467 void* const CNBuffer = malloc(CNBuffSize);
468 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
469 void* const compressedBuffer = malloc(compressedBufferSize);
470 void* const decodedBuffer = malloc(CNBuffSize);
471 int testResult = 0;
472 unsigned testNb=0;
473 size_t cSize;
474
475 /* Create compressible noise */
476 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
477 DISPLAY("Not enough memory, aborting\n");
478 testResult = 1;
479 goto _end;
480 }
481 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
482
483 /* Basic tests */
484 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++);
485 { const char* errorString = ZSTD_getErrorName(0);
486 DISPLAYLEVEL(3, "OK : %s \n", errorString);
487 }
488
489 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++);
490 { const char* errorString = ZSTD_getErrorName(499);
491 DISPLAYLEVEL(3, "OK : %s \n", errorString);
492 }
493
494 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++);
495 { int const mcl = ZSTD_minCLevel();
496 DISPLAYLEVEL(3, "%i (OK) \n", mcl);
497 }
498
499 DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++);
500 { int const defaultCLevel = ZSTD_defaultCLevel();
501 if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error;
502 DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel);
503 }
504
505 DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
506 { unsigned const vn = ZSTD_versionNumber();
507 DISPLAYLEVEL(3, "%u (OK) \n", vn);
508 }
509
510 DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
511 {
512 ZSTD_compressionParameters params;
513 memset(¶ms, 0, sizeof(params));
514 params.windowLog = 10;
515 params.hashLog = 19;
516 params.chainLog = 19;
517 params = ZSTD_adjustCParams(params, 1000, 100000);
518 if (params.hashLog != 18) goto _output_error;
519 if (params.chainLog != 17) goto _output_error;
520 }
521 DISPLAYLEVEL(3, "OK \n");
522
523 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
524 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
525 if (cctx==NULL) goto _output_error;
526 CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
527 compressedBuffer, compressedBufferSize,
528 CNBuffer, CNBuffSize, 1) );
529 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
530
531 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
532 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
533 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize);
534 }
535 ZSTD_freeCCtx(cctx);
536 }
537
538 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++);
539 {
540 char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
541 size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8);
542 if (!ZSTD_isError(size)) goto _output_error;
543 }
544 DISPLAYLEVEL(3, "OK \n");
545
546 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
547 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
548 if (rSize != CNBuffSize) goto _output_error;
549 }
550 DISPLAYLEVEL(3, "OK \n");
551
552 DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++);
553 { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
554 if (rSize != CNBuffSize) goto _output_error;
555 }
556 DISPLAYLEVEL(3, "OK \n");
557
558 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
559 { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
560 if (rSize != CNBuffSize) goto _output_error;
561 }
562 DISPLAYLEVEL(3, "OK \n");
563
564 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
565 {
566 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
567 if (bound != CNBuffSize) goto _output_error;
568 }
569 DISPLAYLEVEL(3, "OK \n");
570
571 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
572 {
573 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
574 if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
575 }
576 DISPLAYLEVEL(3, "OK \n");
577
578 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
579 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
580 if (r != CNBuffSize) goto _output_error; }
581 DISPLAYLEVEL(3, "OK \n");
582
583 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
584 { size_t u;
585 for (u=0; u<CNBuffSize; u++) {
586 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
587 } }
588 DISPLAYLEVEL(3, "OK \n");
589
590 DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++);
591 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
592 ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 };
593 ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 };
594 if (cctx==NULL) goto _output_error;
595 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */
596 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */
597 ZSTD_freeCCtx(cctx);
598 }
599 DISPLAYLEVEL(3, "OK \n");
600
601 DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++);
602 {
603 ZSTD_parameters params = ZSTD_getParams(3, 0, 0);
604 assert(!ZSTD_checkCParams(params.cParams));
605 }
606 DISPLAYLEVEL(3, "OK \n");
607
608 DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++);
609 {
610 ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
611 assert(dctx != NULL);
612 assert(ZSTD_sizeof_DCtx(dctx) != 0);
613 ZSTD_freeDCtx(dctx);
614 }
615 DISPLAYLEVEL(3, "OK \n");
616
617 DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++);
618 {
619 /* %p takes a void*. In ISO C, it's illegal to cast a function pointer
620 * to a data pointer. (Although in POSIX you're required to be allowed
621 * to do it...) So we have to fall back to our trusty friend memcpy. */
622 unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) =
623 ZSTD_getDictID_fromDDict;
624 ZSTD_DStream* (* const funcptr_createDStream)(
625 ZSTD_customMem customMem) = ZSTD_createDStream_advanced;
626 void (* const funcptr_copyDCtx)(
627 ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx;
628 ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) =
629 ZSTD_nextInputType;
630 const void *voidptr_getDictID;
631 const void *voidptr_createDStream;
632 const void *voidptr_copyDCtx;
633 const void *voidptr_nextInputType;
634 DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID));
635 memcpy(
636 (void*)&voidptr_getDictID,
637 (const void*)&funcptr_getDictID,
638 sizeof(void*));
639 memcpy(
640 (void*)&voidptr_createDStream,
641 (const void*)&funcptr_createDStream,
642 sizeof(void*));
643 memcpy(
644 (void*)&voidptr_copyDCtx,
645 (const void*)&funcptr_copyDCtx,
646 sizeof(void*));
647 memcpy(
648 (void*)&voidptr_nextInputType,
649 (const void*)&funcptr_nextInputType,
650 sizeof(void*));
651 DISPLAYLEVEL(3, "%p ", voidptr_getDictID);
652 DISPLAYLEVEL(3, "%p ", voidptr_createDStream);
653 DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx);
654 DISPLAYLEVEL(3, "%p ", voidptr_nextInputType);
655 }
656 DISPLAYLEVEL(3, ": OK \n");
657
658 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
659 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
660 { size_t const r = ZSTD_decompress_usingDict(dctx,
661 decodedBuffer, CNBuffSize,
662 compressedBuffer, cSize,
663 NULL, 0);
664 if (r != CNBuffSize) goto _output_error;
665 }
666 ZSTD_freeDCtx(dctx);
667 }
668 DISPLAYLEVEL(3, "OK \n");
669
670 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
671 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
672 { size_t const r = ZSTD_decompress_usingDDict(dctx,
673 decodedBuffer, CNBuffSize,
674 compressedBuffer, cSize,
675 NULL);
676 if (r != CNBuffSize) goto _output_error;
677 }
678 ZSTD_freeDCtx(dctx);
679 }
680 DISPLAYLEVEL(3, "OK \n");
681
682 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
683 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
684 if (!ZSTD_isError(r)) goto _output_error;
685 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
686 DISPLAYLEVEL(3, "OK \n");
687
688 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
689 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
690 if (!ZSTD_isError(r)) goto _output_error;
691 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
692 DISPLAYLEVEL(3, "OK \n");
693
694 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++);
695 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize);
696 if (!ZSTD_isError(r)) goto _output_error;
697 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
698 DISPLAYLEVEL(3, "OK \n");
699
700 DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++);
701 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize);
702 if (!ZSTD_isError(r)) goto _output_error;
703 if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; }
704 DISPLAYLEVEL(3, "OK \n");
705
706 DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++);
707 { /* create compressed buffer with checksumming enabled */
708 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
709 if (!cctx) {
710 DISPLAY("Not enough memory, aborting\n");
711 testResult = 1;
712 goto _end;
713 }
714 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
715 CHECK_VAR(cSize, ZSTD_compress2(cctx,
716 compressedBuffer, compressedBufferSize,
717 CNBuffer, CNBuffSize) );
718 ZSTD_freeCCtx(cctx);
719 }
720 { /* copy the compressed buffer and corrupt the checksum */
721 size_t r;
722 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
723 if (!dctx) {
724 DISPLAY("Not enough memory, aborting\n");
725 testResult = 1;
726 goto _end;
727 }
728
729 ((char*)compressedBuffer)[cSize-1] += 1;
730 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
731 if (!ZSTD_isError(r)) goto _output_error;
732 if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error;
733
734 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum));
735 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
736 if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */
737 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
738 if (ZSTD_isError(r)) goto _output_error;
739
740 ZSTD_freeDCtx(dctx);
741 }
742 DISPLAYLEVEL(3, "OK \n");
743
744
745 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
746 { /* create compressed buffer with content size missing */
747 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
748 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
749 CHECK_VAR(cSize, ZSTD_compress2(cctx,
750 compressedBuffer, compressedBufferSize,
751 CNBuffer, CNBuffSize) );
752 ZSTD_freeCCtx(cctx);
753 }
754 { /* ensure frame content size is missing */
755 ZSTD_frameHeader zfh;
756 size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
757 if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
758 }
759 { /* ensure CNBuffSize <= decompressBound */
760 unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
761 if (CNBuffSize > bound) goto _output_error;
762 }
763 DISPLAYLEVEL(3, "OK \n");
764
765 DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++);
766 {
767 size_t const largeFrameSrcSize = 200;
768 size_t const smallFrameSrcSize = 10;
769 size_t const nbFrames = 256;
770
771 size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0;
772 int sizeReduced = 0;
773
774 BYTE* const dst = (BYTE*)compressedBuffer;
775 ZSTD_DCtx* dctx = ZSTD_createDCtx();
776
777 /* create a large frame and then a bunch of small frames */
778 size_t srcSize = ZSTD_compress((void*)dst,
779 compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
780 for (i = 0; i < nbFrames; i++)
781 srcSize += ZSTD_compress((void*)(dst + srcSize),
782 compressedBufferSize - srcSize, CNBuffer,
783 smallFrameSrcSize, 3);
784
785 /* decompressStream and make sure that dctx size was reduced at least once */
786 while (consumed < srcSize) {
787 ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
788 ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0};
789 ZSTD_decompressStream(dctx, &out, &in);
790 consumed += in.pos;
791 produced += out.pos;
792
793 /* success! size was reduced from the previous frame */
794 if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx))
795 sizeReduced = 1;
796
797 prevDCtxSize = ZSTD_sizeof_DCtx(dctx);
798 }
799
800 assert(sizeReduced);
801
802 ZSTD_freeDCtx(dctx);
803 }
804 DISPLAYLEVEL(3, "OK \n");
805
806 {
807 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
808 ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1);
809 ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
810 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
811
812 DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++);
813 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1));
814 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
815 DISPLAYLEVEL(3, "OK \n");
816
817 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++);
818 CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1));
819 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
820 DISPLAYLEVEL(3, "OK \n");
821
822 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++);
823 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict));
824 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
825 DISPLAYLEVEL(3, "OK \n");
826
827 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++);
828 CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params));
829 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
830 DISPLAYLEVEL(3, "OK \n");
831
832 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++);
833 CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams));
834 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
835 DISPLAYLEVEL(3, "OK \n");
836
837 ZSTD_freeCDict(cdict);
838 ZSTD_freeCCtx(cctx);
839 }
840
841 DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
842 {
843 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
844
845 size_t const size = (1U << 10);
846 size_t const dstCapacity = ZSTD_compressBound(size);
847 void* dict = (void*)malloc(size);
848 void* src = (void*)malloc(size);
849 void* dst = (void*)malloc(dstCapacity);
850
851 RDG_genBuffer(dict, size, 0.5, 0.5, seed);
852 RDG_genBuffer(src, size, 0.5, 0.5, seed);
853
854 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
855 assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
856
857 ZSTD_freeCCtx(cctx);
858 free(dict);
859 free(src);
860 free(dst);
861 }
862 DISPLAYLEVEL(3, "OK \n");
863
864 DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
865 {
866 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
867 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
868 void* dict = (void*)malloc(CNBuffSize);
869 int nbWorkers;
870
871 for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) {
872 RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
873 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
874
875 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
876 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
877 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
878 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
879 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
880 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
881 CHECK_Z(cSize);
882 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize));
883 }
884
885 ZSTD_freeCCtx(cctx);
886 ZSTD_freeDCtx(dctx);
887 free(dict);
888 }
889 DISPLAYLEVEL(3, "OK \n");
890
891 DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++);
892 {
893 size_t const testSize = 1024;
894 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
895 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
896 char* dict = (char*)malloc(2 * testSize);
897 int ldmEnabled, level;
898
899 RDG_genBuffer(dict, testSize, 0.5, 0.5, seed);
900 RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
901 memcpy(dict + testSize, CNBuffer, testSize);
902 for (level = 1; level <= 5; ++level) {
903 for (ldmEnabled = 0; ldmEnabled <= 1; ++ldmEnabled) {
904 size_t cSize0;
905 XXH64_hash_t compressedChecksum0;
906
907 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
908 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level));
909 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled));
910 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1));
911
912 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
913 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize);
914 CHECK_Z(cSize);
915 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize));
916
917 cSize0 = cSize;
918 compressedChecksum0 = XXH64(compressedBuffer, cSize, 0);
919
920 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
921 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize);
922 CHECK_Z(cSize);
923
924 if (cSize != cSize0) goto _output_error;
925 if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error;
926 }
927 }
928
929 ZSTD_freeCCtx(cctx);
930 ZSTD_freeDCtx(dctx);
931 free(dict);
932 }
933 DISPLAYLEVEL(3, "OK \n");
934
935 DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
936 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
937 ZSTD_DCtx* dctx = ZSTD_createDCtx();
938 size_t const srcSize = 300 KB;
939 size_t const flushSize = 128 KB + 5;
940 size_t const dstSize = ZSTD_compressBound(srcSize);
941 char* src = (char*)CNBuffer;
942 char* dst = (char*)compressedBuffer;
943
944 ZSTD_outBuffer out = { dst, dstSize, 0 };
945 ZSTD_inBuffer in = { src, flushSize, 0 };
946
947 if (!cctx || !dctx) {
948 DISPLAY("Not enough memory, aborting\n");
949 testResult = 1;
950 goto _end;
951 }
952
953 RDG_genBuffer(src, srcSize, 0.5, 0.5, seed);
954 /* Force an LDM to exist that crosses block boundary into uncompressible block */
955 memcpy(src + 125 KB, src, 3 KB + 5);
956
957 /* Enable MT, LDM, and opt parser */
958 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
959 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
960 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
961 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
962
963 /* Flushes a block of 128 KB and block of 5 bytes */
964 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
965
966 /* Compress the rest */
967 in.size = 300 KB;
968 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
969
970 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos));
971
972 ZSTD_freeCCtx(cctx);
973 ZSTD_freeDCtx(dctx);
974 }
975 DISPLAYLEVEL(3, "OK \n");
976
977 DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++);
978 {
979 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
980 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
981 void* dict = (void*)malloc(CNBuffSize);
982 size_t const kWindowLog = 10;
983 size_t const kWindowSize = (size_t)1 << kWindowLog;
984 size_t const dictSize = kWindowSize * 10;
985 size_t const srcSize1 = kWindowSize / 2;
986 size_t const srcSize2 = kWindowSize * 10;
987
988 if (CNBuffSize < dictSize) goto _output_error;
989
990 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
991 RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed);
992
993 /* Enable checksum to verify round trip. */
994 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
995 /* Disable content size to skip single-pass decompression. */
996 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
997 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
998 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
999 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
1000 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
1001 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
1002 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3));
1003
1004 /* Round trip once with a dictionary. */
1005 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1006 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
1007 CHECK_Z(cSize);
1008 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1009 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
1010 /* Streaming decompression to catch out of bounds offsets. */
1011 {
1012 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1013 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1014 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1015 CHECK_Z(dSize);
1016 if (dSize != 0) goto _output_error;
1017 }
1018
1019 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
1020 /* Round trip once with a dictionary. */
1021 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1022 {
1023 ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
1024 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1025 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1026 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1027 cSize = out.pos;
1028 }
1029 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1030 {
1031 ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
1032 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1033 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1034 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1035 cSize = out.pos;
1036 }
1037 /* Streaming decompression to catch out of bounds offsets. */
1038 {
1039 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1040 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1041 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1042 CHECK_Z(dSize);
1043 if (dSize != 0) goto _output_error;
1044 }
1045
1046 ZSTD_freeCCtx(cctx);
1047 ZSTD_freeDCtx(dctx);
1048 free(dict);
1049 }
1050 DISPLAYLEVEL(3, "OK \n");
1051
1052 /* Note: this test takes 0.5 seconds to run */
1053 DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
1054 {
1055 /* test a big buffer so that ldm can take effect */
1056 size_t const size = 100 MB;
1057 int const windowLog = 27;
1058 size_t const dstSize = ZSTD_compressBound(size);
1059
1060 void* dict = (void*)malloc(size);
1061 void* src = (void*)malloc(size);
1062 void* dst = (void*)malloc(dstSize);
1063 void* recon = (void*)malloc(size);
1064
1065 size_t refPrefixCompressedSize = 0;
1066 size_t refPrefixLdmComrpessedSize = 0;
1067 size_t reconSize = 0;
1068
1069 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1070 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1071
1072 /* make dict and src the same uncompressible data */
1073 RDG_genBuffer(src, size, 0, 0, seed);
1074 memcpy(dict, src, size);
1075 assert(!memcmp(dict, src, size));
1076
1077 /* set level 1 and windowLog to cover src */
1078 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1));
1079 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog));
1080
1081 /* compress on level 1 using just refPrefix and no ldm */
1082 ZSTD_CCtx_refPrefix(cctx, dict, size);
1083 refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1084 assert(!ZSTD_isError(refPrefixCompressedSize));
1085
1086 /* test round trip just refPrefix */
1087 ZSTD_DCtx_refPrefix(dctx, dict, size);
1088 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize);
1089 assert(!ZSTD_isError(reconSize));
1090 assert(reconSize == size);
1091 assert(!memcmp(recon, src, size));
1092
1093 /* compress on level 1 using refPrefix and ldm */
1094 ZSTD_CCtx_refPrefix(cctx, dict, size);;
1095 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1))
1096 refPrefixLdmComrpessedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1097 assert(!ZSTD_isError(refPrefixLdmComrpessedSize));
1098
1099 /* test round trip refPrefix + ldm*/
1100 ZSTD_DCtx_refPrefix(dctx, dict, size);
1101 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmComrpessedSize);
1102 assert(!ZSTD_isError(reconSize));
1103 assert(reconSize == size);
1104 assert(!memcmp(recon, src, size));
1105
1106 /* make sure that refPrefixCompressedSize is significantly greater */
1107 assert(refPrefixCompressedSize > 10 * refPrefixLdmComrpessedSize);
1108 /* make sure the ldm comrpessed size is less than 1% of original */
1109 assert((double)refPrefixLdmComrpessedSize / (double)size < 0.01);
1110
1111 ZSTD_freeDCtx(dctx);
1112 ZSTD_freeCCtx(cctx);
1113 free(recon);
1114 free(dict);
1115 free(src);
1116 free(dst);
1117 }
1118 DISPLAYLEVEL(3, "OK \n");
1119
1120 DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
1121 {
1122 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1123 const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;
1124 size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize);
1125
1126 /* This is the number of bytes to stream before ending. This value
1127 * was obtained by trial and error :/. */
1128
1129 const size_t streamCompressThreshold = 161792;
1130 const size_t streamCompressDelta = 1024;
1131
1132 /* The first 1/5 of the buffer is compressible and the last 4/5 is
1133 * uncompressible. This is an approximation of the type of data
1134 * the fuzzer generated to catch this bug. Streams like this were making
1135 * zstd generate noCompress superblocks (which are larger than the src
1136 * they come from). Do this enough times, and we'll run out of room
1137 * and throw a dstSize_tooSmall error. */
1138
1139 const size_t compressiblePartSize = srcSize/5;
1140 const size_t uncompressiblePartSize = srcSize-compressiblePartSize;
1141 RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed);
1142 RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed);
1143
1144 /* Setting target block size so that superblock is used */
1145
1146 assert(cctx != NULL);
1147 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81);
1148
1149 { size_t read;
1150 for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) {
1151 ZSTD_inBuffer in = {src, streamCompressDelta, 0};
1152 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1153 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1154 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1155 src += streamCompressDelta; srcSize -= streamCompressDelta;
1156 dst += out.pos; dstCapacity -= out.pos;
1157 } }
1158
1159 /* This is trying to catch a dstSize_tooSmall error */
1160
1161 { ZSTD_inBuffer in = {src, srcSize, 0};
1162 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1163 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1164 }
1165 ZSTD_freeCCtx(cctx);
1166 }
1167 DISPLAYLEVEL(3, "OK \n");
1168
1169 DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++);
1170 /* Generate the same data 20 times over */
1171 { size_t const avgChunkSize = CNBuffSize / 20;
1172 size_t b;
1173 for (b = 0; b < CNBuffSize; b += avgChunkSize) {
1174 size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize);
1175 RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed);
1176 } }
1177 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1178 size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1179 size_t const allowedExpansion = (CNBuffSize * 3 / 1000);
1180 size_t superCSize;
1181 CHECK_Z(normalCSize);
1182 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
1183 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000);
1184 superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1185 CHECK_Z(superCSize);
1186 if (superCSize > normalCSize + allowedExpansion) {
1187 DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion);
1188 goto _output_error;
1189 }
1190 ZSTD_freeCCtx(cctx);
1191 }
1192 DISPLAYLEVEL(3, "OK \n");
1193
1194 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed);
1195 DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++)
1196 /* This tests whether or not we leave enough room for the checksum at the end
1197 * of the dst buffer. The bug that motivated this test was found by the
1198 * stream_round_trip fuzzer but this crashes for the same reason and is
1199 * far more compact than re-creating the stream_round_trip fuzzer's code path */
1200 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1201 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64);
1202 assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278)));
1203 ZSTD_freeCCtx(cctx);
1204 }
1205 DISPLAYLEVEL(3, "OK \n");
1206
1207 DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
1208 { int level = -1;
1209 ZSTD_CCtx* cctx = ZSTD_createCCtx();
1210 if (!cctx) goto _output_error;
1211 for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
1212 CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
1213 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
1214 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
1215 }
1216 ZSTD_freeCCtx(cctx);
1217 }
1218 DISPLAYLEVEL(3, "OK \n");
1219
1220 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
1221 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1222 size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
1223 if (ZSTD_isError(r)) goto _output_error;
1224 if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
1225 ZSTD_freeCCtx(cctx);
1226 cSize = r;
1227 }
1228 DISPLAYLEVEL(3, "OK \n");
1229
1230 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++);
1231 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize);
1232 if (ZSTD_isError(r)) goto _output_error;
1233 if (r != 0) goto _output_error;
1234 }
1235 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1236 ZSTD_outBuffer output;
1237 if (cctx==NULL) goto _output_error;
1238 output.dst = compressedBuffer;
1239 output.size = compressedBufferSize;
1240 output.pos = 0;
1241 CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */
1242 CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */
1243 CHECK_Z( ZSTD_endStream(cctx, &output) );
1244 ZSTD_freeCCtx(cctx);
1245 /* single scan decompression */
1246 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos);
1247 if (ZSTD_isError(r)) goto _output_error;
1248 if (r != 0) goto _output_error;
1249 }
1250 /* streaming decompression */
1251 { ZSTD_DCtx* const dstream = ZSTD_createDStream();
1252 ZSTD_inBuffer dinput;
1253 ZSTD_outBuffer doutput;
1254 size_t ipos;
1255 if (dstream==NULL) goto _output_error;
1256 dinput.src = compressedBuffer;
1257 dinput.size = 0;
1258 dinput.pos = 0;
1259 doutput.dst = NULL;
1260 doutput.size = 0;
1261 doutput.pos = 0;
1262 CHECK_Z ( ZSTD_initDStream(dstream) );
1263 for (ipos=1; ipos<=output.pos; ipos++) {
1264 dinput.size = ipos;
1265 CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) );
1266 }
1267 if (doutput.pos != 0) goto _output_error;
1268 ZSTD_freeDStream(dstream);
1269 }
1270 }
1271 DISPLAYLEVEL(3, "OK \n");
1272
1273 DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++);
1274 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1275 ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1276 assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
1277 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
1278 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
1279
1280 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1281 { size_t const inSize = 2* 128 KB;
1282 size_t const outSize = ZSTD_compressBound(inSize);
1283 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
1284 /* will fail if blockSize is not resized */
1285 }
1286 ZSTD_freeCCtx(cctx);
1287 }
1288 DISPLAYLEVEL(3, "OK \n");
1289
1290 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++);
1291 { size_t const sampleSize = 30;
1292 int i;
1293 for (i=0; i<20; i++)
1294 ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */
1295 memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
1296 for (i=1; i<=19; i++) {
1297 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1298 size_t size1, size2;
1299 DISPLAYLEVEL(5, "l%i ", i);
1300 size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1301 CHECK_Z(size1);
1302
1303 size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1304 CHECK_Z(size2);
1305 CHECK_EQ(size1, size2);
1306
1307 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) );
1308 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize);
1309 CHECK_Z(size2);
1310 CHECK_EQ(size1, size2);
1311
1312 size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */
1313 CHECK_Z(size2);
1314 CHECK_EQ(size1, size2);
1315
1316 { ZSTD_inBuffer inb;
1317 ZSTD_outBuffer outb;
1318 inb.src = CNBuffer;
1319 inb.pos = 0;
1320 inb.size = sampleSize;
1321 outb.dst = compressedBuffer;
1322 outb.pos = 0;
1323 outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */
1324 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );
1325 assert(inb.pos == inb.size);
1326 CHECK_EQ(size1, outb.pos);
1327 }
1328
1329 ZSTD_freeCCtx(cctx);
1330 }
1331 }
1332 DISPLAYLEVEL(3, "OK \n");
1333
1334 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++);
1335 { size_t const sampleSize = 1024;
1336 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1337 ZSTD_inBuffer inb;
1338 ZSTD_outBuffer outb;
1339 inb.src = CNBuffer;
1340 inb.pos = 0;
1341 inb.size = 0;
1342 outb.dst = compressedBuffer;
1343 outb.pos = 0;
1344 outb.size = compressedBufferSize;
1345 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) );
1346
1347 inb.size = sampleSize; /* start with something, so that context is already used */
1348 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1349 assert(inb.pos == inb.size);
1350 outb.pos = 0; /* cancel output */
1351
1352 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) );
1353 inb.size = 4; /* too small size : compression will be skipped */
1354 inb.pos = 0;
1355 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1356 assert(inb.pos == inb.size);
1357
1358 inb.size += 5; /* too small size : compression will be skipped */
1359 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1360 assert(inb.pos == inb.size);
1361
1362 inb.size += 11; /* small enough to attempt compression */
1363 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1364 assert(inb.pos == inb.size);
1365
1366 assert(inb.pos < sampleSize);
1367 inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */
1368 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1369 assert(inb.pos == inb.size);
1370 ZSTD_freeCCtx(cctx);
1371 }
1372 DISPLAYLEVEL(3, "OK \n");
1373
1374 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
1375 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1376 ZSTD_outBuffer out = {NULL, 0, 0};
1377 ZSTD_inBuffer in = {NULL, 0, 0};
1378 int value;
1379
1380 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1381 CHECK_EQ(value, 3);
1382 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1383 CHECK_EQ(value, 0);
1384 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
1385 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1386 CHECK_EQ(value, 3);
1387 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1388 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1389 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
1390 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1391 CHECK_EQ(value, 7);
1392 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1393 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1394 /* Start a compression job */
1395 ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1396 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1397 CHECK_EQ(value, 7);
1398 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1399 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1400 /* Reset the CCtx */
1401 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
1402 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1403 CHECK_EQ(value, 7);
1404 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1405 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1406 /* Reset the parameters */
1407 ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
1408 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1409 CHECK_EQ(value, 3);
1410 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1411 CHECK_EQ(value, 0);
1412
1413 ZSTD_freeCCtx(cctx);
1414 }
1415 DISPLAYLEVEL(3, "OK \n");
1416
1417 DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
1418 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1419 ZSTD_outBuffer out = {NULL, 0, 0};
1420 ZSTD_inBuffer in = {NULL, 0, 0};
1421 int value;
1422
1423 /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt),
1424 * we should not modify the actual parameter specified by the user within the CCtx
1425 */
1426 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27));
1427 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt));
1428
1429 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1430 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value));
1431 CHECK_EQ(value, 0);
1432
1433 ZSTD_freeCCtx(cctx);
1434 }
1435 DISPLAYLEVEL(3, "OK \n");
1436
1437 /* this test is really too long, and should be made faster */
1438 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
1439 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1440 ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1441 size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
1442 size_t cnb;
1443 assert(cctx != NULL);
1444 params.fParams.contentSizeFlag = 0;
1445 params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
1446 for (cnb = 0; cnb < nbCompressions; ++cnb) {
1447 DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions);
1448 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1449 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
1450 }
1451 ZSTD_freeCCtx(cctx);
1452 }
1453 DISPLAYLEVEL(3, "OK \n");
1454
1455 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
1456 { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
1457 assert(largeCCtx != NULL);
1458 CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
1459 CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
1460 { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
1461 { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
1462 assert(smallCCtx != NULL);
1463 CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1464 { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
1465 DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
1466 largeCCtxSize>>10, smallCCtxSize>>10);
1467 assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
1468 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
1469 }
1470 ZSTD_freeCCtx(smallCCtx);
1471 }
1472 { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
1473 * currently defined as 128x, but could be adjusted in the future.
1474 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
1475 unsigned u;
1476 for (u=0; u<maxNbAttempts; u++) {
1477 CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1478 if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
1479 }
1480 DISPLAYLEVEL(5, "size down after %u attempts : ", u);
1481 if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
1482 }
1483 }
1484 ZSTD_freeCCtx(largeCCtx);
1485 }
1486 DISPLAYLEVEL(3, "OK \n");
1487
1488 /* Static CCtx tests */
1489 #define STATIC_CCTX_LEVEL 4
1490 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1491 { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
1492 void* const staticCCtxBuffer = malloc(staticCStreamSize);
1493 size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
1494 void* const staticDCtxBuffer = malloc(staticDCtxSize);
1495 DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize);
1496 if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
1497 free(staticCCtxBuffer);
1498 free(staticDCtxBuffer);
1499 DISPLAY("Not enough memory, aborting\n");
1500 testResult = 1;
1501 goto _end;
1502 }
1503 { size_t const smallInSize = 32 KB;
1504 ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0);
1505 size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small);
1506 size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL);
1507 ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize);
1508 ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
1509 DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize);
1510 DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize);
1511 if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
1512 DISPLAYLEVEL(3, "OK \n");
1513
1514 DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++);
1515 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1516 compressedBuffer, compressedBufferSize,
1517 CNBuffer, smallInSize, STATIC_CCTX_LEVEL) );
1518 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1519 (unsigned)cSize, (double)cSize/smallInSize*100);
1520
1521 DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++);
1522 { size_t const r = ZSTD_compressCCtx(staticCCtx,
1523 compressedBuffer, compressedBufferSize,
1524 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL);
1525 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error;
1526 }
1527 DISPLAYLEVEL(3, "OK \n");
1528
1529 DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
1530 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
1531 DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx);
1532 if (staticCCtx == NULL) goto _output_error;
1533 DISPLAYLEVEL(3, "OK \n");
1534
1535 DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++);
1536 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1537 compressedBuffer, compressedBufferSize,
1538 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1539 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1540 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1541
1542 DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++);
1543 { int nbc;
1544 assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */
1545 assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3);
1546 for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) {
1547 CHECK_Z(ZSTD_compressCCtx(staticCCtx,
1548 compressedBuffer, compressedBufferSize,
1549 (char*)CNBuffer + nbc, smallInSize,
1550 STATIC_CCTX_LEVEL) );
1551 } }
1552 DISPLAYLEVEL(3, "OK \n")
1553
1554 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1555 CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) );
1556 DISPLAYLEVEL(3, "OK \n");
1557
1558 DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++);
1559 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1560 compressedBuffer, compressedBufferSize,
1561 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1562 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1563 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1564
1565 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
1566 { size_t const r = ZSTD_decompressDCtx(staticDCtx,
1567 decodedBuffer, CNBuffSize,
1568 compressedBuffer, cSize);
1569 if (r != CNBuffSize) goto _output_error; }
1570 DISPLAYLEVEL(3, "OK \n");
1571
1572 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1573 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
1574 DISPLAYLEVEL(3, "OK \n");
1575
1576 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
1577 { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
1578 if (!ZSTD_isError(r)) goto _output_error; }
1579 DISPLAYLEVEL(3, "OK \n");
1580
1581 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
1582 CHECK( ZSTD_compressBegin(staticCCtx, 1) );
1583 DISPLAYLEVEL(3, "OK \n");
1584
1585 DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
1586 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1587 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1588 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1589 size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */
1590 if (!ZSTD_isError(r)) goto _output_error;
1591 }
1592 DISPLAYLEVEL(3, "OK \n");
1593
1594 DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++);
1595 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize);
1596 assert(staticCCtx != NULL);
1597 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1598 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1599 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1600 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1601 }
1602 DISPLAYLEVEL(3, "OK \n");
1603
1604 DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1);
1605 CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */
1606 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1607 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1608 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1609 }
1610 DISPLAYLEVEL(3, "OK \n");
1611
1612 DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++);
1613 { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
1614 if (!ZSTD_isError(r)) goto _output_error; }
1615 DISPLAYLEVEL(3, "OK \n");
1616
1617 DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++);
1618 CHECK_Z( ZSTD_initDStream(staticDCtx) );
1619 { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
1620 ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
1621 size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
1622 if (!ZSTD_isError(r)) goto _output_error;
1623 }
1624 DISPLAYLEVEL(3, "OK \n");
1625 }
1626 free(staticCCtxBuffer);
1627 free(staticDCtxBuffer);
1628 }
1629
1630 DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++);
1631 { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1);
1632 size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1);
1633 size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1);
1634 size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1);
1635
1636 if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error;
1637 if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error;
1638 }
1639 DISPLAYLEVEL(3, "OK \n");
1640
1641
1642 /* ZSTDMT simple MT compression test */
1643 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
1644 { ZSTD_CCtx* const mtctx = ZSTD_createCCtx();
1645 if (mtctx==NULL) {
1646 DISPLAY("mtctx : not enough memory, aborting \n");
1647 testResult = 1;
1648 goto _end;
1649 }
1650 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
1651 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
1652 DISPLAYLEVEL(3, "OK \n");
1653
1654 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
1655 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
1656 compressedBuffer, compressedBufferSize,
1657 CNBuffer, CNBuffSize) );
1658 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1659
1660 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
1661 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
1662 if (rSize != CNBuffSize) {
1663 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize);
1664 goto _output_error;
1665 } }
1666 DISPLAYLEVEL(3, "OK \n");
1667
1668 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
1669 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1670 if (r != CNBuffSize) goto _output_error; }
1671 DISPLAYLEVEL(3, "OK \n");
1672
1673 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1674 { size_t u;
1675 for (u=0; u<CNBuffSize; u++) {
1676 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
1677 } }
1678 DISPLAYLEVEL(3, "OK \n");
1679
1680 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
1681 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
1682 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
1683 CHECK( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
1684 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
1685 compressedBuffer, compressedBufferSize,
1686 CNBuffer, CNBuffSize) );
1687 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1688
1689 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
1690 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1691 if (r != CNBuffSize) goto _output_error; }
1692 DISPLAYLEVEL(3, "OK \n");
1693
1694 ZSTD_freeCCtx(mtctx);
1695 }
1696
1697 DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++);
1698 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1699 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1700 char out[32];
1701 if (cctx == NULL || dctx == NULL) goto _output_error;
1702 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
1703 CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
1704 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
1705
1706 CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
1707 { char const* outPtr = out;
1708 ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
1709 ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
1710 size_t dSize;
1711 CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) );
1712 if (dSize != 0) goto _output_error;
1713 }
1714
1715 ZSTD_freeDCtx(dctx);
1716 ZSTD_freeCCtx(cctx);
1717 }
1718
1719 DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
1720 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1721 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
1722 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1723 CHECK(cSize);
1724 ZSTD_freeCCtx(cctx);
1725 }
1726 DISPLAYLEVEL(3, "OK \n");
1727
1728 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
1729 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1730 size_t cSize1, cSize2;
1731 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
1732 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
1733 cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1734 CHECK(cSize1);
1735 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
1736 cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1737 CHECK(cSize2);
1738 CHECK_LT(cSize1, cSize2);
1739 ZSTD_freeCCtx(cctx);
1740 }
1741 DISPLAYLEVEL(3, "OK \n");
1742
1743 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++)
1744 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1745 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
1746 * ZSTDMT is forced to not take the shortcut.
1747 */
1748 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
1749 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
1750 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
1751 CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
1752 ZSTD_freeCCtx(cctx);
1753 }
1754 DISPLAYLEVEL(3, "OK \n");
1755
1756 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
1757 { ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
1758 int const jobSize = 512 KB;
1759 int value;
1760 /* Check that the overlap log and job size are unset. */
1761 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
1762 CHECK_EQ(value, 0);
1763 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
1764 CHECK_EQ(value, 0);
1765 /* Set and check the overlap log and job size. */
1766 CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
1767 CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
1768 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
1769 CHECK_EQ(value, 5);
1770 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
1771 CHECK_EQ(value, jobSize);
1772 /* Set the number of workers and check the overlap log and job size. */
1773 CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
1774 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
1775 CHECK_EQ(value, 5);
1776 CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
1777 CHECK_EQ(value, jobSize);
1778 ZSTD_freeCCtxParams(params);
1779 }
1780 DISPLAYLEVEL(3, "OK \n");
1781
1782 /* Simple API multiframe test */
1783 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);
1784 { size_t off = 0;
1785 int i;
1786 int const segs = 4;
1787 /* only use the first half so we don't push against size limit of compressedBuffer */
1788 size_t const segSize = (CNBuffSize / 2) / segs;
1789
1790 const U32 skipLen = 129 KB;
1791 char* const skipBuff = (char*)malloc(skipLen);
1792 assert(skipBuff != NULL);
1793 memset(skipBuff, 0, skipLen);
1794 for (i = 0; i < segs; i++) {
1795 CHECK_NEWV(r, ZSTD_compress(
1796 (BYTE*)compressedBuffer + off, CNBuffSize - off,
1797 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
1798 5) );
1799 off += r;
1800 if (i == segs/2) {
1801 /* insert skippable frame */
1802 size_t const skippableSize =
1803 ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize,
1804 skipBuff, skipLen, seed % 15);
1805 CHECK_Z(skippableSize);
1806 off += skippableSize;
1807 }
1808 }
1809 cSize = off;
1810 free(skipBuff);
1811 }
1812 DISPLAYLEVEL(3, "OK \n");
1813
1814 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++);
1815 { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
1816 if (r != CNBuffSize / 2) goto _output_error; }
1817 DISPLAYLEVEL(3, "OK \n");
1818
1819 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
1820 { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize);
1821 if (bound != CNBuffSize / 2) goto _output_error; }
1822 DISPLAYLEVEL(3, "OK \n");
1823
1824 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
1825 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
1826 if (r != CNBuffSize / 2) goto _output_error; }
1827 DISPLAYLEVEL(3, "OK \n");
1828
1829 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1830 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
1831 DISPLAYLEVEL(3, "OK \n");
1832
1833 /* Simple API skippable frame test */
1834 DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++);
1835 { U32 i;
1836 unsigned readMagic;
1837 unsigned long long receivedSize;
1838 size_t skippableSize;
1839 const U32 skipLen = 129 KB;
1840 char* const skipBuff = (char*)malloc(skipLen);
1841 assert(skipBuff != NULL);
1842 for (i = 0; i < skipLen; i++)
1843 skipBuff[i] = (char) ((seed + i) % 256);
1844 skippableSize = ZSTD_writeSkippableFrame(
1845 compressedBuffer, compressedBufferSize,
1846 skipBuff, skipLen, seed % 15);
1847 CHECK_Z(skippableSize);
1848 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
1849 receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize);
1850 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
1851 CHECK_EQ(seed % 15, readMagic);
1852 if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error;
1853
1854 free(skipBuff);
1855 }
1856 DISPLAYLEVEL(3, "OK \n");
1857
1858 DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++);
1859 {
1860 unsigned readMagic;
1861 unsigned long long receivedSize;
1862 size_t skippableSize;
1863 skippableSize = ZSTD_writeSkippableFrame(
1864 compressedBuffer, compressedBufferSize,
1865 CNBuffer, 0, seed % 15);
1866 CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize);
1867 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
1868 receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize);
1869 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
1870 CHECK_EQ(seed % 15, readMagic);
1871 }
1872 DISPLAYLEVEL(3, "OK \n");
1873
1874 /* Dictionary and CCtx Duplication tests */
1875 { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
1876 ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
1877 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1878 static const size_t dictSize = 551;
1879 assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
1880
1881 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
1882 { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
1883 if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
1884 DISPLAYLEVEL(3, "OK \n");
1885
1886 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
1887 CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
1888 CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
1889 DISPLAYLEVEL(3, "OK \n");
1890
1891 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
1892 cSize = 0;
1893 CHECKPLUS(r, ZSTD_compressEnd(ctxOrig,
1894 compressedBuffer, compressedBufferSize,
1895 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
1896 cSize += r);
1897 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1898
1899 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
1900 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
1901 decodedBuffer, CNBuffSize,
1902 compressedBuffer, cSize,
1903 CNBuffer, dictSize),
1904 if (r != CNBuffSize - dictSize) goto _output_error);
1905 DISPLAYLEVEL(3, "OK \n");
1906
1907 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
1908 { size_t const cSizeOrig = cSize;
1909 cSize = 0;
1910 CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated,
1911 compressedBuffer, compressedBufferSize,
1912 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
1913 cSize += r);
1914 if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
1915 }
1916 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
1917
1918 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
1919 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
1920 decodedBuffer, CNBuffSize,
1921 compressedBuffer, cSize,
1922 CNBuffer, dictSize),
1923 if (r != CNBuffSize - dictSize) goto _output_error);
1924 DISPLAYLEVEL(3, "OK \n");
1925
1926 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++);
1927 { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
1928 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
1929 if (r != CNBuffSize - dictSize) goto _output_error;
1930 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict));
1931 ZSTD_freeDDict(ddict);
1932 }
1933
1934 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
1935 { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
1936 void* const ddictBuffer = malloc(ddictBufferSize);
1937 if (ddictBuffer == NULL) goto _output_error;
1938 { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
1939 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
1940 if (r != CNBuffSize - dictSize) goto _output_error;
1941 }
1942 free(ddictBuffer);
1943 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize);
1944 }
1945
1946 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
1947 { size_t const testSize = CNBuffSize / 3;
1948 CHECK( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
1949 CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
1950
1951 CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
1952 (const char*)CNBuffer + dictSize, testSize) );
1953 { ZSTD_frameHeader zfh;
1954 if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
1955 if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
1956 } }
1957 DISPLAYLEVEL(3, "OK \n");
1958
1959 /* Note : these tests should be replaced by proper regression tests,
1960 * but existing ones do not focus on small data + dictionary + all levels.
1961 */
1962 if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
1963 size_t const flatdictSize = 22 KB;
1964 size_t const contentSize = 9 KB;
1965 const void* const dict = (const char*)CNBuffer;
1966 const void* const contentStart = (const char*)dict + flatdictSize;
1967 /* These upper bounds are generally within a few bytes of the compressed size */
1968 size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
1969 3770, 3770, 3770, 3750, 3750,
1970 3742, 3675, 3674, 3665, 3664,
1971 3663, 3662, 3661, 3660, 3660,
1972 3660, 3660, 3660 };
1973 size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940,
1974 2950, 2950, 2925, 2900, 2891,
1975 2910, 2910, 2910, 2780, 2775,
1976 2765, 2760, 2755, 2754, 2753,
1977 2753, 2753, 2753 };
1978 int l = 1;
1979 int const maxLevel = ZSTD_maxCLevel();
1980 /* clevels with strategies that support rowhash on small inputs */
1981 int rowLevel = 4;
1982 int const rowLevelEnd = 8;
1983
1984 DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
1985 assert(maxLevel == 22);
1986 RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
1987 DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n",
1988 (unsigned long long)XXH64(contentStart, contentSize, 0),
1989 (unsigned long long)XXH64(dict, flatdictSize, 0));
1990
1991 for ( ; l <= maxLevel; l++) {
1992 size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize,
1993 contentStart, contentSize, l);
1994 if (nodict_cSize > target_nodict_cSize[l]) {
1995 DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n",
1996 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
1997 goto _output_error;
1998 }
1999 DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n",
2000 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2001 }
2002 for ( l=1 ; l <= maxLevel; l++) {
2003 size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig,
2004 compressedBuffer, compressedBufferSize,
2005 contentStart, contentSize,
2006 dict, flatdictSize,
2007 l);
2008 if (wdict_cSize > target_wdict_cSize[l]) {
2009 DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n",
2010 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2011 goto _output_error;
2012 }
2013 DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
2014 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2015 }
2016 /* Compression with ZSTD_compress2 and row match finder force enabled.
2017 * Give some slack for force-enabled row matchfinder since we're on a small input (9KB)
2018 */
2019 for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5;
2020 for (l=1 ; l <= maxLevel; l++) {
2021 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2022 size_t nodict_cSize;
2023 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l);
2024 ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable);
2025 nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize,
2026 contentStart, contentSize);
2027 if (nodict_cSize > target_nodict_cSize[l]) {
2028 DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n",
2029 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2030 ZSTD_freeCCtx(cctx);
2031 goto _output_error;
2032 }
2033 DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n",
2034 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2035 ZSTD_freeCCtx(cctx);
2036 }
2037 /* Dict compression with DMS */
2038 for ( l=1 ; l <= maxLevel; l++) {
2039 size_t wdict_cSize;
2040 CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) );
2041 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) );
2042 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) );
2043 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) );
2044 wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize);
2045 if (wdict_cSize > target_wdict_cSize[l]) {
2046 DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n",
2047 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2048 goto _output_error;
2049 }
2050 DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n",
2051 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2052 }
2053
2054 DISPLAYLEVEL(4, "compression efficiency tests OK \n");
2055 }
2056
2057 ZSTD_freeCCtx(ctxOrig);
2058 ZSTD_freeCCtx(ctxDuplicated);
2059 ZSTD_freeDCtx(dctx);
2060 }
2061
2062 /* Dictionary and dictBuilder tests */
2063 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2064 size_t const dictBufferCapacity = 16 KB;
2065 void* const dictBuffer = malloc(dictBufferCapacity);
2066 size_t const totalSampleSize = 1 MB;
2067 size_t const sampleUnitSize = 8 KB;
2068 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
2069 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
2070 size_t dictSize;
2071 U32 dictID;
2072 size_t dictHeaderSize;
2073 size_t dictBufferFixedSize = 144;
2074 unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
2075 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2076 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
2077 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
2078 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2079 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2080 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
2081 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
2082 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
2083 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
2084 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
2085 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69};
2086
2087 if (dictBuffer==NULL || samplesSizes==NULL) {
2088 free(dictBuffer);
2089 free(samplesSizes);
2090 goto _output_error;
2091 }
2092
2093 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++);
2094 assert(compressedBufferSize >= totalSampleSize);
2095 { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; }
2096 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2097 { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2098 decodedBuffer, samplesSizes, nbSamples);
2099 if (ZDICT_isError(sDictSize)) goto _output_error;
2100 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize);
2101 }
2102
2103 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++);
2104 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2105 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2106 CNBuffer, samplesSizes, nbSamples);
2107 if (ZDICT_isError(dictSize)) goto _output_error;
2108 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2109
2110 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++);
2111 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2112 { ZDICT_cover_params_t coverParams;
2113 memset(&coverParams, 0, sizeof(coverParams));
2114 coverParams.steps = 8;
2115 coverParams.nbThreads = 4;
2116 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2117 dictBuffer, dictBufferCapacity,
2118 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2119 &coverParams);
2120 if (ZDICT_isError(dictSize)) goto _output_error;
2121 }
2122 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2123
2124 DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++);
2125 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2126 { ZDICT_cover_params_t coverParams;
2127 memset(&coverParams, 0, sizeof(coverParams));
2128 coverParams.steps = 8;
2129 coverParams.nbThreads = 4;
2130 coverParams.shrinkDict = 1;
2131 coverParams.shrinkDictMaxRegression = 1;
2132 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2133 dictBuffer, dictBufferCapacity,
2134 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2135 &coverParams);
2136 if (ZDICT_isError(dictSize)) goto _output_error;
2137 }
2138 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2139
2140 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++);
2141 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2142 { ZDICT_fastCover_params_t fastCoverParams;
2143 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2144 fastCoverParams.steps = 8;
2145 fastCoverParams.nbThreads = 4;
2146 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2147 dictBuffer, dictBufferCapacity,
2148 CNBuffer, samplesSizes, nbSamples,
2149 &fastCoverParams);
2150 if (ZDICT_isError(dictSize)) goto _output_error;
2151 }
2152 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2153
2154 DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++);
2155 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2156 { ZDICT_fastCover_params_t fastCoverParams;
2157 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2158 fastCoverParams.steps = 8;
2159 fastCoverParams.nbThreads = 4;
2160 fastCoverParams.shrinkDict = 1;
2161 fastCoverParams.shrinkDictMaxRegression = 1;
2162 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2163 dictBuffer, dictBufferCapacity,
2164 CNBuffer, samplesSizes, nbSamples,
2165 &fastCoverParams);
2166 if (ZDICT_isError(dictSize)) goto _output_error;
2167 }
2168 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2169
2170 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2171 dictID = ZDICT_getDictID(dictBuffer, dictSize);
2172 if (dictID==0) goto _output_error;
2173 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2174
2175 DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++);
2176 dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize);
2177 if (dictHeaderSize==0) goto _output_error;
2178 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2179
2180 DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
2181 { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize);
2182 if (dictHeaderSize != 115) goto _output_error;
2183 }
2184 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2185
2186 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
2187 cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
2188 CNBuffer, CNBuffSize,
2189 dictBuffer, dictSize, 4);
2190 if (ZSTD_isError(cSize)) goto _output_error;
2191 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2192
2193 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++);
2194 { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
2195 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2196 }
2197 DISPLAYLEVEL(3, "OK \n");
2198
2199 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2200 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2201 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2202 }
2203 DISPLAYLEVEL(3, "OK \n");
2204
2205 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2206 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2207 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2208 decodedBuffer, CNBuffSize,
2209 compressedBuffer, cSize,
2210 dictBuffer, dictSize),
2211 if (r != CNBuffSize) goto _output_error);
2212 ZSTD_freeDCtx(dctx);
2213 }
2214 DISPLAYLEVEL(3, "OK \n");
2215
2216 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
2217 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2218 size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef);
2219 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize);
2220 }
2221
2222 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
2223 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2224 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
2225 ZSTD_dlm_byRef, ZSTD_dct_auto,
2226 cParams, ZSTD_defaultCMem);
2227 assert(cdict != NULL);
2228 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict));
2229 assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict));
2230 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
2231 CNBuffer, CNBuffSize, cdict);
2232 ZSTD_freeCDict(cdict);
2233 if (ZSTD_isError(cSize)) goto _output_error;
2234 }
2235 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2236
2237 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2238 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2239 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2240 }
2241 DISPLAYLEVEL(3, "OK \n");
2242
2243 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2244 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2245 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2246 decodedBuffer, CNBuffSize,
2247 compressedBuffer, cSize,
2248 dictBuffer, dictSize),
2249 if (r != CNBuffSize) goto _output_error);
2250 ZSTD_freeDCtx(dctx);
2251 }
2252 DISPLAYLEVEL(3, "OK \n");
2253
2254 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
2255 { int const maxLevel = ZSTD_maxCLevel();
2256 int level;
2257 for (level = 1; level <= maxLevel; ++level) {
2258 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize);
2259 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
2260 void* const cdictBuffer = malloc(cdictSize);
2261 if (cdictBuffer==NULL) goto _output_error;
2262 { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict(
2263 cdictBuffer, cdictSize,
2264 dictBuffer, dictSize,
2265 ZSTD_dlm_byCopy, ZSTD_dct_auto,
2266 cParams);
2267 if (cdict == NULL) {
2268 DISPLAY("ZSTD_initStaticCDict failed ");
2269 goto _output_error;
2270 }
2271 cSize = ZSTD_compress_usingCDict(cctx,
2272 compressedBuffer, compressedBufferSize,
2273 CNBuffer, MIN(10 KB, CNBuffSize), cdict);
2274 if (ZSTD_isError(cSize)) {
2275 DISPLAY("ZSTD_compress_usingCDict failed ");
2276 goto _output_error;
2277 } }
2278 free(cdictBuffer);
2279 } }
2280 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2281
2282 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
2283 { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
2284 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2285 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
2286 assert(cdict != NULL);
2287 cSize = ZSTD_compress_usingCDict_advanced(cctx,
2288 compressedBuffer, compressedBufferSize,
2289 CNBuffer, CNBuffSize,
2290 cdict, fParams);
2291 ZSTD_freeCDict(cdict);
2292 if (ZSTD_isError(cSize)) goto _output_error;
2293 }
2294 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2295
2296 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++);
2297 { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2298 if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
2299 }
2300 DISPLAYLEVEL(3, "OK (unknown)\n");
2301
2302 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2303 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2304 assert(dctx != NULL);
2305 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2306 decodedBuffer, CNBuffSize,
2307 compressedBuffer, cSize,
2308 dictBuffer, dictSize),
2309 if (r != CNBuffSize) goto _output_error);
2310 ZSTD_freeDCtx(dctx);
2311 }
2312 DISPLAYLEVEL(3, "OK \n");
2313
2314 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
2315 { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
2316 p.fParams.noDictIDFlag = 1;
2317 cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
2318 CNBuffer, CNBuffSize,
2319 dictBuffer, dictSize, p);
2320 if (ZSTD_isError(cSize)) goto _output_error;
2321 }
2322 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2323
2324 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2325 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2326 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2327 decodedBuffer, CNBuffSize,
2328 compressedBuffer, cSize,
2329 dictBuffer, dictSize),
2330 if (r != CNBuffSize) goto _output_error);
2331 ZSTD_freeDCtx(dctx);
2332 }
2333 DISPLAYLEVEL(3, "OK \n");
2334
2335 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
2336 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2337 assert(dctx != NULL);
2338 { const size_t ret = ZSTD_decompress_usingDict(
2339 dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
2340 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
2341 if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
2342 goto _output_error;
2343 }
2344 ZSTD_freeDCtx(dctx);
2345 }
2346 DISPLAYLEVEL(3, "OK \n");
2347
2348 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
2349 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2350 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2351 if (cdict==NULL) goto _output_error;
2352 ZSTD_freeCDict(cdict);
2353 }
2354 DISPLAYLEVEL(3, "OK \n");
2355
2356 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
2357 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2358 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2359 if (cdict!=NULL) goto _output_error;
2360 ZSTD_freeCDict(cdict);
2361 }
2362 DISPLAYLEVEL(3, "OK \n");
2363
2364 { char* rawDictBuffer = (char*)malloc(dictSize);
2365 assert(rawDictBuffer);
2366 memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2);
2367 memset(rawDictBuffer + dictSize - 2, 0, 2);
2368 MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY);
2369
2370 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
2371 {
2372 size_t ret;
2373 /* Either operation is allowed to fail, but one must fail. */
2374 ret = ZSTD_CCtx_loadDictionary_advanced(
2375 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
2376 if (!ZSTD_isError(ret)) {
2377 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2378 if (!ZSTD_isError(ret)) goto _output_error;
2379 }
2380 }
2381 DISPLAYLEVEL(3, "OK \n");
2382
2383 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
2384 {
2385 size_t ret;
2386 ret = ZSTD_CCtx_loadDictionary_advanced(
2387 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
2388 if (ZSTD_isError(ret)) goto _output_error;
2389 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2390 if (ZSTD_isError(ret)) goto _output_error;
2391 }
2392 DISPLAYLEVEL(3, "OK \n");
2393
2394 DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++);
2395 { size_t const srcSize = MIN(CNBuffSize, 100);
2396 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2397 /* Force the dictionary to be reloaded in raw content mode */
2398 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad));
2399 CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent));
2400 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
2401 CHECK_Z(cSize);
2402 }
2403 DISPLAYLEVEL(3, "OK \n");
2404
2405 free(rawDictBuffer);
2406 }
2407
2408 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
2409 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
2410 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2411 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2412 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2413 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2414 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2415 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2416 ZSTD_freeCDict(cdict);
2417 }
2418 DISPLAYLEVEL(3, "OK \n");
2419
2420 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++);
2421 {
2422 size_t size1, size2;
2423 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2424 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2425 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2426 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2427 if (ZSTD_isError(size1)) goto _output_error;
2428
2429 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2430 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2431 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2432 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2433 if (ZSTD_isError(size2)) goto _output_error;
2434
2435 if (size1 != size2) goto _output_error;
2436 }
2437 DISPLAYLEVEL(3, "OK \n");
2438
2439 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++);
2440 {
2441 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2442 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2443 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2444 }
2445 DISPLAYLEVEL(3, "OK \n");
2446
2447 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++);
2448 {
2449 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2450 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2451 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2452 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2453 ZSTD_freeCDict(cdict);
2454 }
2455 DISPLAYLEVEL(3, "OK \n");
2456
2457 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++);
2458 {
2459 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2460 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2461 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2462 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2463 ZSTD_freeCDict(cdict);
2464 }
2465 DISPLAYLEVEL(3, "OK \n");
2466
2467 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++);
2468 {
2469 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2470 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2471 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2472 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2473 ZSTD_freeCDict(cdict);
2474 }
2475 DISPLAYLEVEL(3, "OK \n");
2476
2477 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++);
2478 {
2479 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2480 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2481 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2482 }
2483 DISPLAYLEVEL(3, "OK \n");
2484
2485 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++);
2486 {
2487 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2488 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2489 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2490 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2491 ZSTD_freeCDict(cdict);
2492 }
2493 DISPLAYLEVEL(3, "OK \n");
2494
2495 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++);
2496 {
2497 size_t size1, size2;
2498 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2499 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2500 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2501 if (ZSTD_isError(size1)) goto _output_error;
2502
2503 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
2504 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2505 if (ZSTD_isError(size2)) goto _output_error;
2506
2507 if (size1 != size2) goto _output_error;
2508 }
2509 DISPLAYLEVEL(3, "OK \n");
2510
2511 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++);
2512 {
2513 size_t size1, size2;
2514 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2515 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2516 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2517 if (ZSTD_isError(size1)) goto _output_error;
2518
2519 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2520 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2521 if (ZSTD_isError(size2)) goto _output_error;
2522
2523 if (size1 == size2) goto _output_error;
2524 }
2525 DISPLAYLEVEL(3, "OK \n");
2526
2527 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2528 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) );
2529 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2530 CHECK_Z(cSize);
2531 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++);
2532 {
2533 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2534 size_t ret;
2535 /* We should fail to decompress without a dictionary. */
2536 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2537 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2538 if (!ZSTD_isError(ret)) goto _output_error;
2539 /* We should succeed to decompress with the dictionary. */
2540 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2541 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) );
2542 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2543 /* The dictionary should persist across calls. */
2544 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2545 /* When we reset the context the dictionary is cleared. */
2546 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2547 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2548 if (!ZSTD_isError(ret)) goto _output_error;
2549 ZSTD_freeDCtx(dctx);
2550 }
2551 DISPLAYLEVEL(3, "OK \n");
2552
2553 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
2554 {
2555 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2556 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
2557 size_t ret;
2558 /* We should succeed to decompress with the ddict. */
2559 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2560 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
2561 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2562 /* The ddict should persist across calls. */
2563 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2564 /* When we reset the context the ddict is cleared. */
2565 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2566 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2567 if (!ZSTD_isError(ret)) goto _output_error;
2568 ZSTD_freeDCtx(dctx);
2569 ZSTD_freeDDict(ddict);
2570 }
2571 DISPLAYLEVEL(3, "OK \n");
2572
2573 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
2574 {
2575 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2576 size_t ret;
2577 /* We should succeed to decompress with the prefix. */
2578 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2579 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
2580 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2581 /* The prefix should be cleared after the first compression. */
2582 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2583 if (!ZSTD_isError(ret)) goto _output_error;
2584 ZSTD_freeDCtx(dctx);
2585 }
2586 DISPLAYLEVEL(3, "OK \n");
2587
2588 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
2589 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2590 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
2591 CNBuffer, samplesSizes, nbSamples);
2592 if (ZDICT_isError(dictSize)) goto _output_error;
2593 /* Set all the repcodes to non-default */
2594 {
2595 BYTE* dictPtr = (BYTE*)dictBuffer;
2596 BYTE* dictLimit = dictPtr + dictSize - 12;
2597 /* Find the repcodes */
2598 while (dictPtr < dictLimit &&
2599 (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
2600 MEM_readLE32(dictPtr + 8) != 8)) {
2601 ++dictPtr;
2602 }
2603 if (dictPtr >= dictLimit) goto _output_error;
2604 MEM_writeLE32(dictPtr + 0, 10);
2605 MEM_writeLE32(dictPtr + 4, 10);
2606 MEM_writeLE32(dictPtr + 8, 10);
2607 /* Set the last 8 bytes to 'x' */
2608 memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
2609 }
2610 /* The optimal parser checks all the repcodes.
2611 * Make sure at least one is a match >= targetLength so that it is
2612 * immediately chosen. This will make sure that the compressor and
2613 * decompressor agree on at least one of the repcodes.
2614 */
2615 { size_t dSize;
2616 BYTE data[1024];
2617 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2618 ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
2619 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
2620 ZSTD_dlm_byRef, ZSTD_dct_auto,
2621 cParams, ZSTD_defaultCMem);
2622 assert(dctx != NULL); assert(cdict != NULL);
2623 memset(data, 'x', sizeof(data));
2624 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
2625 data, sizeof(data), cdict);
2626 ZSTD_freeCDict(cdict);
2627 if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
2628 dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
2629 if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
2630 if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
2631 ZSTD_freeDCtx(dctx);
2632 }
2633 DISPLAYLEVEL(3, "OK \n");
2634
2635 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++);
2636 {
2637 const size_t numDicts = 128;
2638 const size_t numFrames = 4;
2639 size_t i;
2640 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2641 ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts);
2642 ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts);
2643 U32 dictIDSeed = seed;
2644 /* Create new compressed buffer that will hold frames with differing dictIDs */
2645 char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */
2646
2647 ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize);
2648 /* Create a bunch of DDicts with random dict IDs */
2649 for (i = 0; i < numDicts; ++i) {
2650 U32 currDictID = FUZ_rand(&dictIDSeed);
2651 MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID);
2652 ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize);
2653 cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3);
2654 if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) {
2655 goto _output_error;
2656 }
2657 }
2658 /* Compress a few frames using random CDicts */
2659 {
2660 size_t off = 0;
2661 /* only use the first half so we don't push against size limit of compressedBuffer */
2662 size_t const segSize = (CNBuffSize / 2) / numFrames;
2663 for (i = 0; i < numFrames; i++) {
2664 size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts;
2665 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2666 { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx,
2667 (BYTE*)compressedBuffer + off, CNBuffSize - off,
2668 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
2669 cdictTable[dictIdx]));
2670 off += r;
2671 }
2672 }
2673 cSize = off;
2674 }
2675
2676 /* We should succeed to decompression even though different dicts were used on different frames */
2677 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2678 ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts);
2679 /* Reference every single ddict we made */
2680 for (i = 0; i < numDicts; ++i) {
2681 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i]));
2682 }
2683 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
2684 /* Streaming decompression should also work */
2685 {
2686 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
2687 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
2688 while (in.pos < in.size) {
2689 CHECK_Z(ZSTD_decompressStream(dctx, &out, &in));
2690 }
2691 }
2692 ZSTD_freeDCtx(dctx);
2693 for (i = 0; i < numDicts; ++i) {
2694 ZSTD_freeCDict(cdictTable[i]);
2695 ZSTD_freeDDict(ddictTable[i]);
2696 }
2697 free(dictBufferMulti);
2698 free(ddictTable);
2699 free(cdictTable);
2700 }
2701 DISPLAYLEVEL(3, "OK \n");
2702
2703 ZSTD_freeCCtx(cctx);
2704 free(dictBuffer);
2705 free(samplesSizes);
2706 }
2707
2708 /* COVER dictionary builder tests */
2709 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2710 size_t dictSize = 16 KB;
2711 size_t optDictSize = dictSize;
2712 void* dictBuffer = malloc(dictSize);
2713 size_t const totalSampleSize = 1 MB;
2714 size_t const sampleUnitSize = 8 KB;
2715 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
2716 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
2717 U32 seed32 = seed;
2718 ZDICT_cover_params_t params;
2719 U32 dictID;
2720
2721 if (dictBuffer==NULL || samplesSizes==NULL) {
2722 free(dictBuffer);
2723 free(samplesSizes);
2724 goto _output_error;
2725 }
2726
2727 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
2728 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2729 memset(¶ms, 0, sizeof(params));
2730 params.d = 1 + (FUZ_rand(&seed32) % 16);
2731 params.k = params.d + (FUZ_rand(&seed32) % 256);
2732 dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize,
2733 CNBuffer, samplesSizes, nbSamples,
2734 params);
2735 if (ZDICT_isError(dictSize)) goto _output_error;
2736 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2737
2738 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2739 dictID = ZDICT_getDictID(dictBuffer, dictSize);
2740 if (dictID==0) goto _output_error;
2741 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2742
2743 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
2744 memset(¶ms, 0, sizeof(params));
2745 params.steps = 4;
2746 optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize,
2747 CNBuffer, samplesSizes,
2748 nbSamples / 4, ¶ms);
2749 if (ZDICT_isError(optDictSize)) goto _output_error;
2750 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize);
2751
2752 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2753 dictID = ZDICT_getDictID(dictBuffer, optDictSize);
2754 if (dictID==0) goto _output_error;
2755 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2756
2757 ZSTD_freeCCtx(cctx);
2758 free(dictBuffer);
2759 free(samplesSizes);
2760 }
2761
2762 /* Decompression defense tests */
2763 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++);
2764 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
2765 if (!ZSTD_isError(r)) goto _output_error;
2766 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
2767 DISPLAYLEVEL(3, "OK \n");
2768
2769 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++);
2770 ((char*)(CNBuffer))[0] = 1;
2771 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
2772 if (!ZSTD_isError(r)) goto _output_error; }
2773 DISPLAYLEVEL(3, "OK \n");
2774
2775 /* content size verification test */
2776 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++);
2777 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2778 size_t const srcSize = 5000;
2779 size_t const wrongSrcSize = (srcSize + 1000);
2780 ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
2781 params.fParams.contentSizeFlag = 1;
2782 CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
2783 { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
2784 if (!ZSTD_isError(result)) goto _output_error;
2785 if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
2786 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result));
2787 }
2788 ZSTD_freeCCtx(cctx);
2789 }
2790
2791 /* negative compression level test : ensure simple API and advanced API produce same result */
2792 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++);
2793 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2794 size_t const srcSize = CNBuffSize / 5;
2795 int const compressionLevel = -1;
2796
2797 assert(cctx != NULL);
2798 { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize,
2799 CNBuffer, srcSize, compressionLevel);
2800 if (ZSTD_isError(cSize_1pass)) goto _output_error;
2801
2802 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
2803 { size_t const compressionResult = ZSTD_compress2(cctx,
2804 compressedBuffer, compressedBufferSize,
2805 CNBuffer, srcSize);
2806 DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
2807 if (ZSTD_isError(compressionResult)) goto _output_error;
2808 if (compressionResult != cSize_1pass) goto _output_error;
2809 } }
2810 ZSTD_freeCCtx(cctx);
2811 }
2812 DISPLAYLEVEL(3, "OK \n");
2813
2814 /* parameters order test */
2815 { size_t const inputSize = CNBuffSize / 2;
2816 U64 xxh64;
2817
2818 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2819 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
2820 assert(cctx != NULL);
2821 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
2822 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
2823 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
2824 { size_t const compressedSize = ZSTD_compress2(cctx,
2825 compressedBuffer, ZSTD_compressBound(inputSize),
2826 CNBuffer, inputSize);
2827 CHECK(compressedSize);
2828 cSize = compressedSize;
2829 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
2830 }
2831 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
2832 ZSTD_freeCCtx(cctx);
2833 }
2834
2835 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2836 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
2837 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
2838 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) );
2839 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
2840 { size_t const result = ZSTD_compress2(cctx,
2841 compressedBuffer, ZSTD_compressBound(inputSize),
2842 CNBuffer, inputSize);
2843 CHECK(result);
2844 if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
2845 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
2846 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
2847 }
2848 ZSTD_freeCCtx(cctx);
2849 }
2850 }
2851
2852 /* advanced parameters for decompression */
2853 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2854 assert(dctx != NULL);
2855
2856 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
2857 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
2858 CHECK(bounds.error);
2859 }
2860 DISPLAYLEVEL(3, "OK \n");
2861
2862 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
2863 { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
2864 if (!ZSTD_isError(sr)) goto _output_error;
2865 }
2866 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
2867 if (!ZSTD_isError(bounds.error)) goto _output_error;
2868 }
2869 DISPLAYLEVEL(3, "OK \n");
2870
2871 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
2872 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
2873 if (!ZSTD_isError(sr)) goto _output_error;
2874 }
2875 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
2876 if (!ZSTD_isError(sr)) goto _output_error;
2877 }
2878 DISPLAYLEVEL(3, "OK \n");
2879
2880 ZSTD_freeDCtx(dctx);
2881 }
2882
2883
2884 /* custom formats tests */
2885 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2886 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2887 size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
2888 assert(dctx != NULL); assert(cctx != NULL);
2889
2890 /* basic block compression */
2891 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
2892 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
2893 { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
2894 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
2895 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
2896 if (result != 0) goto _output_error;
2897 if (in.pos != in.size) goto _output_error;
2898 cSize = out.pos;
2899 }
2900 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
2901
2902 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++);
2903 { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2904 if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
2905 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult));
2906 }
2907
2908 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
2909 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2910 CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
2911 { ZSTD_frameHeader zfh;
2912 size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
2913 if (zfhrt != 0) goto _output_error;
2914 }
2915 /* one shot */
2916 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2917 if (result != inputSize) goto _output_error;
2918 DISPLAYLEVEL(3, "one-shot OK, ");
2919 }
2920 /* streaming */
2921 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
2922 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
2923 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
2924 if (result != 0) goto _output_error;
2925 if (in.pos != in.size) goto _output_error;
2926 if (out.pos != inputSize) goto _output_error;
2927 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
2928 }
2929
2930 /* basic block compression */
2931 DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
2932 CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
2933 { ZSTD_inBuffer in = { CNBuffer, 0, 0 };
2934 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
2935 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
2936 if (result != 0) goto _output_error;
2937 if (in.pos != in.size) goto _output_error;
2938 cSize = out.pos;
2939 }
2940 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
2941
2942 DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
2943 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
2944 CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
2945 /* one shot */
2946 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2947 if (result != 0) goto _output_error;
2948 DISPLAYLEVEL(3, "one-shot OK, ");
2949 }
2950 /* streaming */
2951 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
2952 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
2953 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
2954 if (result != 0) goto _output_error;
2955 if (in.pos != in.size) goto _output_error;
2956 if (out.pos != 0) goto _output_error;
2957 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
2958 }
2959
2960 ZSTD_freeCCtx(cctx);
2961 ZSTD_freeDCtx(dctx);
2962 }
2963
2964 DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++);
2965 {
2966 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2967 /* Attempt to future proof this to new parameters. */
2968 int const maxParam = 2000;
2969 int param;
2970 if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error;
2971 for (param = 0; param < maxParam; ++param) {
2972 ZSTD_dParameter dParam = (ZSTD_dParameter)param;
2973 ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam);
2974 int value1;
2975 int value2;
2976 int check;
2977 if (ZSTD_isError(bounds.error))
2978 continue;
2979 CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
2980 value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
2981 CHECK(ZSTD_DCtx_setParameter(dctx, dParam, value2));
2982 CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
2983 if (check != value2) goto _output_error;
2984 CHECK(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
2985 CHECK(ZSTD_DCtx_getParameter(dctx, dParam, &check));
2986 if (check != value1) goto _output_error;
2987 }
2988 ZSTD_freeDCtx(dctx);
2989 }
2990 DISPLAYLEVEL(3, "OK \n");
2991
2992 /* block API tests */
2993 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2994 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2995 static const size_t dictSize = 65 KB;
2996 static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
2997 size_t cSize2;
2998 assert(cctx != NULL); assert(dctx != NULL);
2999
3000 /* basic block compression */
3001 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
3002 CHECK( ZSTD_compressBegin(cctx, 5) );
3003 CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
3004 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
3005 DISPLAYLEVEL(3, "OK \n");
3006
3007 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
3008 CHECK( ZSTD_decompressBegin(dctx) );
3009 { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3010 if (r != blockSize) goto _output_error; }
3011 DISPLAYLEVEL(3, "OK \n");
3012
3013 /* very long stream of block compression */
3014 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
3015 CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
3016 CHECK( ZSTD_getBlockSize(cctx) >= blockSize);
3017 { U64 const toCompress = 5000000000ULL; /* > 4 GB */
3018 U64 compressed = 0;
3019 while (compressed < toCompress) {
3020 size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
3021 assert(blockCSize != 0);
3022 if (ZSTD_isError(blockCSize)) goto _output_error;
3023 compressed += blockCSize;
3024 } }
3025 DISPLAYLEVEL(3, "OK \n");
3026
3027 /* dictionary block compression */
3028 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
3029 CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
3030 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
3031 RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
3032 { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
3033 assert(r == 0); /* non-compressible block */ }
3034 memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */
3035 CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
3036 (char*)CNBuffer+dictSize+2*blockSize, blockSize));
3037 DISPLAYLEVEL(3, "OK \n");
3038
3039 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
3040 CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
3041 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
3042 if (r != blockSize) {
3043 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3044 goto _output_error;
3045 } }
3046 memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize);
3047 ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
3048 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
3049 if (r != blockSize) {
3050 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3051 goto _output_error;
3052 } }
3053 assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */
3054 DISPLAYLEVEL(3, "OK \n");
3055
3056 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
3057 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
3058 if (cdict==NULL) goto _output_error;
3059 CHECK( ZSTD_compressBegin_usingCDict(cctx, cdict) );
3060 CHECK( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
3061 ZSTD_freeCDict(cdict);
3062 }
3063 DISPLAYLEVEL(3, "OK \n");
3064
3065 ZSTD_freeCCtx(cctx);
3066 ZSTD_freeDCtx(dctx);
3067 }
3068
3069 /* long rle test */
3070 { size_t sampleSize = 0;
3071 size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
3072 DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
3073 memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
3074 sampleSize += 256 KB - 1;
3075 memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
3076 sampleSize += 96 KB;
3077 cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
3078 if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
3079 { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
3080 if (regenSize!=sampleSize) goto _output_error; }
3081 DISPLAYLEVEL(3, "OK \n");
3082 }
3083
3084 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++);
3085 {
3086 size_t srcSize = 150 KB;
3087 BYTE* src = (BYTE*)CNBuffer;
3088 BYTE* decoded = (BYTE*)compressedBuffer;
3089
3090 ZSTD_CCtx* cctx = ZSTD_createCCtx();
3091 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3092 size_t seqsSize;
3093
3094 if (seqs == NULL) goto _output_error;
3095 assert(cctx != NULL);
3096 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
3097 /* Populate src with random data */
3098 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
3099
3100 /* Test with block delimiters roundtrip */
3101 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3102 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters);
3103 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3104
3105 /* Test no block delimiters roundtrip */
3106 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
3107 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters);
3108 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3109
3110 ZSTD_freeCCtx(cctx);
3111 free(seqs);
3112 }
3113 DISPLAYLEVEL(3, "OK \n");
3114
3115 DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
3116 {
3117 size_t srcSize = 500 KB;
3118 BYTE* src = (BYTE*)CNBuffer;
3119 BYTE* dst = (BYTE*)compressedBuffer;
3120 size_t dstSize = ZSTD_compressBound(srcSize);
3121 size_t decompressSize = srcSize;
3122 char* decompressBuffer = (char*)malloc(decompressSize);
3123 size_t compressedSize;
3124 size_t dSize;
3125
3126 ZSTD_CCtx* cctx = ZSTD_createCCtx();
3127 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3128 size_t seqsSize;
3129
3130 if (seqs == NULL) goto _output_error;
3131 assert(cctx != NULL);
3132
3133 /* Populate src with random data */
3134 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
3135
3136 /* Test with block delimiters roundtrip */
3137 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3138 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3139 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
3140 compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize);
3141 if (ZSTD_isError(compressedSize)) {
3142 DISPLAY("Error in sequence compression with block delims\n");
3143 goto _output_error;
3144 }
3145 dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3146 if (ZSTD_isError(dSize)) {
3147 DISPLAY("Error in sequence compression roundtrip with block delims\n");
3148 goto _output_error;
3149 }
3150 assert(!memcmp(decompressBuffer, src, srcSize));
3151
3152 /* Test with no block delimiters roundtrip */
3153 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
3154 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3155 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
3156 compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize);
3157 if (ZSTD_isError(compressedSize)) {
3158 DISPLAY("Error in sequence compression with no block delims\n");
3159 goto _output_error;
3160 }
3161 dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3162 if (ZSTD_isError(dSize)) {
3163 DISPLAY("Error in sequence compression roundtrip with no block delims\n");
3164 goto _output_error;
3165 }
3166 assert(!memcmp(decompressBuffer, src, srcSize));
3167
3168 ZSTD_freeCCtx(cctx);
3169 free(decompressBuffer);
3170 free(seqs);
3171 }
3172 DISPLAYLEVEL(3, "OK \n");
3173
3174 /* Multiple blocks of zeros test */
3175 #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
3176 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3177 memset(CNBuffer, 0, LONGZEROSLENGTH);
3178 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) );
3179 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100);
3180
3181 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3182 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) );
3183 if (r != LONGZEROSLENGTH) goto _output_error; }
3184 DISPLAYLEVEL(3, "OK \n");
3185
3186 /* All zeroes test (test bug #137) */
3187 #define ZEROESLENGTH 100
3188 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
3189 memset(CNBuffer, 0, ZEROESLENGTH);
3190 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
3191 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100);
3192
3193 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
3194 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
3195 if (r != ZEROESLENGTH) goto _output_error; }
3196 DISPLAYLEVEL(3, "OK \n");
3197
3198 /* nbSeq limit test */
3199 #define _3BYTESTESTLENGTH 131000
3200 #define NB3BYTESSEQLOG 9
3201 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
3202 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
3203 /* creates a buffer full of 3-bytes sequences */
3204 { BYTE _3BytesSeqs[NB3BYTESSEQ][3];
3205 U32 rSeed = 1;
3206
3207 /* create batch of 3-bytes sequences */
3208 { int i;
3209 for (i=0; i < NB3BYTESSEQ; i++) {
3210 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
3211 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
3212 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
3213 } }
3214
3215 /* randomly fills CNBuffer with prepared 3-bytes sequences */
3216 { int i;
3217 for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
3218 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
3219 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
3220 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
3221 ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
3222 } } }
3223 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++);
3224 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3225 size_t const maxNbSeq = _3BYTESTESTLENGTH / 3;
3226 size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
3227 size_t nbSeq = 1;
3228 while (nbSeq <= maxNbSeq) {
3229 CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
3230 /* Check every sequence for the first 100, then skip more rapidly. */
3231 if (nbSeq < 100) {
3232 ++nbSeq;
3233 } else {
3234 nbSeq += (nbSeq >> 2);
3235 }
3236 }
3237 ZSTD_freeCCtx(cctx);
3238 }
3239 DISPLAYLEVEL(3, "OK \n");
3240
3241 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
3242 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
3243 CNBuffer, _3BYTESTESTLENGTH, 19) );
3244 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
3245
3246 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
3247 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
3248 if (r != _3BYTESTESTLENGTH) goto _output_error; }
3249 DISPLAYLEVEL(3, "OK \n");
3250
3251
3252 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++);
3253 RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
3254 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3255 size_t const bound = ZSTD_compressBound(CNBuffSize);
3256 size_t size = 1;
3257 while (size <= CNBuffSize) {
3258 CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
3259 /* Check every size for the first 100, then skip more rapidly. */
3260 if (size < 100) {
3261 ++size;
3262 } else {
3263 size += (size >> 2);
3264 }
3265 }
3266 ZSTD_freeCCtx(cctx);
3267 }
3268 DISPLAYLEVEL(3, "OK \n");
3269
3270 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
3271 { /* Train a dictionary on low characters */
3272 size_t dictSize = 16 KB;
3273 void* const dictBuffer = malloc(dictSize);
3274 size_t const totalSampleSize = 1 MB;
3275 size_t const sampleUnitSize = 8 KB;
3276 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3277 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3278 if (!dictBuffer || !samplesSizes) goto _output_error;
3279 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3280 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
3281 if (ZDICT_isError(dictSize)) goto _output_error;
3282 /* Reverse the characters to make the dictionary ill suited */
3283 { U32 u;
3284 for (u = 0; u < CNBuffSize; ++u) {
3285 ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
3286 }
3287 }
3288 { /* Compress the data */
3289 size_t const inputSize = 500;
3290 size_t const outputSize = ZSTD_compressBound(inputSize);
3291 void* const outputBuffer = malloc(outputSize);
3292 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3293 if (!outputBuffer || !cctx) goto _output_error;
3294 CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
3295 free(outputBuffer);
3296 ZSTD_freeCCtx(cctx);
3297 }
3298
3299 free(dictBuffer);
3300 free(samplesSizes);
3301 }
3302 DISPLAYLEVEL(3, "OK \n");
3303
3304
3305 /* findFrameCompressedSize on skippable frames */
3306 DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++);
3307 { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
3308 size_t const frameSrcSize = 13;
3309 if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
3310 DISPLAYLEVEL(3, "OK \n");
3311
3312 /* error string tests */
3313 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
3314 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
3315 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
3316 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
3317 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
3318 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
3319 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
3320 DISPLAYLEVEL(3, "OK \n");
3321
3322 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
3323 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
3324 {
3325 size_t const size = MIN(128 KB, CNBuffSize);
3326 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3327 ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
3328 ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
3329 ZSTD_frameHeader lgHeader;
3330 ZSTD_frameHeader smHeader;
3331
3332 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
3333 CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
3334 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
3335 CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
3336
3337 if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
3338
3339 ZSTD_freeCDict(smCDict);
3340 ZSTD_freeCDict(lgCDict);
3341 ZSTD_freeCCtx(cctx);
3342 }
3343 DISPLAYLEVEL(3, "OK \n");
3344
3345 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++);
3346 {
3347 short norm[32];
3348 unsigned count[32];
3349 unsigned const tableLog = 5;
3350 size_t const nbSeq = 32;
3351 unsigned const maxSymbolValue = 31;
3352 size_t i;
3353
3354 for (i = 0; i < 32; ++i)
3355 count[i] = 1;
3356 /* Calling FSE_normalizeCount() on a uniform distribution should not
3357 * cause a division by zero.
3358 */
3359 FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1);
3360 }
3361 DISPLAYLEVEL(3, "OK \n");
3362
3363 DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++);
3364 {
3365 size_t const outBufSize = 9;
3366 short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18};
3367 unsigned const tableLog = 5;
3368 unsigned const maxSymbolValue = 10;
3369 BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE));
3370
3371 /* Ensure that this write doesn't write out of bounds, and that
3372 * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1.
3373 */
3374 FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog);
3375 free(outBuf);
3376 }
3377 DISPLAYLEVEL(3, "OK \n");
3378
3379 #ifdef ZSTD_MULTITHREAD
3380 DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++);
3381 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
3382 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
3383 size_t const dstSize = ZSTD_compressBound(srcSize);
3384 void* const src = CNBuffer;
3385 void* const dst = compressedBuffer;
3386 void* dict = (void*)malloc(srcSize);
3387
3388 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
3389 RDG_genBuffer(dict, srcSize, compressibility, 0., seed);
3390
3391 /* Make sure there is no ZSTD_MAGIC_NUMBER */
3392 memset(dict, 0, sizeof(U32));
3393
3394 /* something more than 1 */
3395 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
3396 /* lie and claim this is a full dict */
3397 CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict));
3398
3399 { ZSTD_outBuffer out = {dst, dstSize, 0};
3400 ZSTD_inBuffer in = {src, srcSize, 0};
3401 /* should fail because its not a full dict like we said it was */
3402 assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)));
3403 }
3404
3405 ZSTD_freeCCtx(cctx);
3406 free(dict);
3407 }
3408 DISPLAYLEVEL(3, "OK \n");
3409
3410 DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++);
3411 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
3412 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
3413 size_t const dictSize = 10;
3414 size_t const dstSize = ZSTD_compressBound(srcSize);
3415 void* const src = CNBuffer;
3416 void* const dst = compressedBuffer;
3417 void* dict = (void*)malloc(dictSize);
3418
3419 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
3420 RDG_genBuffer(dict, dictSize, compressibility, 0., seed);
3421
3422 /* Make sure there is no ZSTD_MAGIC_NUMBER */
3423 memset(dict, 0, sizeof(U32));
3424
3425 /* Enable MT, LDM, and use refPrefix() for a small dict */
3426 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
3427 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
3428 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
3429
3430 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
3431
3432 ZSTD_freeCCtx(cctx);
3433 free(dict);
3434 }
3435 DISPLAYLEVEL(3, "OK \n");
3436
3437 DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++);
3438 {
3439 ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0);
3440 ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0);
3441 ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400);
3442 ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000);
3443 ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000);
3444
3445 assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict)));
3446 assert(!memcmp(&medium, &mediumDict, sizeof(medium)));
3447 assert(!memcmp(&large, &largeDict, sizeof(large)));
3448 }
3449 DISPLAYLEVEL(3, "OK \n");
3450
3451 DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++);
3452 {
3453 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0);
3454 ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400);
3455 ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400);
3456
3457 assert(smallSrcAndDict.windowLog == 10);
3458 assert(!memcmp(&cParams, &smallDict, sizeof(cParams)));
3459 }
3460 DISPLAYLEVEL(3, "OK \n");
3461
3462 DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
3463 {
3464 int level = 1;
3465 size_t prevSize = 0;
3466 for (; level < ZSTD_maxCLevel(); ++level) {
3467 size_t const currSize = ZSTD_estimateCCtxSize(level);
3468 if (prevSize > currSize) {
3469 DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
3470 prevSize, level-1, currSize, level);
3471 goto _output_error;
3472 }
3473 prevSize = currSize;
3474 }
3475 }
3476 DISPLAYLEVEL(3, "OK \n");
3477
3478 DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
3479 {
3480 size_t const kSizeIncrement = 2 KB;
3481 int level = -3;
3482
3483 for (; level <= ZSTD_maxCLevel(); ++level) {
3484 size_t dictSize = 0;
3485 for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
3486 size_t srcSize = 2 KB;
3487 for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
3488 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
3489 size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
3490 size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
3491 if (cctxSizeUsingLevel < cctxSizeUsingCParams
3492 || ZSTD_isError(cctxSizeUsingCParams)
3493 || ZSTD_isError(cctxSizeUsingLevel)) {
3494 DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
3495 level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
3496 goto _output_error;
3497 } } } } }
3498 DISPLAYLEVEL(3, "OK \n");
3499
3500 DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++)
3501 {
3502 int const threadPoolTestResult = threadPoolTests();
3503 if (threadPoolTestResult) {
3504 goto _output_error;
3505 }
3506 }
3507 DISPLAYLEVEL(3, "thread pool tests OK \n");
3508
3509 #endif /* ZSTD_MULTITHREAD */
3510
3511 _end:
3512 free(CNBuffer);
3513 free(compressedBuffer);
3514 free(decodedBuffer);
3515 return testResult;
3516
3517 _output_error:
3518 testResult = 1;
3519 DISPLAY("Error detected in Unit tests ! \n");
3520 goto _end;
3521 }
3522
longUnitTests(U32 const seed,double compressibility)3523 static int longUnitTests(U32 const seed, double compressibility)
3524 {
3525 size_t const CNBuffSize = 5 MB;
3526 void* const CNBuffer = malloc(CNBuffSize);
3527 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
3528 void* const compressedBuffer = malloc(compressedBufferSize);
3529 void* const decodedBuffer = malloc(CNBuffSize);
3530 int testResult = 0;
3531 unsigned testNb=0;
3532 size_t cSize;
3533
3534 /* Create compressible noise */
3535 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
3536 DISPLAY("Not enough memory, aborting\n");
3537 testResult = 1;
3538 goto _end;
3539 }
3540 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
3541
3542 /* note : this test is rather long, it would be great to find a way to speed up its execution */
3543 DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++);
3544 { int cLevel;
3545 size_t approxIndex = 0;
3546 size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */
3547
3548 /* Provision enough space in a static context so that we can do all
3549 * this without ever reallocating, which would reset the indices. */
3550 size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22);
3551 void* const staticCCtxBuffer = malloc(staticCCtxSize);
3552 ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
3553
3554 /* bump the indices so the following compressions happen at high
3555 * indices. */
3556 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
3557 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
3558 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3559 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
3560 while (approxIndex <= (maxIndex / 4) * 3) {
3561 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
3562 approxIndex += in.pos;
3563 CHECK(in.pos == in.size);
3564 in.pos = 0;
3565 out.pos = 0;
3566 }
3567 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
3568 }
3569
3570 /* spew a bunch of stuff into the table area */
3571 for (cLevel = 1; cLevel <= 22; cLevel++) {
3572 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
3573 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
3574 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3575 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
3576 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
3577 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
3578 approxIndex += in.pos;
3579 }
3580
3581 /* now crank the indices so we overflow */
3582 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
3583 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
3584 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3585 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
3586 while (approxIndex <= maxIndex) {
3587 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
3588 approxIndex += in.pos;
3589 CHECK(in.pos == in.size);
3590 in.pos = 0;
3591 out.pos = 0;
3592 }
3593 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
3594 }
3595
3596 /* do a bunch of compressions again in low indices and ensure we don't
3597 * hit untracked invalid indices */
3598 for (cLevel = 1; cLevel <= 22; cLevel++) {
3599 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
3600 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
3601 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3602 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
3603 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
3604 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
3605 approxIndex += in.pos;
3606 }
3607
3608 free(staticCCtxBuffer);
3609 }
3610 DISPLAYLEVEL(3, "OK \n");
3611
3612 DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++);
3613 { size_t cSizeLdm;
3614 size_t cSizeNoLdm;
3615 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3616
3617 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed);
3618
3619 /* Enable checksum to verify round trip. */
3620 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3621 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1));
3622 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
3623
3624 /* Round trip once with ldm. */
3625 cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3626 CHECK_Z(cSizeLdm);
3627 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm));
3628
3629 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3630 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3631 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0));
3632 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
3633
3634 /* Round trip once without ldm. */
3635 cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3636 CHECK_Z(cSizeNoLdm);
3637 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm));
3638
3639 if (cSizeLdm > cSizeNoLdm) {
3640 DISPLAY("Using long mode should not cause regressions for btopt+\n");
3641 testResult = 1;
3642 goto _end;
3643 }
3644
3645 ZSTD_freeCCtx(cctx);
3646 }
3647 DISPLAYLEVEL(3, "OK \n");
3648
3649 DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++);
3650 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3651 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3652 size_t dictSize = CNBuffSize;
3653 void* dict = (void*)malloc(dictSize);
3654 ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams();
3655 ZSTD_dictAttachPref_e const attachPrefs[] = {
3656 ZSTD_dictDefaultAttach,
3657 ZSTD_dictForceAttach,
3658 ZSTD_dictForceCopy,
3659 ZSTD_dictForceLoad,
3660 ZSTD_dictDefaultAttach,
3661 ZSTD_dictForceAttach,
3662 ZSTD_dictForceCopy,
3663 ZSTD_dictForceLoad
3664 };
3665 int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1};
3666 int cLevel;
3667 int i;
3668
3669 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
3670 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
3671
3672 CHECK(cctx_params != NULL);
3673
3674 for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
3675 DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
3676 for (cLevel = 4; cLevel < 13; cLevel++) {
3677 for (i = 0; i < 8; ++i) {
3678 ZSTD_dictAttachPref_e const attachPref = attachPrefs[i];
3679 int const enableDDS = enableDedicatedDictSearch[i];
3680 ZSTD_CDict* cdict;
3681
3682 DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i);
3683
3684 ZSTD_CCtxParams_init(cctx_params, cLevel);
3685 CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS));
3686
3687 cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem);
3688 CHECK(cdict != NULL);
3689
3690 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3691 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref));
3692
3693 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3694 CHECK_Z(cSize);
3695 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
3696
3697 DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize);
3698
3699 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3700 ZSTD_freeCDict(cdict);
3701 } } }
3702
3703 ZSTD_freeCCtx(cctx);
3704 ZSTD_freeDCtx(dctx);
3705 ZSTD_freeCCtxParams(cctx_params);
3706 free(dict);
3707 }
3708 DISPLAYLEVEL(3, "OK \n");
3709
3710 _end:
3711 free(CNBuffer);
3712 free(compressedBuffer);
3713 free(decodedBuffer);
3714 return testResult;
3715
3716 _output_error:
3717 testResult = 1;
3718 DISPLAY("Error detected in Unit tests ! \n");
3719 goto _end;
3720 }
3721
3722
findDiff(const void * buf1,const void * buf2,size_t max)3723 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
3724 {
3725 const BYTE* b1 = (const BYTE*)buf1;
3726 const BYTE* b2 = (const BYTE*)buf2;
3727 size_t u;
3728 for (u=0; u<max; u++) {
3729 if (b1[u] != b2[u]) break;
3730 }
3731 return u;
3732 }
3733
3734
FUZ_makeParams(ZSTD_compressionParameters cParams,ZSTD_frameParameters fParams)3735 static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
3736 {
3737 ZSTD_parameters params;
3738 params.cParams = cParams;
3739 params.fParams = fParams;
3740 return params;
3741 }
3742
FUZ_rLogLength(U32 * seed,U32 logLength)3743 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
3744 {
3745 size_t const lengthMask = ((size_t)1 << logLength) - 1;
3746 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
3747 }
3748
FUZ_randomLength(U32 * seed,U32 maxLog)3749 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
3750 {
3751 U32 const logLength = FUZ_rand(seed) % maxLog;
3752 return FUZ_rLogLength(seed, logLength);
3753 }
3754
3755 #undef CHECK
3756 #define CHECK(cond, ...) { \
3757 if (cond) { \
3758 DISPLAY("Error => "); \
3759 DISPLAY(__VA_ARGS__); \
3760 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
3761 goto _output_error; \
3762 } }
3763
3764 #undef CHECK_Z
3765 #define CHECK_Z(f) { \
3766 size_t const err = f; \
3767 if (ZSTD_isError(err)) { \
3768 DISPLAY("Error => %s : %s ", \
3769 #f, ZSTD_getErrorName(err)); \
3770 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
3771 goto _output_error; \
3772 } }
3773
3774
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,U32 const maxDurationS,double compressibility,int bigTests)3775 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
3776 {
3777 static const U32 maxSrcLog = 23;
3778 static const U32 maxSampleLog = 22;
3779 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
3780 size_t const dstBufferSize = (size_t)1<<maxSampleLog;
3781 size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
3782 BYTE* cNoiseBuffer[5];
3783 BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
3784 BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
3785 BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
3786 ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
3787 ZSTD_CCtx* const ctx = ZSTD_createCCtx();
3788 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3789 U32 result = 0;
3790 unsigned testNb = 0;
3791 U32 coreSeed = seed;
3792 UTIL_time_t const startClock = UTIL_getTime();
3793 U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
3794 int const cLevelLimiter = bigTests ? 3 : 2;
3795
3796 /* allocation */
3797 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
3798 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
3799 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
3800 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
3801 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
3802 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
3803 || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
3804 "Not enough memory, fuzzer tests cancelled");
3805
3806 /* Create initial samples */
3807 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
3808 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
3809 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
3810 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
3811 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
3812
3813 /* catch up testNb */
3814 for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
3815
3816 /* main test loop */
3817 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
3818 BYTE* srcBuffer; /* jumping pointer */
3819 U32 lseed;
3820 size_t sampleSize, maxTestSize, totalTestSize;
3821 size_t cSize, totalCSize, totalGenSize;
3822 U64 crcOrig;
3823 BYTE* sampleBuffer;
3824 const BYTE* dict;
3825 size_t dictSize;
3826
3827 /* notification */
3828 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
3829 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
3830
3831 FUZ_rand(&coreSeed);
3832 { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
3833
3834 /* srcBuffer selection [0-4] */
3835 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
3836 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
3837 else {
3838 buffNb >>= 3;
3839 if (buffNb & 7) {
3840 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
3841 buffNb = tnb[buffNb >> 3];
3842 } else {
3843 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
3844 buffNb = tnb[buffNb >> 3];
3845 } }
3846 srcBuffer = cNoiseBuffer[buffNb];
3847 }
3848
3849 /* select src segment */
3850 sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
3851
3852 /* create sample buffer (to catch read error with valgrind & sanitizers) */
3853 sampleBuffer = (BYTE*)malloc(sampleSize);
3854 CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
3855 { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
3856 memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
3857 crcOrig = XXH64(sampleBuffer, sampleSize, 0);
3858
3859 /* compression tests */
3860 { int const cLevelPositive = (int)
3861 ( FUZ_rand(&lseed) %
3862 ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) )
3863 + 1;
3864 int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ?
3865 - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */
3866 cLevelPositive;
3867 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel);
3868 cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
3869 CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
3870
3871 /* compression failure test : too small dest buffer */
3872 assert(cSize > 3);
3873 { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;
3874 const size_t tooSmallSize = cSize - missing;
3875 const unsigned endMark = 0x4DC2B1A9;
3876 memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark));
3877 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
3878 testNb, (unsigned)tooSmallSize, (unsigned)missing);
3879 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
3880 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
3881 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck));
3882 CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); }
3883 } }
3884
3885 /* frame header decompression test */
3886 { ZSTD_frameHeader zfh;
3887 CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
3888 CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
3889 }
3890
3891 /* Decompressed size test */
3892 { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
3893 CHECK(rSize != sampleSize, "decompressed size incorrect");
3894 }
3895
3896 /* successful decompression test */
3897 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb);
3898 { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
3899 size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
3900 CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize);
3901 { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
3902 CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize);
3903 } }
3904
3905 free(sampleBuffer); /* no longer useful after this point */
3906
3907 /* truncated src decompression test */
3908 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb);
3909 { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
3910 size_t const tooSmallSize = cSize - missing;
3911 void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
3912 CHECK(cBufferTooSmall == NULL, "not enough memory !");
3913 memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
3914 { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
3915 CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
3916 free(cBufferTooSmall);
3917 }
3918
3919 /* too small dst decompression test */
3920 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb);
3921 if (sampleSize > 3) {
3922 size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
3923 size_t const tooSmallSize = sampleSize - missing;
3924 static const BYTE token = 0xA9;
3925 dstBuffer[tooSmallSize] = token;
3926 { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
3927 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
3928 CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
3929 }
3930
3931 /* noisy src decompression test */
3932 if (cSize > 6) {
3933 /* insert noise into src */
3934 { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
3935 size_t pos = 4; /* preserve magic number (too easy to detect) */
3936 for (;;) {
3937 /* keep some original src */
3938 { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
3939 size_t const mask = (1<<nbBits) - 1;
3940 size_t const skipLength = FUZ_rand(&lseed) & mask;
3941 pos += skipLength;
3942 }
3943 if (pos >= cSize) break;
3944 /* add noise */
3945 { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
3946 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
3947 size_t const mask = (1<<nbBits) - 1;
3948 size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
3949 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
3950 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
3951 memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
3952 pos += noiseLength;
3953 } } }
3954
3955 /* decompress noisy source */
3956 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb);
3957 { U32 const endMark = 0xA9B1C3D6;
3958 memcpy(dstBuffer+sampleSize, &endMark, 4);
3959 { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
3960 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
3961 CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
3962 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize);
3963 }
3964 { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
3965 CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
3966 } } } /* noisy src decompression test */
3967
3968 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
3969 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
3970 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
3971 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
3972 int const cLevel = (FUZ_rand(&lseed) %
3973 (ZSTD_maxCLevel() -
3974 (MAX(testLog, dictLog) / cLevelLimiter))) +
3975 1;
3976 maxTestSize = FUZ_rLogLength(&lseed, testLog);
3977 if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
3978
3979 dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
3980 dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
3981
3982 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
3983 testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize);
3984
3985 if (FUZ_rand(&lseed) & 0xF) {
3986 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
3987 } else {
3988 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
3989 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
3990 !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
3991 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
3992 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
3993 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
3994 }
3995 CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
3996 }
3997
3998 { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
3999 U32 n;
4000 XXH64_state_t xxhState;
4001 XXH64_reset(&xxhState, 0);
4002 for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
4003 size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
4004 size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
4005
4006 if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
4007 if (totalTestSize+segmentSize > maxTestSize) break;
4008
4009 { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
4010 CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
4011 cSize += compressResult;
4012 }
4013 XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
4014 memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
4015 totalTestSize += segmentSize;
4016 }
4017
4018 { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
4019 CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
4020 cSize += flushResult;
4021 }
4022 crcOrig = XXH64_digest(&xxhState);
4023 }
4024
4025 /* streaming decompression test */
4026 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
4027 /* ensure memory requirement is good enough (should always be true) */
4028 { ZSTD_frameHeader zfh;
4029 CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
4030 "ZSTD_getFrameHeader(): error retrieving frame information");
4031 { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
4032 CHECK_Z(roundBuffSize);
4033 CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN),
4034 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)",
4035 (unsigned)roundBuffSize, (unsigned)totalTestSize );
4036 } }
4037 if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
4038 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
4039 totalCSize = 0;
4040 totalGenSize = 0;
4041 while (totalCSize < cSize) {
4042 size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
4043 size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
4044 CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize));
4045 totalGenSize += genSize;
4046 totalCSize += inSize;
4047 }
4048 CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
4049 CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
4050 CHECK (totalCSize != cSize, "compressed data should be fully read")
4051 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
4052 CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
4053 (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize);
4054 }
4055 } /* for ( ; (testNb <= nbTests) */
4056 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
4057
4058 _cleanup:
4059 ZSTD_freeCCtx(refCtx);
4060 ZSTD_freeCCtx(ctx);
4061 ZSTD_freeDCtx(dctx);
4062 free(cNoiseBuffer[0]);
4063 free(cNoiseBuffer[1]);
4064 free(cNoiseBuffer[2]);
4065 free(cNoiseBuffer[3]);
4066 free(cNoiseBuffer[4]);
4067 free(cBuffer);
4068 free(dstBuffer);
4069 free(mirrorBuffer);
4070 return result;
4071
4072 _output_error:
4073 result = 1;
4074 goto _cleanup;
4075 }
4076
4077
4078 /*_*******************************************************
4079 * Command line
4080 *********************************************************/
FUZ_usage(const char * programName)4081 static int FUZ_usage(const char* programName)
4082 {
4083 DISPLAY( "Usage :\n");
4084 DISPLAY( " %s [args]\n", programName);
4085 DISPLAY( "\n");
4086 DISPLAY( "Arguments :\n");
4087 DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault);
4088 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
4089 DISPLAY( " -s# : Select seed (default:prompt user)\n");
4090 DISPLAY( " -t# : Select starting test number (default:0)\n");
4091 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default);
4092 DISPLAY( " -v : verbose\n");
4093 DISPLAY( " -p : pause at the end\n");
4094 DISPLAY( " -h : display help and exit\n");
4095 return 0;
4096 }
4097
4098 /*! readU32FromChar() :
4099 @return : unsigned integer value read from input in `char` format
4100 allows and interprets K, KB, KiB, M, MB and MiB suffix.
4101 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
4102 Note : function result can overflow if digit string > MAX_UINT */
readU32FromChar(const char ** stringPtr)4103 static unsigned readU32FromChar(const char** stringPtr)
4104 {
4105 unsigned result = 0;
4106 while ((**stringPtr >='0') && (**stringPtr <='9'))
4107 result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
4108 if ((**stringPtr=='K') || (**stringPtr=='M')) {
4109 result <<= 10;
4110 if (**stringPtr=='M') result <<= 10;
4111 (*stringPtr)++ ;
4112 if (**stringPtr=='i') (*stringPtr)++;
4113 if (**stringPtr=='B') (*stringPtr)++;
4114 }
4115 return result;
4116 }
4117
4118 /** longCommandWArg() :
4119 * check if *stringPtr is the same as longCommand.
4120 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
4121 * @return 0 and doesn't modify *stringPtr otherwise.
4122 */
longCommandWArg(const char ** stringPtr,const char * longCommand)4123 static int longCommandWArg(const char** stringPtr, const char* longCommand)
4124 {
4125 size_t const comSize = strlen(longCommand);
4126 int const result = !strncmp(*stringPtr, longCommand, comSize);
4127 if (result) *stringPtr += comSize;
4128 return result;
4129 }
4130
main(int argc,const char ** argv)4131 int main(int argc, const char** argv)
4132 {
4133 U32 seed = 0;
4134 int seedset = 0;
4135 int argNb;
4136 int nbTests = nbTestsDefault;
4137 int testNb = 0;
4138 int proba = FUZ_compressibility_default;
4139 double probfloat;
4140 int result = 0;
4141 U32 mainPause = 0;
4142 U32 maxDuration = 0;
4143 int bigTests = 1;
4144 int longTests = 0;
4145 U32 memTestsOnly = 0;
4146 const char* const programName = argv[0];
4147
4148 /* Check command line */
4149 for (argNb=1; argNb<argc; argNb++) {
4150 const char* argument = argv[argNb];
4151 if(!argument) continue; /* Protection if argument empty */
4152
4153 /* Handle commands. Aggregated commands are allowed */
4154 if (argument[0]=='-') {
4155
4156 if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; }
4157
4158 if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
4159 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
4160 if (!strcmp(argument, "--long-tests")) { longTests=1; continue; }
4161 if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; }
4162
4163 argument++;
4164 while (*argument!=0) {
4165 switch(*argument)
4166 {
4167 case 'h':
4168 return FUZ_usage(programName);
4169
4170 case 'v':
4171 argument++;
4172 g_displayLevel++;
4173 break;
4174
4175 case 'q':
4176 argument++;
4177 g_displayLevel--;
4178 break;
4179
4180 case 'p': /* pause at the end */
4181 argument++;
4182 mainPause = 1;
4183 break;
4184
4185 case 'i':
4186 argument++; maxDuration = 0;
4187 nbTests = (int)readU32FromChar(&argument);
4188 break;
4189
4190 case 'T':
4191 argument++;
4192 nbTests = 0;
4193 maxDuration = readU32FromChar(&argument);
4194 if (*argument=='s') argument++; /* seconds */
4195 if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
4196 if (*argument=='n') argument++;
4197 break;
4198
4199 case 's':
4200 argument++;
4201 seedset = 1;
4202 seed = readU32FromChar(&argument);
4203 break;
4204
4205 case 't':
4206 argument++;
4207 testNb = (int)readU32FromChar(&argument);
4208 break;
4209
4210 case 'P': /* compressibility % */
4211 argument++;
4212 proba = (int)readU32FromChar(&argument);
4213 if (proba>100) proba = 100;
4214 break;
4215
4216 default:
4217 return (FUZ_usage(programName), 1);
4218 } } } } /* for (argNb=1; argNb<argc; argNb++) */
4219
4220 /* Get Seed */
4221 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
4222
4223 if (!seedset) {
4224 time_t const t = time(NULL);
4225 U32 const h = XXH32(&t, sizeof(t), 1);
4226 seed = h % 10000;
4227 }
4228
4229 DISPLAY("Seed = %u\n", (unsigned)seed);
4230 if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba);
4231
4232 probfloat = ((double)proba) / 100;
4233
4234 if (memTestsOnly) {
4235 g_displayLevel = MAX(3, g_displayLevel);
4236 return FUZ_mallocTests(seed, probfloat, memTestsOnly);
4237 }
4238
4239 if (nbTests < testNb) nbTests = testNb;
4240
4241 if (testNb==0) {
4242 result = basicUnitTests(0, probfloat); /* constant seed for predictability */
4243
4244 if (!result && longTests) {
4245 result = longUnitTests(0, probfloat);
4246 }
4247 }
4248 if (!result)
4249 result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100, bigTests);
4250 if (mainPause) {
4251 int unused;
4252 DISPLAY("Press Enter \n");
4253 unused = getchar();
4254 (void)unused;
4255 }
4256 return result;
4257 }
4258