1 /*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12 /*-************************************
13 * Compiler specific
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
19 #endif
20
21
22 /*-************************************
23 * Includes
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #include <time.h> /* time(), time_t */
29 #undef NDEBUG /* always enable assert() */
30 #include <assert.h>
31 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
32 #include "debug.h" /* DEBUG_STATIC_ASSERT */
33 #include "fse.h"
34 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
35 #include "zstd.h" /* ZSTD_VERSION_STRING */
36 #include "zstd_errors.h" /* ZSTD_getErrorCode */
37 #define ZDICT_STATIC_LINKING_ONLY
38 #include "zdict.h" /* ZDICT_trainFromBuffer */
39 #include "mem.h"
40 #include "datagen.h" /* RDG_genBuffer */
41 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
42 #include "xxhash.h" /* XXH64 */
43 #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
44 /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
45 #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
46 #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */
47
48
49 /*-************************************
50 * Constants
51 **************************************/
52 #define GB *(1U<<30)
53
54 static const int FUZ_compressibility_default = 50;
55 static const int nbTestsDefault = 30000;
56
57
58 /*-************************************
59 * Display Macros
60 **************************************/
61 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
62 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
63 static U32 g_displayLevel = 2;
64
65 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
66 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
67
68 #define DISPLAYUPDATE(l, ...) \
69 if (g_displayLevel>=l) { \
70 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
71 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
72 if (g_displayLevel>=4) fflush(stderr); } \
73 }
74
75
76 /*-*******************************************************
77 * Compile time test
78 *********************************************************/
79 #undef MIN
80 #undef MAX
81 /* Declaring the function, to avoid -Wmissing-prototype */
82 void FUZ_bug976(void);
FUZ_bug976(void)83 void FUZ_bug976(void)
84 { /* these constants shall not depend on MIN() macro */
85 DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
86 DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
87 }
88
89
90 /*-*******************************************************
91 * Internal functions
92 *********************************************************/
93 #define MIN(a,b) ((a)<(b)?(a):(b))
94 #define MAX(a,b) ((a)>(b)?(a):(b))
95
96 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(U32 * src)97 static U32 FUZ_rand(U32* src)
98 {
99 static const U32 prime1 = 2654435761U;
100 static const U32 prime2 = 2246822519U;
101 U32 rand32 = *src;
102 rand32 *= prime1;
103 rand32 += prime2;
104 rand32 = FUZ_rotl32(rand32, 13);
105 *src = rand32;
106 return rand32 >> 5;
107 }
108
FUZ_highbit32(U32 v32)109 static U32 FUZ_highbit32(U32 v32)
110 {
111 unsigned nbBits = 0;
112 if (v32==0) return 0;
113 while (v32) v32 >>= 1, nbBits++;
114 return nbBits;
115 }
116
117
118 /*=============================================
119 * Test macros
120 =============================================*/
121 #define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
122
123 #define CHECK_Z(f) { \
124 size_t const err = f; \
125 if (ZSTD_isError(err)) { \
126 DISPLAY("Error => %s : %s ", \
127 #f, ZSTD_getErrorName(err)); \
128 exit(1); \
129 } }
130
131 #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
132 #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
133 #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
134
135 #define CHECK_OP(op, lhs, rhs) { \
136 if (!((lhs) op (rhs))) { \
137 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
138 exit(1); \
139 } \
140 }
141 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
142 #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
143
144
145 /*=============================================
146 * Memory Tests
147 =============================================*/
148 #if defined(__APPLE__) && defined(__MACH__)
149
150 #include <malloc/malloc.h> /* malloc_size */
151
152 typedef struct {
153 unsigned long long totalMalloc;
154 size_t currentMalloc;
155 size_t peakMalloc;
156 unsigned nbMalloc;
157 unsigned nbFree;
158 } mallocCounter_t;
159
160 static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };
161
FUZ_mallocDebug(void * counter,size_t size)162 static void* FUZ_mallocDebug(void* counter, size_t size)
163 {
164 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
165 void* const ptr = malloc(size);
166 if (ptr==NULL) return NULL;
167 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
168 (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */
169 mcPtr->totalMalloc += size;
170 mcPtr->currentMalloc += size;
171 if (mcPtr->currentMalloc > mcPtr->peakMalloc)
172 mcPtr->peakMalloc = mcPtr->currentMalloc;
173 mcPtr->nbMalloc += 1;
174 return ptr;
175 }
176
FUZ_freeDebug(void * counter,void * address)177 static void FUZ_freeDebug(void* counter, void* address)
178 {
179 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
180 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10));
181 mcPtr->nbFree += 1;
182 mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */
183 free(address);
184 }
185
FUZ_displayMallocStats(mallocCounter_t count)186 static void FUZ_displayMallocStats(mallocCounter_t count)
187 {
188 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n",
189 (unsigned)(count.peakMalloc >> 10),
190 count.nbMalloc,
191 (unsigned)(count.totalMalloc >> 10));
192 }
193
FUZ_mallocTests_internal(unsigned seed,double compressibility,unsigned part,void * inBuffer,size_t inSize,void * outBuffer,size_t outSize)194 static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
195 void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
196 {
197 /* test only played in verbose mode, as they are long */
198 if (g_displayLevel<3) return 0;
199
200 /* Create compressible noise */
201 if (!inBuffer || !outBuffer) {
202 DISPLAY("Not enough memory, aborting\n");
203 exit(1);
204 }
205 RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
206
207 /* simple compression tests */
208 if (part <= 1)
209 { int compressionLevel;
210 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
211 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
212 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
213 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
214 CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
215 ZSTD_freeCCtx(cctx);
216 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
217 FUZ_displayMallocStats(malcount);
218 } }
219
220 /* streaming compression tests */
221 if (part <= 2)
222 { int compressionLevel;
223 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
224 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
225 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
226 ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
227 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
228 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
229 CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
230 CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
231 CHECK_Z( ZSTD_endStream(cstream, &out) );
232 ZSTD_freeCStream(cstream);
233 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
234 FUZ_displayMallocStats(malcount);
235 } }
236
237 /* advanced MT API test */
238 if (part <= 3)
239 { int nbThreads;
240 for (nbThreads=1; nbThreads<=4; nbThreads++) {
241 int compressionLevel;
242 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
243 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
244 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
245 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
246 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
247 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
248 CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
249 ZSTD_freeCCtx(cctx);
250 DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
251 nbThreads, compressionLevel);
252 FUZ_displayMallocStats(malcount);
253 } } }
254
255 /* advanced MT streaming API test */
256 if (part <= 4)
257 { int nbThreads;
258 for (nbThreads=1; nbThreads<=4; nbThreads++) {
259 int compressionLevel;
260 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
261 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
262 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
263 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
264 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
265 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
266 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
267 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
268 CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
269 while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
270 ZSTD_freeCCtx(cctx);
271 DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ",
272 nbThreads, compressionLevel);
273 FUZ_displayMallocStats(malcount);
274 } } }
275
276 return 0;
277 }
278
FUZ_mallocTests(unsigned seed,double compressibility,unsigned part)279 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
280 {
281 size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
282 size_t const outSize = ZSTD_compressBound(inSize);
283 void* const inBuffer = malloc(inSize);
284 void* const outBuffer = malloc(outSize);
285 int result;
286
287 /* Create compressible noise */
288 if (!inBuffer || !outBuffer) {
289 DISPLAY("Not enough memory, aborting \n");
290 exit(1);
291 }
292
293 result = FUZ_mallocTests_internal(seed, compressibility, part,
294 inBuffer, inSize, outBuffer, outSize);
295
296 free(inBuffer);
297 free(outBuffer);
298 return result;
299 }
300
301 #else
302
FUZ_mallocTests(unsigned seed,double compressibility,unsigned part)303 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
304 {
305 (void)seed; (void)compressibility; (void)part;
306 return 0;
307 }
308
309 #endif
310
FUZ_decodeSequences(BYTE * dst,ZSTD_Sequence * seqs,size_t seqsSize,BYTE * src,size_t size,ZSTD_SequenceFormat_e format)311 static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
312 BYTE* src, size_t size, ZSTD_SequenceFormat_e format)
313 {
314 size_t i;
315 size_t j;
316 for(i = 0; i < seqsSize; ++i) {
317 assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size);
318 assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size);
319 if (format == ZSTD_sf_noBlockDelimiters) {
320 assert(seqs[i].matchLength != 0 || seqs[i].offset != 0);
321 }
322
323 memcpy(dst, src, seqs[i].litLength);
324 dst += seqs[i].litLength;
325 src += seqs[i].litLength;
326 size -= seqs[i].litLength;
327
328 if (seqs[i].offset != 0) {
329 for (j = 0; j < seqs[i].matchLength; ++j)
330 dst[j] = dst[(ptrdiff_t)(j - seqs[i].offset)];
331 dst += seqs[i].matchLength;
332 src += seqs[i].matchLength;
333 size -= seqs[i].matchLength;
334 }
335 }
336 if (format == ZSTD_sf_noBlockDelimiters) {
337 memcpy(dst, src, size);
338 }
339 }
340
FUZ_getLitSize(const ZSTD_Sequence * seqs,size_t nbSeqs)341 static size_t FUZ_getLitSize(const ZSTD_Sequence* seqs, size_t nbSeqs)
342 {
343 size_t n, litSize = 0;
344 assert(seqs != NULL);
345 for (n=0; n<nbSeqs; n++) {
346 litSize += seqs[n].litLength;
347 }
348 return litSize;
349 }
350
351 static void
FUZ_transferLiterals(void * dst,size_t dstCapacity,const void * src,size_t srcSize,const ZSTD_Sequence * seqs,size_t nbSeqs)352 FUZ_transferLiterals(void* dst, size_t dstCapacity,
353 const void* src, size_t srcSize,
354 const ZSTD_Sequence* seqs, size_t nbSeqs)
355 {
356 size_t n;
357 const char* ip = (const char*)src;
358 char* op = (char*)dst;
359 size_t const litSize = FUZ_getLitSize(seqs, nbSeqs);
360 assert(litSize <= dstCapacity);
361 for (n=0; n<nbSeqs; n++) {
362 size_t const ll = seqs[n].litLength;
363 memcpy(op, ip, ll);
364 op += ll;
365 ip += ll + seqs[n].matchLength;
366 }
367 assert((size_t)(ip - (const char*)src) == srcSize);
368 }
369
370 #ifdef ZSTD_MULTITHREAD
371
372 typedef struct {
373 ZSTD_CCtx* cctx;
374 ZSTD_threadPool* pool;
375 void* CNBuffer;
376 size_t CNBuffSize;
377 void* compressedBuffer;
378 size_t compressedBufferSize;
379 void* decodedBuffer;
380 int err;
381 } threadPoolTests_compressionJob_payload;
382
threadPoolTests_compressionJob(void * payload)383 static void* threadPoolTests_compressionJob(void* payload) {
384 threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload;
385 size_t cSize;
386 if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1;
387 cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize);
388 if (ZSTD_isError(cSize)) args->err = 1;
389 if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1;
390 return payload;
391 }
392
threadPoolTests(void)393 static int threadPoolTests(void) {
394 int testResult = 0;
395 size_t err;
396
397 size_t const CNBuffSize = 5 MB;
398 void* const CNBuffer = malloc(CNBuffSize);
399 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
400 void* const compressedBuffer = malloc(compressedBufferSize);
401 void* const decodedBuffer = malloc(CNBuffSize);
402
403 size_t const kPoolNumThreads = 8;
404
405 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0);
406
407 DISPLAYLEVEL(3, "thread pool test : threadPool reuse roundtrips: ");
408 {
409 ZSTD_CCtx* cctx = ZSTD_createCCtx();
410 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
411
412 size_t nbThreads = 1;
413 for (; nbThreads <= kPoolNumThreads; ++nbThreads) {
414 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
415 ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads);
416 err = ZSTD_CCtx_refThreadPool(cctx, pool);
417 if (ZSTD_isError(err)) {
418 DISPLAYLEVEL(3, "refThreadPool error!\n");
419 ZSTD_freeCCtx(cctx);
420 goto _output_error;
421 }
422 err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
423 if (ZSTD_isError(err)) {
424 DISPLAYLEVEL(3, "Compression error!\n");
425 ZSTD_freeCCtx(cctx);
426 goto _output_error;
427 }
428 err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err);
429 if (ZSTD_isError(err)) {
430 DISPLAYLEVEL(3, "Decompression error!\n");
431 ZSTD_freeCCtx(cctx);
432 goto _output_error;
433 }
434 }
435
436 ZSTD_freeCCtx(cctx);
437 ZSTD_freeThreadPool(pool);
438 }
439 DISPLAYLEVEL(3, "OK \n");
440
441 DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: ");
442 {
443 void* const decodedBuffer2 = malloc(CNBuffSize);
444 void* const compressedBuffer2 = malloc(compressedBufferSize);
445 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
446 ZSTD_CCtx* cctx1 = ZSTD_createCCtx();
447 ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
448
449 ZSTD_pthread_t t1;
450 ZSTD_pthread_t t2;
451 threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize,
452 compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */};
453 threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize,
454 compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */};
455
456 ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2);
457 ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2);
458 ZSTD_CCtx_refThreadPool(cctx1, pool);
459 ZSTD_CCtx_refThreadPool(cctx2, pool);
460
461 ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
462 ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
463 ZSTD_pthread_join(t1);
464 ZSTD_pthread_join(t2);
465
466 assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
467 free(decodedBuffer2);
468 free(compressedBuffer2);
469
470 ZSTD_freeThreadPool(pool);
471 ZSTD_freeCCtx(cctx1);
472 ZSTD_freeCCtx(cctx2);
473
474 if (p1.err || p2.err) goto _output_error;
475 }
476 DISPLAYLEVEL(3, "OK \n");
477
478 _end:
479 free(CNBuffer);
480 free(compressedBuffer);
481 free(decodedBuffer);
482 return testResult;
483
484 _output_error:
485 testResult = 1;
486 DISPLAY("Error detected in Unit tests ! \n");
487 goto _end;
488 }
489 #endif /* ZSTD_MULTITHREAD */
490
491 /*=============================================
492 * Unit tests
493 =============================================*/
494
test_compressBound(unsigned tnb)495 static void test_compressBound(unsigned tnb)
496 {
497 DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
498
499 /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
500 * for a large range of known valid values */
501 DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
502 { int s;
503 for (s=0; s<30; s++) {
504 size_t const w = (size_t)1 << s;
505 CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
506 } }
507
508 /* Ensure error if srcSize too big */
509 { size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
510 CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
511 CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
512 }
513
514 DISPLAYLEVEL(3, "OK \n");
515 }
516
test_decompressBound(unsigned tnb)517 static void test_decompressBound(unsigned tnb)
518 {
519 DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
520
521 /* Simple compression, with size : should provide size; */
522 { const char example[] = "abcd";
523 char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
524 size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
525 CHECK_Z(cSize);
526 CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
527 }
528
529 /* Simple small compression without size : should provide 1 block size */
530 { char cBuffer[ZSTD_COMPRESSBOUND(0)];
531 ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
532 ZSTD_inBuffer in = { NULL, 0, 0 };
533 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
534 assert(cctx);
535 CHECK_Z( ZSTD_initCStream(cctx, 0) );
536 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
537 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
538 CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
539 ZSTD_freeCCtx(cctx);
540 }
541
542 /* Attempt to overflow 32-bit intermediate multiplication result
543 * This requires dBound >= 4 GB, aka 2^32.
544 * This requires 2^32 / 2^17 = 2^15 blocks
545 * => create 2^15 blocks (can be empty, or just 1 byte). */
546 { const char input[] = "a";
547 size_t const nbBlocks = (1 << 15) + 1;
548 size_t blockNb;
549 size_t const outCapacity = 1 << 18; /* large margin */
550 char* const outBuffer = malloc (outCapacity);
551 ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
552 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
553 assert(cctx);
554 assert(outBuffer);
555 CHECK_Z( ZSTD_initCStream(cctx, 0) );
556 for (blockNb=0; blockNb<nbBlocks; blockNb++) {
557 ZSTD_inBuffer in = { input, sizeof(input), 0 };
558 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
559 CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
560 }
561 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
562 CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000ULL /* 4 GB */ );
563 ZSTD_freeCCtx(cctx);
564 free(outBuffer);
565 }
566
567 DISPLAYLEVEL(3, "OK \n");
568 }
569
test_setCParams(unsigned tnb)570 static void test_setCParams(unsigned tnb)
571 {
572 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
573 ZSTD_compressionParameters cparams;
574 assert(cctx);
575
576 DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
577
578 /* valid cparams */
579 cparams = ZSTD_getCParams(1, 0, 0);
580 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
581
582 /* invalid cparams (must fail) */
583 cparams.windowLog = 99;
584 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
585
586 free(cctx);
587 DISPLAYLEVEL(3, "OK \n");
588 }
589
test_blockSplitter_incompressibleExpansionProtection(unsigned testNb,unsigned seed)590 static void test_blockSplitter_incompressibleExpansionProtection(unsigned testNb, unsigned seed)
591 {
592 DISPLAYLEVEL(3, "test%3i : Check block splitter doesn't oversplit incompressible data (seed %u): ", testNb, seed);
593 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
594 size_t const srcSize = 256 * 1024; /* needs to be at least 2 blocks */
595 void* incompressible = malloc(srcSize);
596 size_t const dstCapacity = ZSTD_compressBound(srcSize);
597 void* cBuffer = malloc(dstCapacity);
598 size_t const chunkSize = 8 KB;
599 size_t const nbChunks = srcSize / chunkSize;
600 size_t chunkNb, cSizeNoSplit, cSizeWithSplit;
601 assert(cctx != NULL);
602 assert(incompressible != NULL);
603 assert(cBuffer != NULL);
604
605 /* let's fill input with random noise (incompressible) */
606 RDG_genBuffer(incompressible, srcSize, 0.0, 0.0, seed);
607
608 /* this pattern targets the fastest _byChunk variant's sampling (level 3).
609 * manually checked that, without the @savings protection, it would over-split.
610 */
611 for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
612 BYTE* const p = (BYTE*)incompressible + chunkNb * chunkSize;
613 size_t const samplingRate = 43;
614 int addOrRemove = chunkNb % 2;
615 size_t n;
616 for (n=0; n<chunkSize; n+=samplingRate) {
617 if (addOrRemove) {
618 p[n] &= 0x80;
619 } else {
620 p[n] |= 0x80;
621 }
622 }
623 }
624
625 /* run first without splitting */
626 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 1 /* no split */);
627 cSizeNoSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize);
628
629 /* run with sample43 splitter, check it's still the same */
630 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 3 /* sample43, fastest _byChunk variant */);
631 cSizeWithSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize);
632
633 if (cSizeWithSplit != cSizeNoSplit) {
634 DISPLAYLEVEL(1, "invalid compressed size: cSizeWithSplit %u != %u cSizeNoSplit \n",
635 (unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit);
636 abort();
637 }
638 DISPLAYLEVEL(4, "compressed size: cSizeWithSplit %u == %u cSizeNoSplit : ",
639 (unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit);
640
641 free(incompressible);
642 free(cBuffer);
643 ZSTD_freeCCtx(cctx);
644 }
645 DISPLAYLEVEL(3, "OK \n");
646 }
647
648 /* ============================================================= */
649
basicUnitTests(U32 const seed,double compressibility)650 static int basicUnitTests(U32 const seed, double compressibility)
651 {
652 size_t const CNBuffSize = 5 MB;
653 void* const CNBuffer = malloc(CNBuffSize);
654 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
655 void* const compressedBuffer = malloc(compressedBufferSize);
656 void* const decodedBuffer = malloc(CNBuffSize);
657 int testResult = 0;
658 unsigned testNb=0;
659 size_t cSize;
660
661 /* Create compressible noise */
662 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
663 DISPLAY("Not enough memory, aborting\n");
664 testResult = 1;
665 goto _end;
666 }
667 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
668
669 /* Basic tests */
670 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++);
671 { const char* errorString = ZSTD_getErrorName(0);
672 DISPLAYLEVEL(3, "OK : %s \n", errorString);
673 }
674
675 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++);
676 { const char* errorString = ZSTD_getErrorName(499);
677 DISPLAYLEVEL(3, "OK : %s \n", errorString);
678 }
679
680 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++);
681 { int const mcl = ZSTD_minCLevel();
682 DISPLAYLEVEL(3, "%i (OK) \n", mcl);
683 }
684
685 DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++);
686 { int const defaultCLevel = ZSTD_defaultCLevel();
687 if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error;
688 DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel);
689 }
690
691 DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
692 { unsigned const vn = ZSTD_versionNumber();
693 DISPLAYLEVEL(3, "%u (OK) \n", vn);
694 }
695
696 test_compressBound(testNb++);
697
698 test_decompressBound(testNb++);
699
700 test_setCParams(testNb++);
701
702 DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
703 {
704 ZSTD_compressionParameters params;
705 memset(¶ms, 0, sizeof(params));
706 params.windowLog = 10;
707 params.hashLog = 19;
708 params.chainLog = 19;
709 params = ZSTD_adjustCParams(params, 1000, 100000);
710 if (params.hashLog != 18) goto _output_error;
711 if (params.chainLog != 17) goto _output_error;
712 }
713 DISPLAYLEVEL(3, "OK \n");
714
715 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
716 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
717 if (cctx==NULL) goto _output_error;
718 CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
719 compressedBuffer, compressedBufferSize,
720 CNBuffer, CNBuffSize, 1) );
721 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
722
723 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
724 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
725 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize);
726 }
727 ZSTD_freeCCtx(cctx);
728 }
729
730 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++);
731 {
732 char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
733 size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8);
734 if (!ZSTD_isError(size)) goto _output_error;
735 }
736 DISPLAYLEVEL(3, "OK \n");
737
738 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
739 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
740 if (rSize != CNBuffSize) goto _output_error;
741 }
742 DISPLAYLEVEL(3, "OK \n");
743
744 DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++);
745 { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
746 if (rSize != CNBuffSize) goto _output_error;
747 }
748 DISPLAYLEVEL(3, "OK \n");
749
750 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
751 { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
752 if (rSize != CNBuffSize) goto _output_error;
753 }
754 DISPLAYLEVEL(3, "OK \n");
755
756 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
757 {
758 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
759 if (bound != CNBuffSize) goto _output_error;
760 }
761 DISPLAYLEVEL(3, "OK \n");
762
763 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
764 {
765 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
766 if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
767 }
768 DISPLAYLEVEL(3, "OK \n");
769
770 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
771 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
772 if (r != CNBuffSize) goto _output_error; }
773 DISPLAYLEVEL(3, "OK \n");
774
775 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
776 {
777 ZSTD_DCtx* dctx = ZSTD_createDCtx();
778 size_t r;
779 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
780 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
781 if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
782 ZSTD_freeDCtx(dctx);
783 }
784 DISPLAYLEVEL(3, "OK \n");
785
786 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
787 { size_t u;
788 for (u=0; u<CNBuffSize; u++) {
789 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
790 } }
791 DISPLAYLEVEL(3, "OK \n");
792
793 DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++);
794 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
795 ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 };
796 ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 };
797 if (cctx==NULL) goto _output_error;
798 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */
799 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */
800 ZSTD_freeCCtx(cctx);
801 }
802 DISPLAYLEVEL(3, "OK \n");
803
804 DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++);
805 {
806 ZSTD_parameters params = ZSTD_getParams(3, 0, 0);
807 assert(!ZSTD_checkCParams(params.cParams));
808 }
809 DISPLAYLEVEL(3, "OK \n");
810
811 DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++);
812 {
813 ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
814 assert(dctx != NULL);
815 assert(ZSTD_sizeof_DCtx(dctx) != 0);
816 ZSTD_freeDCtx(dctx);
817 }
818 DISPLAYLEVEL(3, "OK \n");
819
820 DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++);
821 {
822 /* %p takes a void*. In ISO C, it's illegal to cast a function pointer
823 * to a data pointer. (Although in POSIX you're required to be allowed
824 * to do it...) So we have to fall back to our trusty friend memcpy. */
825 unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) =
826 ZSTD_getDictID_fromDDict;
827 ZSTD_DStream* (* const funcptr_createDStream)(
828 ZSTD_customMem customMem) = ZSTD_createDStream_advanced;
829 void (* const funcptr_copyDCtx)(
830 ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx;
831 ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) =
832 ZSTD_nextInputType;
833 const void *voidptr_getDictID;
834 const void *voidptr_createDStream;
835 const void *voidptr_copyDCtx;
836 const void *voidptr_nextInputType;
837 DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID));
838 memcpy(
839 (void*)&voidptr_getDictID,
840 (const void*)&funcptr_getDictID,
841 sizeof(void*));
842 memcpy(
843 (void*)&voidptr_createDStream,
844 (const void*)&funcptr_createDStream,
845 sizeof(void*));
846 memcpy(
847 (void*)&voidptr_copyDCtx,
848 (const void*)&funcptr_copyDCtx,
849 sizeof(void*));
850 memcpy(
851 (void*)&voidptr_nextInputType,
852 (const void*)&funcptr_nextInputType,
853 sizeof(void*));
854 DISPLAYLEVEL(3, "%p ", voidptr_getDictID);
855 DISPLAYLEVEL(3, "%p ", voidptr_createDStream);
856 DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx);
857 DISPLAYLEVEL(3, "%p ", voidptr_nextInputType);
858 }
859 DISPLAYLEVEL(3, ": OK \n");
860
861 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
862 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
863 { size_t const r = ZSTD_decompress_usingDict(dctx,
864 decodedBuffer, CNBuffSize,
865 compressedBuffer, cSize,
866 NULL, 0);
867 if (r != CNBuffSize) goto _output_error;
868 }
869 ZSTD_freeDCtx(dctx);
870 }
871 DISPLAYLEVEL(3, "OK \n");
872
873 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
874 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
875 { size_t const r = ZSTD_decompress_usingDDict(dctx,
876 decodedBuffer, CNBuffSize,
877 compressedBuffer, cSize,
878 NULL);
879 if (r != CNBuffSize) goto _output_error;
880 }
881 ZSTD_freeDCtx(dctx);
882 }
883 DISPLAYLEVEL(3, "OK \n");
884
885 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
886 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
887 if (!ZSTD_isError(r)) goto _output_error;
888 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
889 DISPLAYLEVEL(3, "OK \n");
890
891 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
892 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
893 if (!ZSTD_isError(r)) goto _output_error;
894 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
895 DISPLAYLEVEL(3, "OK \n");
896
897 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++);
898 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize);
899 if (!ZSTD_isError(r)) goto _output_error;
900 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
901 DISPLAYLEVEL(3, "OK \n");
902
903 DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++);
904 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize);
905 if (!ZSTD_isError(r)) goto _output_error;
906 if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; }
907 DISPLAYLEVEL(3, "OK \n");
908
909 DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++);
910 { /* create compressed buffer with checksumming enabled */
911 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
912 if (!cctx) {
913 DISPLAY("Not enough memory, aborting\n");
914 testResult = 1;
915 goto _end;
916 }
917 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
918 CHECK_VAR(cSize, ZSTD_compress2(cctx,
919 compressedBuffer, compressedBufferSize,
920 CNBuffer, CNBuffSize) );
921 ZSTD_freeCCtx(cctx);
922 }
923 { /* copy the compressed buffer and corrupt the checksum */
924 size_t r;
925 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
926 if (!dctx) {
927 DISPLAY("Not enough memory, aborting\n");
928 testResult = 1;
929 goto _end;
930 }
931
932 ((char*)compressedBuffer)[cSize-1] += 1;
933 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
934 if (!ZSTD_isError(r)) goto _output_error;
935 if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error;
936
937 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum));
938 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
939 if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */
940 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
941 if (ZSTD_isError(r)) goto _output_error;
942
943 ZSTD_freeDCtx(dctx);
944 }
945 DISPLAYLEVEL(3, "OK \n");
946
947
948 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
949 { /* create compressed buffer with content size missing */
950 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
951 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
952 CHECK_VAR(cSize, ZSTD_compress2(cctx,
953 compressedBuffer, compressedBufferSize,
954 CNBuffer, CNBuffSize) );
955 ZSTD_freeCCtx(cctx);
956 }
957 { /* ensure frame content size is missing */
958 ZSTD_FrameHeader zfh;
959 size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
960 if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
961 }
962 { /* ensure CNBuffSize <= decompressBound */
963 unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
964 if (CNBuffSize > bound) goto _output_error;
965 }
966 DISPLAYLEVEL(3, "OK \n");
967
968 DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++);
969 {
970 size_t const largeFrameSrcSize = 200;
971 size_t const smallFrameSrcSize = 10;
972 size_t const nbFrames = 256;
973
974 size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0;
975 int sizeReduced = 0;
976
977 BYTE* const dst = (BYTE*)compressedBuffer;
978 ZSTD_DCtx* dctx = ZSTD_createDCtx();
979
980 /* create a large frame and then a bunch of small frames */
981 size_t srcSize = ZSTD_compress((void*)dst,
982 compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
983 for (i = 0; i < nbFrames; i++)
984 srcSize += ZSTD_compress((void*)(dst + srcSize),
985 compressedBufferSize - srcSize, CNBuffer,
986 smallFrameSrcSize, 3);
987
988 /* decompressStream and make sure that dctx size was reduced at least once */
989 while (consumed < srcSize) {
990 ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
991 ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0};
992 ZSTD_decompressStream(dctx, &out, &in);
993 consumed += in.pos;
994 produced += out.pos;
995
996 /* success! size was reduced from the previous frame */
997 if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx))
998 sizeReduced = 1;
999
1000 prevDCtxSize = ZSTD_sizeof_DCtx(dctx);
1001 }
1002
1003 assert(sizeReduced);
1004
1005 ZSTD_freeDCtx(dctx);
1006 }
1007 DISPLAYLEVEL(3, "OK \n");
1008
1009 {
1010 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1011 ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1);
1012 ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
1013 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
1014
1015 DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++);
1016 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1));
1017 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
1018 DISPLAYLEVEL(3, "OK \n");
1019
1020 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++);
1021 CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1));
1022 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
1023 DISPLAYLEVEL(3, "OK \n");
1024
1025 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++);
1026 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict));
1027 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
1028 DISPLAYLEVEL(3, "OK \n");
1029
1030 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++);
1031 CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params));
1032 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
1033 DISPLAYLEVEL(3, "OK \n");
1034
1035 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++);
1036 CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams));
1037 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
1038 DISPLAYLEVEL(3, "OK \n");
1039
1040 ZSTD_freeCDict(cdict);
1041 ZSTD_freeCCtx(cctx);
1042 }
1043
1044 DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2K", testNb++);
1045 {
1046 ZSTD_CCtx* cctx = ZSTD_createCCtx();
1047 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1048 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1049 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048));
1050 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048));
1051
1052 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1053 CHECK_Z(cSize);
1054 CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize));
1055
1056 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024));
1057 CHECK(ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize)));
1058
1059 ZSTD_freeDCtx(dctx);
1060 ZSTD_freeCCtx(cctx);
1061 }
1062
1063 DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
1064 {
1065 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1066
1067 size_t const size = (1U << 10);
1068 size_t const dstCapacity = ZSTD_compressBound(size);
1069 void* dict = (void*)malloc(size);
1070 void* src = (void*)malloc(size);
1071 void* dst = (void*)malloc(dstCapacity);
1072
1073 RDG_genBuffer(dict, size, 0.5, 0.5, seed);
1074 RDG_genBuffer(src, size, 0.5, 0.5, seed);
1075
1076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1077 assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
1078
1079 ZSTD_freeCCtx(cctx);
1080 free(dict);
1081 free(src);
1082 free(dst);
1083 }
1084 DISPLAYLEVEL(3, "OK \n");
1085
1086 DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
1087 {
1088 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1089 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1090 void* dict = (void*)malloc(CNBuffSize);
1091 int nbWorkers;
1092
1093 for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) {
1094 RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
1095 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
1096
1097 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
1098 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1099 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
1100 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1101 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
1102 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1103 CHECK_Z(cSize);
1104 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize));
1105 }
1106
1107 ZSTD_freeCCtx(cctx);
1108 ZSTD_freeDCtx(dctx);
1109 free(dict);
1110 }
1111 DISPLAYLEVEL(3, "OK \n");
1112
1113 DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++);
1114 {
1115 size_t const testSize = 1024;
1116 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1117 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1118 char* dict = (char*)malloc(2 * testSize);
1119 int ldmEnabled, level;
1120
1121 RDG_genBuffer(dict, testSize, 0.5, 0.5, seed);
1122 RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
1123 memcpy(dict + testSize, CNBuffer, testSize);
1124 for (level = 1; level <= 5; ++level) {
1125 for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
1126 size_t cSize0;
1127 XXH64_hash_t compressedChecksum0;
1128
1129 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1130 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level));
1131 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled));
1132 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1));
1133
1134 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1135 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize);
1136 CHECK_Z(cSize);
1137 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize));
1138
1139 cSize0 = cSize;
1140 compressedChecksum0 = XXH64(compressedBuffer, cSize, 0);
1141
1142 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1143 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize);
1144 CHECK_Z(cSize);
1145
1146 if (cSize != cSize0) goto _output_error;
1147 if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error;
1148 }
1149 }
1150
1151 ZSTD_freeCCtx(cctx);
1152 ZSTD_freeDCtx(dctx);
1153 free(dict);
1154 }
1155 DISPLAYLEVEL(3, "OK \n");
1156
1157 DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
1158 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1159 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1160 size_t const srcSize = 300 KB;
1161 size_t const flushSize = 128 KB + 5;
1162 size_t const dstSize = ZSTD_compressBound(srcSize);
1163 char* src = (char*)CNBuffer;
1164 char* dst = (char*)compressedBuffer;
1165
1166 ZSTD_outBuffer out = { dst, dstSize, 0 };
1167 ZSTD_inBuffer in = { src, flushSize, 0 };
1168
1169 if (!cctx || !dctx) {
1170 DISPLAY("Not enough memory, aborting\n");
1171 testResult = 1;
1172 goto _end;
1173 }
1174
1175 RDG_genBuffer(src, srcSize, 0.5, 0.5, seed);
1176 /* Force an LDM to exist that crosses block boundary into uncompressible block */
1177 memcpy(src + 125 KB, src, 3 KB + 5);
1178
1179 /* Enable MT, LDM, and opt parser */
1180 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
1181 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1182 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1183 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1184
1185 /* Flushes a block of 128 KB and block of 5 bytes */
1186 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1187
1188 /* Compress the rest */
1189 in.size = 300 KB;
1190 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1191
1192 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos));
1193
1194 ZSTD_freeCCtx(cctx);
1195 ZSTD_freeDCtx(dctx);
1196 }
1197 DISPLAYLEVEL(3, "OK \n");
1198
1199 DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++);
1200 {
1201 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1202 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1203 void* dict = (void*)malloc(CNBuffSize);
1204 size_t const kWindowLog = 10;
1205 size_t const kWindowSize = (size_t)1 << kWindowLog;
1206 size_t const dictSize = kWindowSize * 10;
1207 size_t const srcSize1 = kWindowSize / 2;
1208 size_t const srcSize2 = kWindowSize * 10;
1209
1210 CHECK(cctx!=NULL);
1211 CHECK(dctx!=NULL);
1212 CHECK(dict!=NULL);
1213 if (CNBuffSize < dictSize) goto _output_error;
1214
1215 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
1216 RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed);
1217
1218 /* Enable checksum to verify round trip. */
1219 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1220 /* Disable content size to skip single-pass decompression. */
1221 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
1222 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
1223 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1224 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
1225 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
1226 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
1227 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3));
1228
1229 /* Round trip once with a dictionary. */
1230 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1231 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
1232 CHECK_Z(cSize);
1233 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1234
1235 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
1236 /* Streaming decompression to catch out of bounds offsets. */
1237 {
1238 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1239 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1240 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1241 CHECK_Z(dSize);
1242 if (dSize != 0) goto _output_error;
1243 }
1244
1245 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
1246 /* Round trip once with a dictionary. */
1247 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1248 { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
1249 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1250 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1251 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1252 cSize = out.pos;
1253 }
1254 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1255
1256 { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
1257 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1258 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1259 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1260 cSize = out.pos;
1261 }
1262 /* Streaming decompression to catch out of bounds offsets. */
1263 { ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1264 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1265 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1266 CHECK_Z(dSize);
1267 if (dSize != 0) goto _output_error;
1268 }
1269
1270 ZSTD_freeCCtx(cctx);
1271 ZSTD_freeDCtx(dctx);
1272 free(dict);
1273 }
1274 DISPLAYLEVEL(3, "OK \n");
1275
1276 /* Note: this test takes 0.5 seconds to run */
1277 DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
1278 {
1279 /* test a big buffer so that ldm can take effect */
1280 size_t const size = 100 MB;
1281 int const windowLog = 27;
1282 size_t const dstSize = ZSTD_compressBound(size);
1283
1284 void* dict = (void*)malloc(size);
1285 void* src = (void*)malloc(size);
1286 void* dst = (void*)malloc(dstSize);
1287 void* recon = (void*)malloc(size);
1288
1289 size_t refPrefixCompressedSize = 0;
1290 size_t refPrefixLdmCompressedSize = 0;
1291 size_t reconSize = 0;
1292
1293 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1294 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1295
1296 /* make dict and src the same uncompressible data */
1297 RDG_genBuffer(src, size, 0, 0, seed);
1298 memcpy(dict, src, size);
1299 assert(!memcmp(dict, src, size));
1300
1301 /* set level 1 and windowLog to cover src */
1302 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1));
1303 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog));
1304
1305 /* compress on level 1 using just refPrefix and no ldm */
1306 ZSTD_CCtx_refPrefix(cctx, dict, size);
1307 refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1308 assert(!ZSTD_isError(refPrefixCompressedSize));
1309
1310 /* test round trip just refPrefix */
1311 ZSTD_DCtx_refPrefix(dctx, dict, size);
1312 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize);
1313 assert(!ZSTD_isError(reconSize));
1314 assert(reconSize == size);
1315 assert(!memcmp(recon, src, size));
1316
1317 /* compress on level 1 using refPrefix and ldm */
1318 ZSTD_CCtx_refPrefix(cctx, dict, size);;
1319 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
1320 refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1321 assert(!ZSTD_isError(refPrefixLdmCompressedSize));
1322
1323 /* test round trip refPrefix + ldm*/
1324 ZSTD_DCtx_refPrefix(dctx, dict, size);
1325 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize);
1326 assert(!ZSTD_isError(reconSize));
1327 assert(reconSize == size);
1328 assert(!memcmp(recon, src, size));
1329
1330 /* make sure that refPrefixCompressedSize is significantly greater */
1331 assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize);
1332 /* make sure the ldm compressed size is less than 1% of original */
1333 assert((double)refPrefixLdmCompressedSize / (double)size < 0.01);
1334
1335 ZSTD_freeDCtx(dctx);
1336 ZSTD_freeCCtx(cctx);
1337 free(recon);
1338 free(dict);
1339 free(src);
1340 free(dst);
1341 }
1342 DISPLAYLEVEL(3, "OK \n");
1343
1344 DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++);
1345 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX);
1346 CHECK_Z(cSize);
1347 CHECK_LT(CNBuffSize, cSize);
1348 {
1349 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1350 size_t const outputSize = (CNBuffSize + margin);
1351 char* output = malloc(outputSize);
1352 char* input = output + outputSize - cSize;
1353 CHECK_LT(cSize, CNBuffSize + margin);
1354 CHECK(output != NULL);
1355 CHECK_Z(margin);
1356 CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX));
1357 memcpy(input, compressedBuffer, cSize);
1358
1359 {
1360 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1361 CHECK_Z(dSize);
1362 CHECK_EQ(dSize, CNBuffSize);
1363 }
1364 CHECK(!memcmp(output, CNBuffer, CNBuffSize));
1365 free(output);
1366 }
1367 DISPLAYLEVEL(3, "OK \n");
1368
1369 DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++);
1370 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1371 CHECK_Z(cSize);
1372 {
1373 size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1374 CHECK_Z(cSize2);
1375 cSize += cSize2;
1376 }
1377 {
1378 size_t const srcSize = (CNBuffSize / 3) * 2;
1379 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1380 size_t const outputSize = (CNBuffSize + margin);
1381 char* output = malloc(outputSize);
1382 char* input = output + outputSize - cSize;
1383 CHECK_LT(cSize, CNBuffSize + margin);
1384 CHECK(output != NULL);
1385 CHECK_Z(margin);
1386 memcpy(input, compressedBuffer, cSize);
1387
1388 {
1389 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1390 CHECK_Z(dSize);
1391 CHECK_EQ(dSize, srcSize);
1392 }
1393 CHECK(!memcmp(output, CNBuffer, srcSize));
1394 free(output);
1395 }
1396 DISPLAYLEVEL(3, "OK \n");
1397
1398 DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++);
1399 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1400 size_t const srcSize = 256 * 1024;
1401 U32 const compressibleLenU32 = 32 * 1024 / 4;
1402 U32 const blockSizeU32 = 128 * 1024 / 4;
1403 U32 const litLenU32 = 64 * 1024 / 4;
1404 U32* data = (U32*)malloc(srcSize);
1405 size_t dSize;
1406
1407 if (data == NULL || cctx == NULL) goto _output_error;
1408
1409 /* Generate data without any matches */
1410 RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U);
1411 /* Generate 32K of compressible data */
1412 RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe);
1413
1414 /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */
1415 data[compressibleLenU32 + 0] = 0xFFFFFFFF;
1416 data[compressibleLenU32 + 1] = 0xEEEEEEEE;
1417 data[compressibleLenU32 + 4] = 0xFFFFFFFF;
1418 data[compressibleLenU32 + 5] = 0xEEEEEEEE;
1419
1420 /* Add a match of offset=16, length=8 at idx=64K + 64.
1421 * This generates a sequence with llen=64K, and repeat code 1.
1422 * The block splitter thought this was ll0, and corrupted the
1423 * repeat offset history.
1424 */
1425 data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD;
1426 data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC;
1427 data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD;
1428 data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC;
1429
1430 /* Add a match of offset=16, length=8 at idx=128K + 16.
1431 * This should generate a sequence with repeat code = 1.
1432 * But the block splitters mistake caused zstd to generate
1433 * repeat code = 2, corrupting the data.
1434 */
1435 data[blockSizeU32] = 0xBBBBBBBB;
1436 data[blockSizeU32 + 1] = 0xAAAAAAAA;
1437 data[blockSizeU32 + 4] = 0xBBBBBBBB;
1438 data[blockSizeU32 + 5] = 0xAAAAAAAA;
1439
1440 /* Generate a golden file from this data in case datagen changes and
1441 * doesn't generate the exact same data. We will also test this golden file.
1442 */
1443 if (0) {
1444 FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb");
1445 fwrite(data, 1, srcSize, f);
1446 fclose(f);
1447 }
1448
1449 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1450 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7));
1451 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitAfterSequences, ZSTD_ps_enable));
1452
1453 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize);
1454 CHECK_Z(cSize);
1455 dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1456 CHECK_Z(dSize);
1457 CHECK_EQ(dSize, srcSize);
1458 CHECK(!memcmp(decodedBuffer, data, srcSize));
1459
1460 free(data);
1461 ZSTD_freeCCtx(cctx);
1462 }
1463 DISPLAYLEVEL(3, "OK \n");
1464
1465 test_blockSplitter_incompressibleExpansionProtection(testNb++, seed);
1466
1467 DISPLAYLEVEL(3, "test%3d : superblock uncompressible data: too many nocompress superblocks : ", testNb++);
1468 {
1469 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1470 const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;
1471 size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize);
1472
1473 /* This is the number of bytes to stream before ending. This value
1474 * was obtained by trial and error :/. */
1475
1476 const size_t streamCompressThreshold = 161792;
1477 const size_t streamCompressDelta = 1024;
1478
1479 /* The first 1/5 of the buffer is compressible and the last 4/5 is
1480 * uncompressible. This is an approximation of the type of data
1481 * the fuzzer generated to catch this bug. Streams like this were making
1482 * zstd generate noCompress superblocks (which are larger than the src
1483 * they come from). Do this enough times, and we'll run out of room
1484 * and throw a dstSize_tooSmall error. */
1485
1486 const size_t compressiblePartSize = srcSize/5;
1487 const size_t uncompressiblePartSize = srcSize-compressiblePartSize;
1488 RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed);
1489 RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed);
1490
1491 /* Setting target block size so that superblock is used */
1492
1493 assert(cctx != NULL);
1494 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81);
1495
1496 { size_t read;
1497 for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) {
1498 ZSTD_inBuffer in = {src, streamCompressDelta, 0};
1499 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1500 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1501 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1502 src += streamCompressDelta; srcSize -= streamCompressDelta;
1503 dst += out.pos; dstCapacity -= out.pos;
1504 } }
1505
1506 /* This is trying to catch a dstSize_tooSmall error */
1507
1508 { ZSTD_inBuffer in = {src, srcSize, 0};
1509 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1510 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1511 }
1512 ZSTD_freeCCtx(cctx);
1513 }
1514 DISPLAYLEVEL(3, "OK \n");
1515
1516 DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++);
1517 /* Generate the same data 20 times over */
1518 { size_t const avgChunkSize = CNBuffSize / 20;
1519 size_t b;
1520 for (b = 0; b < CNBuffSize; b += avgChunkSize) {
1521 size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize);
1522 RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed);
1523 } }
1524 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1525 size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1526 size_t const allowedExpansion = (CNBuffSize * 3 / 1000);
1527 size_t superCSize;
1528 CHECK_Z(normalCSize);
1529 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
1530 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000);
1531 superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1532 CHECK_Z(superCSize);
1533 if (superCSize > normalCSize + allowedExpansion) {
1534 DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion);
1535 goto _output_error;
1536 }
1537 ZSTD_freeCCtx(cctx);
1538 }
1539 DISPLAYLEVEL(3, "OK \n");
1540
1541 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed);
1542 DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++)
1543 /* This tests whether or not we leave enough room for the checksum at the end
1544 * of the dst buffer. The bug that motivated this test was found by the
1545 * stream_round_trip fuzzer but this crashes for the same reason and is
1546 * far more compact than re-creating the stream_round_trip fuzzer's code path */
1547 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1548 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64);
1549 assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278)));
1550 ZSTD_freeCCtx(cctx);
1551 }
1552 DISPLAYLEVEL(3, "OK \n");
1553
1554 DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
1555 { int level = -1;
1556 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1557 if (!cctx) goto _output_error;
1558 for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
1559 CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
1560 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
1561 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
1562 }
1563 ZSTD_freeCCtx(cctx);
1564 }
1565 DISPLAYLEVEL(3, "OK \n");
1566
1567 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
1568 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1569 size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
1570 if (ZSTD_isError(r)) goto _output_error;
1571 if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
1572 ZSTD_freeCCtx(cctx);
1573 cSize = r;
1574 }
1575 DISPLAYLEVEL(3, "OK \n");
1576
1577 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++);
1578 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize);
1579 if (ZSTD_isError(r)) goto _output_error;
1580 if (r != 0) goto _output_error;
1581 }
1582 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1583 ZSTD_outBuffer output;
1584 if (cctx==NULL) goto _output_error;
1585 output.dst = compressedBuffer;
1586 output.size = compressedBufferSize;
1587 output.pos = 0;
1588 CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */
1589 CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */
1590 CHECK_Z( ZSTD_endStream(cctx, &output) );
1591 ZSTD_freeCCtx(cctx);
1592 /* single scan decompression */
1593 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos);
1594 if (ZSTD_isError(r)) goto _output_error;
1595 if (r != 0) goto _output_error;
1596 }
1597 /* streaming decompression */
1598 { ZSTD_DCtx* const dstream = ZSTD_createDStream();
1599 ZSTD_inBuffer dinput;
1600 ZSTD_outBuffer doutput;
1601 size_t ipos;
1602 if (dstream==NULL) goto _output_error;
1603 dinput.src = compressedBuffer;
1604 dinput.size = 0;
1605 dinput.pos = 0;
1606 doutput.dst = NULL;
1607 doutput.size = 0;
1608 doutput.pos = 0;
1609 CHECK_Z ( ZSTD_initDStream(dstream) );
1610 for (ipos=1; ipos<=output.pos; ipos++) {
1611 dinput.size = ipos;
1612 CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) );
1613 }
1614 if (doutput.pos != 0) goto _output_error;
1615 ZSTD_freeDStream(dstream);
1616 }
1617 }
1618 DISPLAYLEVEL(3, "OK \n");
1619
1620 DISPLAYLEVEL(3, "test%3d : reuse CCtx with expanding block size : ", testNb++);
1621 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1622 ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1623 assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
1624 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
1625 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
1626
1627 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */
1628 { size_t const inSize = 2* 128 KB;
1629 size_t const outSize = ZSTD_compressBound(inSize);
1630 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
1631 /* will fail if blockSize is not resized */
1632 }
1633 ZSTD_freeCCtx(cctx);
1634 }
1635 DISPLAYLEVEL(3, "OK \n");
1636
1637 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++);
1638 { size_t const sampleSize = 30;
1639 int i;
1640 for (i=0; i<20; i++)
1641 ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */
1642 memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
1643 for (i=1; i<=19; i++) {
1644 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1645 size_t size1, size2;
1646 DISPLAYLEVEL(5, "l%i ", i);
1647 size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1648 CHECK_Z(size1);
1649
1650 size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1651 CHECK_Z(size2);
1652 CHECK_EQ(size1, size2);
1653
1654 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) );
1655 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize);
1656 CHECK_Z(size2);
1657 CHECK_EQ(size1, size2);
1658
1659 size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */
1660 CHECK_Z(size2);
1661 CHECK_EQ(size1, size2);
1662
1663 { ZSTD_inBuffer inb;
1664 ZSTD_outBuffer outb;
1665 inb.src = CNBuffer;
1666 inb.pos = 0;
1667 inb.size = sampleSize;
1668 outb.dst = compressedBuffer;
1669 outb.pos = 0;
1670 outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */
1671 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );
1672 assert(inb.pos == inb.size);
1673 CHECK_EQ(size1, outb.pos);
1674 }
1675
1676 ZSTD_freeCCtx(cctx);
1677 }
1678 }
1679 DISPLAYLEVEL(3, "OK \n");
1680
1681 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++);
1682 { size_t const sampleSize = 1024;
1683 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1684 ZSTD_inBuffer inb;
1685 ZSTD_outBuffer outb;
1686 inb.src = CNBuffer;
1687 inb.pos = 0;
1688 inb.size = 0;
1689 outb.dst = compressedBuffer;
1690 outb.pos = 0;
1691 outb.size = compressedBufferSize;
1692 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) );
1693
1694 inb.size = sampleSize; /* start with something, so that context is already used */
1695 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1696 assert(inb.pos == inb.size);
1697 outb.pos = 0; /* cancel output */
1698
1699 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) );
1700 inb.size = 4; /* too small size : compression will be skipped */
1701 inb.pos = 0;
1702 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1703 assert(inb.pos == inb.size);
1704
1705 inb.size += 5; /* too small size : compression will be skipped */
1706 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1707 assert(inb.pos == inb.size);
1708
1709 inb.size += 11; /* small enough to attempt compression */
1710 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1711 assert(inb.pos == inb.size);
1712
1713 assert(inb.pos < sampleSize);
1714 inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */
1715 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1716 assert(inb.pos == inb.size);
1717 ZSTD_freeCCtx(cctx);
1718 }
1719 DISPLAYLEVEL(3, "OK \n");
1720
1721 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
1722 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1723 ZSTD_outBuffer out = {NULL, 0, 0};
1724 ZSTD_inBuffer in = {NULL, 0, 0};
1725 int value;
1726
1727 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1728 CHECK_EQ(value, 3);
1729 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1730 CHECK_EQ(value, 0);
1731 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
1732 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1733 CHECK_EQ(value, 3);
1734 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1735 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1736 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
1737 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1738 CHECK_EQ(value, 7);
1739 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1740 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1741 /* Start a compression job */
1742 ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1743 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1744 CHECK_EQ(value, 7);
1745 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1746 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1747 /* Reset the CCtx */
1748 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
1749 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1750 CHECK_EQ(value, 7);
1751 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1752 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1753 /* Reset the parameters */
1754 ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
1755 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1756 CHECK_EQ(value, 3);
1757 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1758 CHECK_EQ(value, 0);
1759
1760 ZSTD_freeCCtx(cctx);
1761 }
1762 DISPLAYLEVEL(3, "OK \n");
1763
1764 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++);
1765 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1766 int value;
1767 ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0);
1768 cparams.strategy = (ZSTD_strategy)-1; /* set invalid value, on purpose */
1769 /* Set invalid cParams == error out, and no change. */
1770 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
1771
1772 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1773 CHECK_EQ(value, 0);
1774 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1775 CHECK_EQ(value, 0);
1776 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1777 CHECK_EQ(value, 0);
1778 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1779 CHECK_EQ(value, 0);
1780 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1781 CHECK_EQ(value, 0);
1782 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1783 CHECK_EQ(value, 0);
1784 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1785 CHECK_EQ(value, 0);
1786
1787 cparams = ZSTD_getCParams(12, 0, 0);
1788 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
1789
1790 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1791 CHECK_EQ(value, (int)cparams.windowLog);
1792 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1793 CHECK_EQ(value, (int)cparams.chainLog);
1794 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1795 CHECK_EQ(value, (int)cparams.hashLog);
1796 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1797 CHECK_EQ(value, (int)cparams.searchLog);
1798 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1799 CHECK_EQ(value, (int)cparams.minMatch);
1800 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1801 CHECK_EQ(value, (int)cparams.targetLength);
1802 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1803 CHECK_EQ(value, (int)cparams.strategy);
1804
1805 ZSTD_freeCCtx(cctx);
1806 }
1807
1808 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++);
1809 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1810 int value;
1811 ZSTD_frameParameters fparams = {0, 1, 1};
1812
1813 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1814 CHECK_EQ(value, 1);
1815 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1816 CHECK_EQ(value, 0);
1817 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1818 CHECK_EQ(value, 1);
1819
1820 CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams));
1821
1822 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1823 CHECK_EQ(value, fparams.contentSizeFlag);
1824 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1825 CHECK_EQ(value, fparams.checksumFlag);
1826 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1827 CHECK_EQ(value, !fparams.noDictIDFlag);
1828
1829 ZSTD_freeCCtx(cctx);
1830 }
1831
1832 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setParams() : ", testNb++);
1833 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1834 int value;
1835 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
1836 params.cParams.strategy = (ZSTD_strategy)-1; /* set invalid value, on purpose */
1837 /* Set invalid params == error out, and no change. */
1838 CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params)));
1839
1840 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1841 CHECK_EQ(value, 0);
1842 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1843 CHECK_EQ(value, 0);
1844 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1845 CHECK_EQ(value, 0);
1846 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1847 CHECK_EQ(value, 0);
1848 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1849 CHECK_EQ(value, 0);
1850 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1851 CHECK_EQ(value, 0);
1852 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1853 CHECK_EQ(value, 0);
1854 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1855 CHECK_EQ(value, 1);
1856 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1857 CHECK_EQ(value, 0);
1858 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1859 CHECK_EQ(value, 1);
1860
1861 params = ZSTD_getParams(12, 0, 0);
1862 params.fParams.contentSizeFlag = 0;
1863 params.fParams.checksumFlag = 1;
1864 params.fParams.noDictIDFlag = 1;
1865 CHECK_Z(ZSTD_CCtx_setParams(cctx, params));
1866
1867 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1868 CHECK_EQ(value, (int)params.cParams.windowLog);
1869 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1870 CHECK_EQ(value, (int)params.cParams.chainLog);
1871 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1872 CHECK_EQ(value, (int)params.cParams.hashLog);
1873 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1874 CHECK_EQ(value, (int)params.cParams.searchLog);
1875 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1876 CHECK_EQ(value, (int)params.cParams.minMatch);
1877 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1878 CHECK_EQ(value, (int)params.cParams.targetLength);
1879 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1880 CHECK_EQ(value, (int)params.cParams.strategy);
1881 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1882 CHECK_EQ(value, params.fParams.contentSizeFlag);
1883 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1884 CHECK_EQ(value, params.fParams.checksumFlag);
1885 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1886 CHECK_EQ(value, !params.fParams.noDictIDFlag);
1887
1888 ZSTD_freeCCtx(cctx);
1889 }
1890
1891 DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
1892 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1893 ZSTD_outBuffer out = {NULL, 0, 0};
1894 ZSTD_inBuffer in = {NULL, 0, 0};
1895 int value;
1896
1897 /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt),
1898 * we should not modify the actual parameter specified by the user within the CCtx
1899 */
1900 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27));
1901 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt));
1902
1903 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1904 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value));
1905 CHECK_EQ(value, 0);
1906
1907 ZSTD_freeCCtx(cctx);
1908 }
1909 DISPLAYLEVEL(3, "OK \n");
1910
1911 /* this test is really too long, and should be made faster */
1912 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
1913 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1914 ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1915 size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
1916 size_t cnb;
1917 assert(cctx != NULL);
1918 params.fParams.contentSizeFlag = 0;
1919 params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
1920 for (cnb = 0; cnb < nbCompressions; ++cnb) {
1921 DISPLAYLEVEL(6, "run %u / %u \n", (unsigned)cnb, (unsigned)nbCompressions);
1922 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */
1923 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
1924 }
1925 ZSTD_freeCCtx(cctx);
1926 }
1927 DISPLAYLEVEL(3, "OK \n");
1928
1929 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
1930 { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
1931 assert(largeCCtx != NULL);
1932 CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
1933 CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
1934 { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
1935 { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
1936 assert(smallCCtx != NULL);
1937 CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1938 { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
1939 DISPLAYLEVEL(5, "(large) %uKB > 32*%uKB (small) : ",
1940 (unsigned)(largeCCtxSize>>10), (unsigned)(smallCCtxSize>>10));
1941 assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
1942 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
1943 }
1944 ZSTD_freeCCtx(smallCCtx);
1945 }
1946 { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
1947 * currently defined as 128x, but could be adjusted in the future.
1948 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
1949 unsigned u;
1950 for (u=0; u<maxNbAttempts; u++) {
1951 CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1952 if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
1953 }
1954 DISPLAYLEVEL(5, "size down after %u attempts : ", u);
1955 if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
1956 }
1957 }
1958 ZSTD_freeCCtx(largeCCtx);
1959 }
1960 DISPLAYLEVEL(3, "OK \n");
1961
1962 /* Static CCtx tests */
1963 #define STATIC_CCTX_LEVEL 4
1964 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1965 { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
1966 void* const staticCCtxBuffer = malloc(staticCStreamSize);
1967 size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
1968 void* const staticDCtxBuffer = malloc(staticDCtxSize);
1969 DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize);
1970 if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
1971 free(staticCCtxBuffer);
1972 free(staticDCtxBuffer);
1973 DISPLAY("Not enough memory, aborting\n");
1974 testResult = 1;
1975 goto _end;
1976 }
1977 { size_t const smallInSize = 32 KB;
1978 ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0);
1979 size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small);
1980 size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL);
1981 ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize);
1982 ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
1983 DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize);
1984 DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize);
1985 if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
1986 DISPLAYLEVEL(3, "OK \n");
1987
1988 DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++);
1989 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1990 compressedBuffer, compressedBufferSize,
1991 CNBuffer, smallInSize, STATIC_CCTX_LEVEL) );
1992 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1993 (unsigned)cSize, (double)cSize/smallInSize*100);
1994
1995 DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++);
1996 { size_t const r = ZSTD_compressCCtx(staticCCtx,
1997 compressedBuffer, compressedBufferSize,
1998 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL);
1999 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error;
2000 }
2001 DISPLAYLEVEL(3, "OK \n");
2002
2003 DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
2004 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
2005 DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx);
2006 if (staticCCtx == NULL) goto _output_error;
2007 DISPLAYLEVEL(3, "OK \n");
2008
2009 DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++);
2010 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2011 compressedBuffer, compressedBufferSize,
2012 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
2013 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
2014 (unsigned)cSize, (double)cSize/CNBuffSize*100);
2015
2016 DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++);
2017 { int nbc;
2018 assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */
2019 assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3);
2020 for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) {
2021 CHECK_Z(ZSTD_compressCCtx(staticCCtx,
2022 compressedBuffer, compressedBufferSize,
2023 (char*)CNBuffer + nbc, smallInSize,
2024 STATIC_CCTX_LEVEL) );
2025 } }
2026 DISPLAYLEVEL(3, "OK \n")
2027
2028 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
2029 CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) );
2030 DISPLAYLEVEL(3, "OK \n");
2031
2032 DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++);
2033 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2034 compressedBuffer, compressedBufferSize,
2035 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
2036 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
2037 (unsigned)cSize, (double)cSize/CNBuffSize*100);
2038
2039 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
2040 { size_t const r = ZSTD_decompressDCtx(staticDCtx,
2041 decodedBuffer, CNBuffSize,
2042 compressedBuffer, cSize);
2043 if (r != CNBuffSize) goto _output_error; }
2044 DISPLAYLEVEL(3, "OK \n");
2045
2046 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2047 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2048 DISPLAYLEVEL(3, "OK \n");
2049
2050 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
2051 { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
2052 if (!ZSTD_isError(r)) goto _output_error; }
2053 DISPLAYLEVEL(3, "OK \n");
2054
2055 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
2056 CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
2057 DISPLAYLEVEL(3, "OK \n");
2058
2059 DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
2060 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
2061 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
2062 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
2063 size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */
2064 if (!ZSTD_isError(r)) goto _output_error;
2065 }
2066 DISPLAYLEVEL(3, "OK \n");
2067
2068 DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++);
2069 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize);
2070 assert(staticCCtx != NULL);
2071 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
2072 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
2073 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
2074 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
2075 }
2076 DISPLAYLEVEL(3, "OK \n");
2077
2078 DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1);
2079 CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */
2080 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
2081 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
2082 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
2083 }
2084 DISPLAYLEVEL(3, "OK \n");
2085
2086 DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++);
2087 { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
2088 if (!ZSTD_isError(r)) goto _output_error; }
2089 DISPLAYLEVEL(3, "OK \n");
2090
2091 DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++);
2092 CHECK_Z( ZSTD_initDStream(staticDCtx) );
2093 { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
2094 ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
2095 size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
2096 if (!ZSTD_isError(r)) goto _output_error;
2097 }
2098 DISPLAYLEVEL(3, "OK \n");
2099
2100 DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
2101 {
2102 // Test ZSTD_estimateCCtxSize_usingCCtxParams
2103 {
2104 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2105 size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2106 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2107 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2108 compressedBuffer, compressedBufferSize,
2109 CNBuffer, CNBuffSize, 3));
2110
2111 {
2112 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2113 decodedBuffer, CNBuffSize,
2114 compressedBuffer, cSize);
2115 if (r != CNBuffSize) goto _output_error;
2116 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2117 }
2118 ZSTD_freeCCtxParams(params);
2119 }
2120
2121 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2122 {
2123 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2124 size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2125 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2126 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2127 compressedBuffer, compressedBufferSize,
2128 CNBuffer, CNBuffSize, 3) );
2129
2130 {
2131 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2132 decodedBuffer, CNBuffSize,
2133 compressedBuffer, cSize);
2134 if (r != CNBuffSize) goto _output_error;
2135 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2136 }
2137 ZSTD_freeCCtxParams(params);
2138 }
2139 }
2140 DISPLAYLEVEL(3, "OK \n");
2141
2142 DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
2143 {
2144 // Test ZSTD_estimateCCtxSize_usingCCtxParams
2145 {
2146 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2147 size_t cctxSizeDefault;
2148 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2149 cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2150 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2151 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2152 compressedBuffer, compressedBufferSize,
2153 CNBuffer, CNBuffSize, 3) );
2154
2155 {
2156 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2157 decodedBuffer, CNBuffSize,
2158 compressedBuffer, cSize);
2159 if (r != CNBuffSize) goto _output_error;
2160 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2161 }
2162 ZSTD_freeCCtxParams(params);
2163 }
2164
2165 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2166 {
2167 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2168 size_t cctxSizeDefault;
2169 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2170 cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2171 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2172 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2173 compressedBuffer, compressedBufferSize,
2174 CNBuffer, CNBuffSize, 3) );
2175
2176 {
2177 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2178 decodedBuffer, CNBuffSize,
2179 compressedBuffer, cSize);
2180 if (r != CNBuffSize) goto _output_error;
2181 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2182 }
2183 ZSTD_freeCCtxParams(params);
2184 }
2185 }
2186 DISPLAYLEVEL(3, "OK \n");
2187 }
2188 free(staticCCtxBuffer);
2189 free(staticDCtxBuffer);
2190 }
2191
2192 DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++);
2193 { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1);
2194 size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1);
2195 size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1);
2196 size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1);
2197
2198 if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error;
2199 if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error;
2200 }
2201 DISPLAYLEVEL(3, "OK \n");
2202
2203
2204 /* ZSTDMT simple MT compression test */
2205 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
2206 { ZSTD_CCtx* const mtctx = ZSTD_createCCtx();
2207 if (mtctx==NULL) {
2208 DISPLAY("mtctx : not enough memory, aborting \n");
2209 testResult = 1;
2210 goto _end;
2211 }
2212 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
2213 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
2214 DISPLAYLEVEL(3, "OK \n");
2215
2216 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
2217 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2218 compressedBuffer, compressedBufferSize,
2219 CNBuffer, CNBuffSize) );
2220 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2221
2222 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
2223 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2224 if (rSize != CNBuffSize) {
2225 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize);
2226 goto _output_error;
2227 } }
2228 DISPLAYLEVEL(3, "OK \n");
2229
2230 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2231 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2232 if (r != CNBuffSize) goto _output_error; }
2233 DISPLAYLEVEL(3, "OK \n");
2234
2235 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2236 { size_t u;
2237 for (u=0; u<CNBuffSize; u++) {
2238 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
2239 } }
2240 DISPLAYLEVEL(3, "OK \n");
2241
2242 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
2243 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
2244 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
2245 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
2246 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2247 compressedBuffer, compressedBufferSize,
2248 CNBuffer, CNBuffSize) );
2249 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2250
2251 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2252 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2253 if (r != CNBuffSize) goto _output_error; }
2254 DISPLAYLEVEL(3, "OK \n");
2255
2256 ZSTD_freeCCtx(mtctx);
2257 }
2258
2259 DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++);
2260 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2261 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2262 char out[32];
2263 if (cctx == NULL || dctx == NULL) goto _output_error;
2264 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
2265 CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
2266 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
2267
2268 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
2269 { char const* outPtr = out;
2270 ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
2271 ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
2272 size_t dSize;
2273 CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) );
2274 if (dSize != 0) goto _output_error;
2275 }
2276
2277 ZSTD_freeDCtx(dctx);
2278 ZSTD_freeCCtx(cctx);
2279 }
2280
2281 DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
2282 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2283 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitAfterSequences, ZSTD_ps_enable) );
2284 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2285 CHECK_Z(cSize);
2286 ZSTD_freeCCtx(cctx);
2287 }
2288 DISPLAYLEVEL(3, "OK \n");
2289
2290 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
2291 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2292 size_t cSize1, cSize2;
2293 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2294 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
2295 cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2296 CHECK_Z(cSize1);
2297 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
2298 cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2299 CHECK_Z(cSize2);
2300 CHECK_LT(cSize1, cSize2);
2301 ZSTD_freeCCtx(cctx);
2302 }
2303 DISPLAYLEVEL(3, "OK \n");
2304
2305 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++)
2306 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2307 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
2308 * ZSTDMT is forced to not take the shortcut.
2309 */
2310 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2311 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
2312 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
2313 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
2314 ZSTD_freeCCtx(cctx);
2315 }
2316 DISPLAYLEVEL(3, "OK \n");
2317
2318 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
2319 { ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2320 int const jobSize = 512 KB;
2321 int value;
2322 /* Check that the overlap log and job size are unset. */
2323 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2324 CHECK_EQ(value, 0);
2325 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2326 CHECK_EQ(value, 0);
2327 /* Set and check the overlap log and job size. */
2328 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
2329 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
2330 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2331 CHECK_EQ(value, 5);
2332 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2333 CHECK_EQ(value, jobSize);
2334 /* Set the number of workers and check the overlap log and job size. */
2335 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
2336 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2337 CHECK_EQ(value, 5);
2338 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2339 CHECK_EQ(value, jobSize);
2340 ZSTD_freeCCtxParams(params);
2341 }
2342 DISPLAYLEVEL(3, "OK \n");
2343
2344 /* Simple API multiframe test */
2345 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);
2346 { size_t off = 0;
2347 int i;
2348 int const segs = 4;
2349 /* only use the first half so we don't push against size limit of compressedBuffer */
2350 size_t const segSize = (CNBuffSize / 2) / segs;
2351
2352 const U32 skipLen = 129 KB;
2353 char* const skipBuff = (char*)malloc(skipLen);
2354 assert(skipBuff != NULL);
2355 memset(skipBuff, 0, skipLen);
2356 for (i = 0; i < segs; i++) {
2357 CHECK_NEWV(r, ZSTD_compress(
2358 (BYTE*)compressedBuffer + off, CNBuffSize - off,
2359 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
2360 5) );
2361 off += r;
2362 if (i == segs/2) {
2363 /* insert skippable frame */
2364 size_t const skippableSize =
2365 ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize,
2366 skipBuff, skipLen, seed % 15);
2367 CHECK_Z(skippableSize);
2368 off += skippableSize;
2369 }
2370 }
2371 cSize = off;
2372 free(skipBuff);
2373 }
2374 DISPLAYLEVEL(3, "OK \n");
2375
2376 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++);
2377 { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
2378 if (r != CNBuffSize / 2) goto _output_error; }
2379 DISPLAYLEVEL(3, "OK \n");
2380
2381 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
2382 { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize);
2383 if (bound != CNBuffSize / 2) goto _output_error; }
2384 DISPLAYLEVEL(3, "OK \n");
2385
2386 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
2387 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
2388 if (r != CNBuffSize / 2) goto _output_error; }
2389 DISPLAYLEVEL(3, "OK \n");
2390
2391 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2392 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
2393 DISPLAYLEVEL(3, "OK \n");
2394
2395 /* Simple API skippable frame test */
2396 DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++);
2397 { U32 i;
2398 unsigned readMagic;
2399 unsigned long long receivedSize;
2400 size_t skippableSize;
2401 const U32 skipLen = 129 KB;
2402 char* const skipBuff = (char*)malloc(skipLen);
2403 assert(skipBuff != NULL);
2404 for (i = 0; i < skipLen; i++)
2405 skipBuff[i] = (char) ((seed + i) % 256);
2406 skippableSize = ZSTD_writeSkippableFrame(
2407 compressedBuffer, compressedBufferSize,
2408 skipBuff, skipLen, seed % 15);
2409 CHECK_Z(skippableSize);
2410 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2411 receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize);
2412 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2413 CHECK_EQ(seed % 15, readMagic);
2414 if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error;
2415
2416 free(skipBuff);
2417 }
2418 DISPLAYLEVEL(3, "OK \n");
2419
2420 DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++);
2421 {
2422 unsigned readMagic;
2423 unsigned long long receivedSize;
2424 size_t skippableSize;
2425 skippableSize = ZSTD_writeSkippableFrame(
2426 compressedBuffer, compressedBufferSize,
2427 CNBuffer, 0, seed % 15);
2428 CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize);
2429 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2430 receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize);
2431 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2432 CHECK_EQ(seed % 15, readMagic);
2433 }
2434 DISPLAYLEVEL(3, "OK \n");
2435
2436 /* Dictionary and CCtx Duplication tests */
2437 { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
2438 ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
2439 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2440 static const size_t dictSize = 551;
2441 assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
2442
2443 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
2444 { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
2445 if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
2446 DISPLAYLEVEL(3, "OK \n");
2447
2448 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
2449 CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
2450 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
2451 DISPLAYLEVEL(3, "OK \n");
2452
2453 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
2454 cSize = 0;
2455 CHECKPLUS(r, ZSTD_compressEnd(ctxOrig,
2456 compressedBuffer, compressedBufferSize,
2457 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2458 cSize += r);
2459 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2460
2461 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
2462 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2463 decodedBuffer, CNBuffSize,
2464 compressedBuffer, cSize,
2465 CNBuffer, dictSize),
2466 if (r != CNBuffSize - dictSize) goto _output_error);
2467 DISPLAYLEVEL(3, "OK \n");
2468
2469 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
2470 { size_t const cSizeOrig = cSize;
2471 cSize = 0;
2472 CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated,
2473 compressedBuffer, compressedBufferSize,
2474 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2475 cSize += r);
2476 if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
2477 }
2478 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2479
2480 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
2481 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2482 decodedBuffer, CNBuffSize,
2483 compressedBuffer, cSize,
2484 CNBuffer, dictSize),
2485 if (r != CNBuffSize - dictSize) goto _output_error);
2486 DISPLAYLEVEL(3, "OK \n");
2487
2488 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++);
2489 { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
2490 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2491 if (r != CNBuffSize - dictSize) goto _output_error;
2492 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict));
2493 ZSTD_freeDDict(ddict);
2494 }
2495
2496 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
2497 { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
2498 void* const ddictBuffer = malloc(ddictBufferSize);
2499 if (ddictBuffer == NULL) goto _output_error;
2500 { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
2501 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2502 if (r != CNBuffSize - dictSize) goto _output_error;
2503 }
2504 free(ddictBuffer);
2505 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize);
2506 }
2507
2508 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
2509 { size_t const testSize = CNBuffSize / 3;
2510 CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
2511 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
2512
2513 CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
2514 (const char*)CNBuffer + dictSize, testSize) );
2515 { ZSTD_FrameHeader zfh;
2516 if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
2517 if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
2518 } }
2519 DISPLAYLEVEL(3, "OK \n");
2520
2521 #if !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \
2522 && !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \
2523 && !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \
2524 && !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
2525 && !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
2526 && !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \
2527 && !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
2528 && !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
2529 /* Note : these tests should be replaced by proper regression tests,
2530 * but existing ones do not focus on small data + dictionary + all levels.
2531 */
2532 if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
2533 size_t const flatdictSize = 22 KB;
2534 size_t const contentSize = 9 KB;
2535 const void* const dict = (const char*)CNBuffer;
2536 const void* const contentStart = (const char*)dict + flatdictSize;
2537 /* These upper bounds are generally within a few bytes of the compressed size */
2538 size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
2539 3770, 3770, 3770, 3750, 3750,
2540 3742, 3675, 3674, 3665, 3664,
2541 3663, 3662, 3661, 3660, 3660,
2542 3660, 3660, 3660 };
2543 size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2840, 2950,
2544 2950, 2950, 2925, 2900, 2892,
2545 2910, 2910, 2910, 2780, 2775,
2546 2765, 2760, 2755, 2754, 2753,
2547 2753, 2753, 2753 };
2548 int l = 1;
2549 int const maxLevel = ZSTD_maxCLevel();
2550 /* clevels with strategies that support rowhash on small inputs */
2551 int rowLevel = 4;
2552 int const rowLevelEnd = 8;
2553
2554 DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
2555 assert(maxLevel == 22);
2556 RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
2557 DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n",
2558 (unsigned long long)XXH64(contentStart, contentSize, 0),
2559 (unsigned long long)XXH64(dict, flatdictSize, 0));
2560
2561 for ( ; l <= maxLevel; l++) {
2562 size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize,
2563 contentStart, contentSize, l);
2564 if (nodict_cSize > target_nodict_cSize[l]) {
2565 DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n",
2566 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2567 goto _output_error;
2568 }
2569 DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n",
2570 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2571 }
2572 for ( l=1 ; l <= maxLevel; l++) {
2573 size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig,
2574 compressedBuffer, compressedBufferSize,
2575 contentStart, contentSize,
2576 dict, flatdictSize,
2577 l);
2578 if (wdict_cSize > target_wdict_cSize[l]) {
2579 DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n",
2580 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2581 goto _output_error;
2582 }
2583 DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
2584 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2585 }
2586 /* Compression with ZSTD_compress2 and row match finder force enabled.
2587 * Give some slack for force-enabled row matchfinder since we're on a small input (9KB)
2588 */
2589 for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5;
2590 for (l=1 ; l <= maxLevel; l++) {
2591 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2592 size_t nodict_cSize;
2593 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l);
2594 ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable);
2595 nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize,
2596 contentStart, contentSize);
2597 if (nodict_cSize > target_nodict_cSize[l]) {
2598 DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n",
2599 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2600 ZSTD_freeCCtx(cctx);
2601 goto _output_error;
2602 }
2603 DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n",
2604 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2605 ZSTD_freeCCtx(cctx);
2606 }
2607 /* Dict compression with DMS */
2608 for ( l=1 ; l <= maxLevel; l++) {
2609 size_t wdict_cSize;
2610 CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) );
2611 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) );
2612 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) );
2613 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) );
2614 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) );
2615 wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize);
2616 if (wdict_cSize > target_wdict_cSize[l]) {
2617 DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n",
2618 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2619 goto _output_error;
2620 }
2621 DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n",
2622 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2623 }
2624
2625 DISPLAYLEVEL(4, "compression efficiency tests OK \n");
2626 }
2627 #endif
2628
2629 ZSTD_freeCCtx(ctxOrig);
2630 ZSTD_freeCCtx(ctxDuplicated);
2631 ZSTD_freeDCtx(dctx);
2632 }
2633
2634 /* Dictionary and dictBuilder tests */
2635 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2636 size_t const dictBufferCapacity = 16 KB;
2637 void* const dictBuffer = malloc(dictBufferCapacity);
2638 size_t const totalSampleSize = 1 MB;
2639 size_t const sampleUnitSize = 8 KB;
2640 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
2641 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
2642 size_t dictSize;
2643 U32 dictID;
2644 size_t dictHeaderSize;
2645 size_t dictBufferFixedSize = 144;
2646 unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
2647 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2648 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
2649 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
2650 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2651 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2652 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
2653 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
2654 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
2655 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
2656 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
2657 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69};
2658
2659 if (dictBuffer==NULL || samplesSizes==NULL) {
2660 free(dictBuffer);
2661 free(samplesSizes);
2662 goto _output_error;
2663 }
2664
2665 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++);
2666 assert(compressedBufferSize >= totalSampleSize);
2667 { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; }
2668 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2669 { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2670 decodedBuffer, samplesSizes, nbSamples);
2671 if (ZDICT_isError(sDictSize)) goto _output_error;
2672 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize);
2673 }
2674
2675 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++);
2676 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2677 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2678 CNBuffer, samplesSizes, nbSamples);
2679 if (ZDICT_isError(dictSize)) goto _output_error;
2680 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2681
2682 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++);
2683 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2684 { ZDICT_cover_params_t coverParams;
2685 memset(&coverParams, 0, sizeof(coverParams));
2686 coverParams.steps = 8;
2687 coverParams.nbThreads = 4;
2688 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2689 dictBuffer, dictBufferCapacity,
2690 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2691 &coverParams);
2692 if (ZDICT_isError(dictSize)) goto _output_error;
2693 }
2694 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2695
2696 DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++);
2697 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2698 { ZDICT_cover_params_t coverParams;
2699 memset(&coverParams, 0, sizeof(coverParams));
2700 coverParams.steps = 8;
2701 coverParams.nbThreads = 4;
2702 coverParams.shrinkDict = 1;
2703 coverParams.shrinkDictMaxRegression = 1;
2704 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2705 dictBuffer, dictBufferCapacity,
2706 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2707 &coverParams);
2708 if (ZDICT_isError(dictSize)) goto _output_error;
2709 }
2710 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2711
2712 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++);
2713 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2714 { ZDICT_fastCover_params_t fastCoverParams;
2715 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2716 fastCoverParams.steps = 8;
2717 fastCoverParams.nbThreads = 4;
2718 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2719 dictBuffer, dictBufferCapacity,
2720 CNBuffer, samplesSizes, nbSamples,
2721 &fastCoverParams);
2722 if (ZDICT_isError(dictSize)) goto _output_error;
2723 }
2724 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2725
2726 DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++);
2727 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2728 { ZDICT_fastCover_params_t fastCoverParams;
2729 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2730 fastCoverParams.steps = 8;
2731 fastCoverParams.nbThreads = 4;
2732 fastCoverParams.shrinkDict = 1;
2733 fastCoverParams.shrinkDictMaxRegression = 1;
2734 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2735 dictBuffer, dictBufferCapacity,
2736 CNBuffer, samplesSizes, nbSamples,
2737 &fastCoverParams);
2738 if (ZDICT_isError(dictSize)) goto _output_error;
2739 }
2740 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2741
2742 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2743 dictID = ZDICT_getDictID(dictBuffer, dictSize);
2744 if (dictID==0) goto _output_error;
2745 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2746
2747 DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++);
2748 dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize);
2749 if (dictHeaderSize==0) goto _output_error;
2750 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2751
2752 DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
2753 { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize);
2754 if (dictHeaderSize != 115) goto _output_error;
2755 }
2756 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2757
2758 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
2759 cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
2760 CNBuffer, CNBuffSize,
2761 dictBuffer, dictSize, 4);
2762 if (ZSTD_isError(cSize)) goto _output_error;
2763 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2764
2765 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++);
2766 { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
2767 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2768 }
2769 DISPLAYLEVEL(3, "OK \n");
2770
2771 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2772 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2773 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2774 }
2775 DISPLAYLEVEL(3, "OK \n");
2776
2777 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2778 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2779 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2780 decodedBuffer, CNBuffSize,
2781 compressedBuffer, cSize,
2782 dictBuffer, dictSize),
2783 if (r != CNBuffSize) goto _output_error);
2784 ZSTD_freeDCtx(dctx);
2785 }
2786 DISPLAYLEVEL(3, "OK \n");
2787
2788 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
2789 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2790 size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef);
2791 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize);
2792 }
2793
2794 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
2795 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2796 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
2797 ZSTD_dlm_byRef, ZSTD_dct_auto,
2798 cParams, ZSTD_defaultCMem);
2799 assert(cdict != NULL);
2800 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict));
2801 assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict));
2802 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
2803 CNBuffer, CNBuffSize, cdict);
2804 ZSTD_freeCDict(cdict);
2805 if (ZSTD_isError(cSize)) goto _output_error;
2806 }
2807 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2808
2809 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2810 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2811 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2812 }
2813 DISPLAYLEVEL(3, "OK \n");
2814
2815 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2816 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2817 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2818 decodedBuffer, CNBuffSize,
2819 compressedBuffer, cSize,
2820 dictBuffer, dictSize),
2821 if (r != CNBuffSize) goto _output_error);
2822 ZSTD_freeDCtx(dctx);
2823 }
2824 DISPLAYLEVEL(3, "OK \n");
2825
2826 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
2827 { int const maxLevel = ZSTD_maxCLevel();
2828 int level;
2829 for (level = 1; level <= maxLevel; ++level) {
2830 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize);
2831 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
2832 void* const cdictBuffer = malloc(cdictSize);
2833 if (cdictBuffer==NULL) goto _output_error;
2834 { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict(
2835 cdictBuffer, cdictSize,
2836 dictBuffer, dictSize,
2837 ZSTD_dlm_byCopy, ZSTD_dct_auto,
2838 cParams);
2839 if (cdict == NULL) {
2840 DISPLAY("ZSTD_initStaticCDict failed ");
2841 goto _output_error;
2842 }
2843 cSize = ZSTD_compress_usingCDict(cctx,
2844 compressedBuffer, compressedBufferSize,
2845 CNBuffer, MIN(10 KB, CNBuffSize), cdict);
2846 if (ZSTD_isError(cSize)) {
2847 DISPLAY("ZSTD_compress_usingCDict failed ");
2848 goto _output_error;
2849 } }
2850 free(cdictBuffer);
2851 } }
2852 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2853
2854 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
2855 { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
2856 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2857 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
2858 assert(cdict != NULL);
2859 cSize = ZSTD_compress_usingCDict_advanced(cctx,
2860 compressedBuffer, compressedBufferSize,
2861 CNBuffer, CNBuffSize,
2862 cdict, fParams);
2863 ZSTD_freeCDict(cdict);
2864 if (ZSTD_isError(cSize)) goto _output_error;
2865 }
2866 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2867
2868 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++);
2869 { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2870 if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
2871 }
2872 DISPLAYLEVEL(3, "OK (unknown)\n");
2873
2874 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2875 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2876 assert(dctx != NULL);
2877 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2878 decodedBuffer, CNBuffSize,
2879 compressedBuffer, cSize,
2880 dictBuffer, dictSize),
2881 if (r != CNBuffSize) goto _output_error);
2882 ZSTD_freeDCtx(dctx);
2883 }
2884 DISPLAYLEVEL(3, "OK \n");
2885
2886 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
2887 { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
2888 p.fParams.noDictIDFlag = 1;
2889 cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
2890 CNBuffer, CNBuffSize,
2891 dictBuffer, dictSize, p);
2892 if (ZSTD_isError(cSize)) goto _output_error;
2893 }
2894 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2895
2896 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2897 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2898 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2899 decodedBuffer, CNBuffSize,
2900 compressedBuffer, cSize,
2901 dictBuffer, dictSize),
2902 if (r != CNBuffSize) goto _output_error);
2903 ZSTD_freeDCtx(dctx);
2904 }
2905 DISPLAYLEVEL(3, "OK \n");
2906
2907 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
2908 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2909 assert(dctx != NULL);
2910 { const size_t ret = ZSTD_decompress_usingDict(
2911 dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
2912 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
2913 if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
2914 goto _output_error;
2915 }
2916 ZSTD_freeDCtx(dctx);
2917 }
2918 DISPLAYLEVEL(3, "OK \n");
2919
2920 DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++);
2921 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2922 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2923 ZSTD_frameParameters const fParams = { 0, 1, 0 };
2924 size_t cBlockSize;
2925 cSize = 0;
2926 CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN));
2927 cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000);
2928 CHECK_Z(cBlockSize);
2929 cSize += cBlockSize;
2930 cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000);
2931 CHECK_Z(cBlockSize);
2932 cSize += cBlockSize;
2933
2934 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
2935
2936 ZSTD_freeCDict(cdict);
2937 ZSTD_freeDCtx(dctx);
2938 }
2939 DISPLAYLEVEL(3, "OK \n");
2940
2941 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
2942 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2943 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2944 if (cdict==NULL) goto _output_error;
2945 ZSTD_freeCDict(cdict);
2946 }
2947 DISPLAYLEVEL(3, "OK \n");
2948
2949 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
2950 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2951 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2952 if (cdict!=NULL) goto _output_error;
2953 ZSTD_freeCDict(cdict);
2954 }
2955 DISPLAYLEVEL(3, "OK \n");
2956
2957 { char* rawDictBuffer = (char*)malloc(dictSize);
2958 assert(rawDictBuffer);
2959 memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2);
2960 memset(rawDictBuffer + dictSize - 2, 0, 2);
2961 MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY);
2962
2963 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
2964 {
2965 size_t ret;
2966 /* Either operation is allowed to fail, but one must fail. */
2967 ret = ZSTD_CCtx_loadDictionary_advanced(
2968 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
2969 if (!ZSTD_isError(ret)) {
2970 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2971 if (!ZSTD_isError(ret)) goto _output_error;
2972 }
2973 }
2974 DISPLAYLEVEL(3, "OK \n");
2975
2976 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
2977 {
2978 size_t ret;
2979 ret = ZSTD_CCtx_loadDictionary_advanced(
2980 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
2981 if (ZSTD_isError(ret)) goto _output_error;
2982 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2983 if (ZSTD_isError(ret)) goto _output_error;
2984 }
2985 DISPLAYLEVEL(3, "OK \n");
2986
2987 DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++);
2988 { size_t const srcSize = MIN(CNBuffSize, 100);
2989 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2990 /* Force the dictionary to be reloaded in raw content mode */
2991 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad));
2992 CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent));
2993 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
2994 CHECK_Z(cSize);
2995 }
2996 DISPLAYLEVEL(3, "OK \n");
2997
2998 free(rawDictBuffer);
2999 }
3000
3001 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
3002 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
3003 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3004 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
3005 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
3006 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
3007 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
3008 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
3009 ZSTD_freeCDict(cdict);
3010 }
3011 DISPLAYLEVEL(3, "OK \n");
3012
3013 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++);
3014 {
3015 size_t size1, size2;
3016 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3017 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
3018 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
3019 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3020 if (ZSTD_isError(size1)) goto _output_error;
3021
3022 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3023 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
3024 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
3025 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3026 if (ZSTD_isError(size2)) goto _output_error;
3027
3028 if (size1 != size2) goto _output_error;
3029 }
3030 DISPLAYLEVEL(3, "OK \n");
3031
3032 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++);
3033 {
3034 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
3035 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
3036 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
3037 }
3038 DISPLAYLEVEL(3, "OK \n");
3039
3040 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++);
3041 {
3042 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
3043 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
3044 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
3045 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
3046 ZSTD_freeCDict(cdict);
3047 }
3048 DISPLAYLEVEL(3, "OK \n");
3049
3050 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++);
3051 {
3052 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
3053 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
3054 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
3055 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
3056 ZSTD_freeCDict(cdict);
3057 }
3058 DISPLAYLEVEL(3, "OK \n");
3059
3060 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++);
3061 {
3062 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
3063 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
3064 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
3065 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
3066 ZSTD_freeCDict(cdict);
3067 }
3068 DISPLAYLEVEL(3, "OK \n");
3069
3070 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++);
3071 {
3072 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
3073 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
3074 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
3075 }
3076 DISPLAYLEVEL(3, "OK \n");
3077
3078 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++);
3079 {
3080 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
3081 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
3082 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
3083 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
3084 ZSTD_freeCDict(cdict);
3085 }
3086 DISPLAYLEVEL(3, "OK \n");
3087
3088 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++);
3089 {
3090 size_t size1, size2;
3091 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3092 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
3093 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3094 if (ZSTD_isError(size1)) goto _output_error;
3095
3096 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
3097 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3098 if (ZSTD_isError(size2)) goto _output_error;
3099
3100 if (size1 != size2) goto _output_error;
3101 }
3102 DISPLAYLEVEL(3, "OK \n");
3103
3104 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++);
3105 {
3106 size_t size1, size2;
3107 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3108 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
3109 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3110 if (ZSTD_isError(size1)) goto _output_error;
3111
3112 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3113 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3114 if (ZSTD_isError(size2)) goto _output_error;
3115
3116 if (size1 == size2) goto _output_error;
3117 }
3118 DISPLAYLEVEL(3, "OK \n");
3119
3120 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3121 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) );
3122 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3123 CHECK_Z(cSize);
3124 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++);
3125 {
3126 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3127 size_t ret;
3128 /* We should fail to decompress without a dictionary. */
3129 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3130 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3131 if (!ZSTD_isError(ret)) goto _output_error;
3132 /* We should succeed to decompress with the dictionary. */
3133 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3134 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) );
3135 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3136 /* The dictionary should persist across calls. */
3137 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3138 /* When we reset the context the dictionary is cleared. */
3139 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3140 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3141 if (!ZSTD_isError(ret)) goto _output_error;
3142 ZSTD_freeDCtx(dctx);
3143 }
3144 DISPLAYLEVEL(3, "OK \n");
3145
3146 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
3147 {
3148 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3149 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
3150 size_t ret;
3151 /* We should succeed to decompress with the ddict. */
3152 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3153 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
3154 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3155 /* The ddict should persist across calls. */
3156 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3157 /* When we reset the context the ddict is cleared. */
3158 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3159 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3160 if (!ZSTD_isError(ret)) goto _output_error;
3161 ZSTD_freeDCtx(dctx);
3162 ZSTD_freeDDict(ddict);
3163 }
3164 DISPLAYLEVEL(3, "OK \n");
3165
3166 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
3167 {
3168 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3169 size_t ret;
3170 /* We should succeed to decompress with the prefix. */
3171 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3172 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
3173 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3174 /* The prefix should be cleared after the first compression. */
3175 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3176 if (!ZSTD_isError(ret)) goto _output_error;
3177 ZSTD_freeDCtx(dctx);
3178 }
3179 DISPLAYLEVEL(3, "OK \n");
3180
3181 DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3182 {
3183 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3184 ZSTD_customMem customMem = {NULL, NULL, NULL};
3185 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3186 ZSTD_CDict* cdict;
3187 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast));
3188 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3189 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3190 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3191 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3192 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3193 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3194 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3195 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3196 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3197 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3198 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3199 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3200 CHECK_Z(cSize);
3201 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3202 ZSTD_freeCDict(cdict);
3203 ZSTD_freeDCtx(dctx);
3204 ZSTD_freeCCtxParams(cctxParams);
3205 }
3206 DISPLAYLEVEL(3, "OK \n");
3207
3208 DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3209 {
3210 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3211 ZSTD_customMem customMem = {NULL, NULL, NULL};
3212 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3213 ZSTD_CDict* cdict;
3214 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast));
3215 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3216 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3217 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3218 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3219 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3220 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3221 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3222 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3223 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3224 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3225 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3226 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3227 CHECK_Z(cSize);
3228 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3229 ZSTD_freeCDict(cdict);
3230 ZSTD_freeDCtx(dctx);
3231 ZSTD_freeCCtxParams(cctxParams);
3232 }
3233 DISPLAYLEVEL(3, "OK \n");
3234
3235 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++);
3236 if (MEM_64bits()) {
3237 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3238 ZSTD_customMem customMem = {NULL, NULL, NULL};
3239 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3240 ZSTD_CDict* cdict;
3241 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy));
3242 /* Force enable row based match finder, and disable dedicated dict search. */
3243 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
3244 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0));
3245 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4));
3246 /* Set windowLog to 29 so hash/chain logs don't get sized down */
3247 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29));
3248 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29));
3249 /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */
3250 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29));
3251 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3252 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3253 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3254 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3255 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3256 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3257 CHECK_Z(cSize);
3258 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3259 ZSTD_freeCDict(cdict);
3260 ZSTD_freeDCtx(dctx);
3261 ZSTD_freeCCtxParams(cctxParams);
3262 }
3263 DISPLAYLEVEL(3, "OK \n");
3264
3265 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
3266 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3267 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
3268 CNBuffer, samplesSizes, nbSamples);
3269 if (ZDICT_isError(dictSize)) goto _output_error;
3270 /* Set all the repcodes to non-default */
3271 {
3272 BYTE* dictPtr = (BYTE*)dictBuffer;
3273 BYTE* dictLimit = dictPtr + dictSize - 12;
3274 /* Find the repcodes */
3275 while (dictPtr < dictLimit &&
3276 (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
3277 MEM_readLE32(dictPtr + 8) != 8)) {
3278 ++dictPtr;
3279 }
3280 if (dictPtr >= dictLimit) goto _output_error;
3281 MEM_writeLE32(dictPtr + 0, 10);
3282 MEM_writeLE32(dictPtr + 4, 10);
3283 MEM_writeLE32(dictPtr + 8, 10);
3284 /* Set the last 8 bytes to 'x' */
3285 memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
3286 }
3287 /* The optimal parser checks all the repcodes.
3288 * Make sure at least one is a match >= targetLength so that it is
3289 * immediately chosen. This will make sure that the compressor and
3290 * decompressor agree on at least one of the repcodes.
3291 */
3292 { size_t dSize;
3293 BYTE data[1024];
3294 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3295 ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
3296 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
3297 ZSTD_dlm_byRef, ZSTD_dct_auto,
3298 cParams, ZSTD_defaultCMem);
3299 assert(dctx != NULL); assert(cdict != NULL);
3300 memset(data, 'x', sizeof(data));
3301 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
3302 data, sizeof(data), cdict);
3303 ZSTD_freeCDict(cdict);
3304 if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
3305 dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
3306 if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
3307 if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
3308 ZSTD_freeDCtx(dctx);
3309 }
3310 DISPLAYLEVEL(3, "OK \n");
3311
3312 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++);
3313 {
3314 const size_t numDicts = 128;
3315 const size_t numFrames = 4;
3316 size_t i;
3317 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3318 ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts);
3319 ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts);
3320 U32 dictIDSeed = seed;
3321 /* Create new compressed buffer that will hold frames with differing dictIDs */
3322 char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */
3323
3324 ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize);
3325 /* Create a bunch of DDicts with random dict IDs */
3326 for (i = 0; i < numDicts; ++i) {
3327 U32 currDictID = FUZ_rand(&dictIDSeed);
3328 MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID);
3329 ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize);
3330 cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3);
3331 if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) {
3332 goto _output_error;
3333 }
3334 }
3335 /* Compress a few frames using random CDicts */
3336 {
3337 size_t off = 0;
3338 /* only use the first half so we don't push against size limit of compressedBuffer */
3339 size_t const segSize = (CNBuffSize / 2) / numFrames;
3340 for (i = 0; i < numFrames; i++) {
3341 size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts;
3342 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3343 { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx,
3344 (BYTE*)compressedBuffer + off, CNBuffSize - off,
3345 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
3346 cdictTable[dictIdx]));
3347 off += r;
3348 }
3349 }
3350 cSize = off;
3351 }
3352
3353 /* We should succeed to decompression even though different dicts were used on different frames */
3354 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3355 ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts);
3356 /* Reference every single ddict we made */
3357 for (i = 0; i < numDicts; ++i) {
3358 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i]));
3359 }
3360 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3361 /* Streaming decompression should also work */
3362 {
3363 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
3364 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
3365 while (in.pos < in.size) {
3366 CHECK_Z(ZSTD_decompressStream(dctx, &out, &in));
3367 }
3368 }
3369 ZSTD_freeDCtx(dctx);
3370 for (i = 0; i < numDicts; ++i) {
3371 ZSTD_freeCDict(cdictTable[i]);
3372 ZSTD_freeDDict(ddictTable[i]);
3373 }
3374 free(dictBufferMulti);
3375 free(ddictTable);
3376 free(cdictTable);
3377 }
3378 DISPLAYLEVEL(3, "OK \n");
3379
3380 ZSTD_freeCCtx(cctx);
3381 free(dictBuffer);
3382 free(samplesSizes);
3383 }
3384
3385 /* COVER dictionary builder tests */
3386 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3387 size_t dictSize = 16 KB;
3388 size_t optDictSize = dictSize;
3389 void* dictBuffer = malloc(dictSize);
3390 size_t const totalSampleSize = 1 MB;
3391 size_t const sampleUnitSize = 8 KB;
3392 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3393 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3394 U32 seed32 = seed;
3395 ZDICT_cover_params_t params;
3396 U32 dictID;
3397
3398 if (dictBuffer==NULL || samplesSizes==NULL) {
3399 free(dictBuffer);
3400 free(samplesSizes);
3401 goto _output_error;
3402 }
3403
3404 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
3405 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3406 memset(¶ms, 0, sizeof(params));
3407 params.d = 1 + (FUZ_rand(&seed32) % 16);
3408 params.k = params.d + (FUZ_rand(&seed32) % 256);
3409 dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize,
3410 CNBuffer, samplesSizes, nbSamples,
3411 params);
3412 if (ZDICT_isError(dictSize)) goto _output_error;
3413 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
3414
3415 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3416 dictID = ZDICT_getDictID(dictBuffer, dictSize);
3417 if (dictID==0) goto _output_error;
3418 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3419
3420 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
3421 memset(¶ms, 0, sizeof(params));
3422 params.steps = 4;
3423 optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize,
3424 CNBuffer, samplesSizes,
3425 nbSamples / 4, ¶ms);
3426 if (ZDICT_isError(optDictSize)) goto _output_error;
3427 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize);
3428
3429 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3430 dictID = ZDICT_getDictID(dictBuffer, optDictSize);
3431 if (dictID==0) goto _output_error;
3432 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3433
3434 ZSTD_freeCCtx(cctx);
3435 free(dictBuffer);
3436 free(samplesSizes);
3437 }
3438
3439 /* Decompression defense tests */
3440 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++);
3441 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
3442 if (!ZSTD_isError(r)) goto _output_error;
3443 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
3444 DISPLAYLEVEL(3, "OK \n");
3445
3446 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++);
3447 ((char*)(CNBuffer))[0] = 1;
3448 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
3449 if (!ZSTD_isError(r)) goto _output_error; }
3450 DISPLAYLEVEL(3, "OK \n");
3451
3452 /* content size verification test */
3453 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++);
3454 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3455 size_t const srcSize = 5000;
3456 size_t const wrongSrcSize = (srcSize + 1000);
3457 ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
3458 params.fParams.contentSizeFlag = 1;
3459 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
3460 { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
3461 if (!ZSTD_isError(result)) goto _output_error;
3462 if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
3463 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result));
3464 }
3465 ZSTD_freeCCtx(cctx);
3466 }
3467
3468 /* negative compression level test : ensure simple API and advanced API produce same result */
3469 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++);
3470 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3471 size_t const srcSize = CNBuffSize / 5;
3472 int const compressionLevel = -1;
3473
3474 assert(cctx != NULL);
3475 { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize,
3476 CNBuffer, srcSize, compressionLevel);
3477 if (ZSTD_isError(cSize_1pass)) goto _output_error;
3478
3479 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
3480 { size_t const compressionResult = ZSTD_compress2(cctx,
3481 compressedBuffer, compressedBufferSize,
3482 CNBuffer, srcSize);
3483 DISPLAYLEVEL(5, "simple=%u vs %u=advanced : ", (unsigned)cSize_1pass, (unsigned)compressionResult);
3484 if (ZSTD_isError(compressionResult)) goto _output_error;
3485 if (compressionResult != cSize_1pass) goto _output_error;
3486 } }
3487 ZSTD_freeCCtx(cctx);
3488 }
3489 DISPLAYLEVEL(3, "OK \n");
3490
3491 /* parameters order test */
3492 { size_t const inputSize = CNBuffSize / 2;
3493 U64 xxh64;
3494
3495 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3496 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
3497 assert(cctx != NULL);
3498 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3499 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3500 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3501 { size_t const compressedSize = ZSTD_compress2(cctx,
3502 compressedBuffer, ZSTD_compressBound(inputSize),
3503 CNBuffer, inputSize);
3504 CHECK_Z(compressedSize);
3505 cSize = compressedSize;
3506 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
3507 }
3508 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3509 ZSTD_freeCCtx(cctx);
3510 }
3511
3512 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
3513 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
3514 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3515 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3516 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3517 { size_t const result = ZSTD_compress2(cctx,
3518 compressedBuffer, ZSTD_compressBound(inputSize),
3519 CNBuffer, inputSize);
3520 CHECK_Z(result);
3521 if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
3522 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
3523 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
3524 }
3525 ZSTD_freeCCtx(cctx);
3526 }
3527 }
3528
3529 /* advanced parameters for decompression */
3530 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3531 assert(dctx != NULL);
3532
3533 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
3534 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
3535 CHECK_Z(bounds.error);
3536 }
3537 DISPLAYLEVEL(3, "OK \n");
3538
3539 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
3540 { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
3541 if (!ZSTD_isError(sr)) goto _output_error;
3542 }
3543 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
3544 if (!ZSTD_isError(bounds.error)) goto _output_error;
3545 }
3546 DISPLAYLEVEL(3, "OK \n");
3547
3548 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
3549 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
3550 if (!ZSTD_isError(sr)) goto _output_error;
3551 }
3552 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
3553 if (!ZSTD_isError(sr)) goto _output_error;
3554 }
3555 DISPLAYLEVEL(3, "OK \n");
3556
3557 ZSTD_freeDCtx(dctx);
3558 }
3559
3560
3561 /* custom formats tests */
3562 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3563 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3564 size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
3565 assert(dctx != NULL); assert(cctx != NULL);
3566
3567 /* basic block compression */
3568 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
3569 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3570 { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
3571 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
3572 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3573 if (result != 0) goto _output_error;
3574 if (in.pos != in.size) goto _output_error;
3575 cSize = out.pos;
3576 }
3577 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3578
3579 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++);
3580 { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3581 if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
3582 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult));
3583 }
3584
3585 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
3586 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3587 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3588 { ZSTD_FrameHeader zfh;
3589 size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
3590 if (zfhrt != 0) goto _output_error;
3591 }
3592 /* one shot */
3593 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3594 if (result != inputSize) goto _output_error;
3595 DISPLAYLEVEL(3, "one-shot OK, ");
3596 }
3597 /* streaming */
3598 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3599 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3600 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3601 if (result != 0) goto _output_error;
3602 if (in.pos != in.size) goto _output_error;
3603 if (out.pos != inputSize) goto _output_error;
3604 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3605 }
3606
3607 /* basic block compression */
3608 DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
3609 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3610 { ZSTD_inBuffer in = { CNBuffer, 0, 0 };
3611 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
3612 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3613 if (result != 0) goto _output_error;
3614 if (in.pos != in.size) goto _output_error;
3615 cSize = out.pos;
3616 }
3617 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
3618
3619 DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
3620 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3621 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3622 /* one shot */
3623 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3624 if (result != 0) goto _output_error;
3625 DISPLAYLEVEL(3, "one-shot OK, ");
3626 }
3627 /* streaming */
3628 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3629 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3630 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3631 if (result != 0) goto _output_error;
3632 if (in.pos != in.size) goto _output_error;
3633 if (out.pos != 0) goto _output_error;
3634 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3635 }
3636
3637 ZSTD_freeCCtx(cctx);
3638 ZSTD_freeDCtx(dctx);
3639 }
3640
3641 DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++);
3642 {
3643 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3644 /* Attempt to future proof this to new parameters. */
3645 int const maxParam = 2000;
3646 int param;
3647 if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error;
3648 for (param = 0; param < maxParam; ++param) {
3649 ZSTD_dParameter dParam = (ZSTD_dParameter)param;
3650 ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam);
3651 int value1;
3652 int value2;
3653 int check;
3654 if (ZSTD_isError(bounds.error))
3655 continue;
3656 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
3657 value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
3658 CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
3659 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3660 if (check != value2) goto _output_error;
3661 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
3662 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3663 if (check != value1) goto _output_error;
3664 }
3665 ZSTD_freeDCtx(dctx);
3666 }
3667 DISPLAYLEVEL(3, "OK \n");
3668
3669 /* block API tests */
3670 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3671 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3672 static const size_t dictSize = 65 KB;
3673 static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
3674 size_t cSize2;
3675 assert(cctx != NULL); assert(dctx != NULL);
3676
3677 /* basic block compression */
3678 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
3679 CHECK_Z( ZSTD_compressBegin(cctx, 5) );
3680 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3681 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
3682 DISPLAYLEVEL(3, "OK \n");
3683
3684 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
3685 CHECK_Z( ZSTD_decompressBegin(dctx) );
3686 { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3687 if (r != blockSize) goto _output_error; }
3688 DISPLAYLEVEL(3, "OK \n");
3689
3690 /* very long stream of block compression */
3691 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
3692 CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
3693 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3694 { U64 const toCompress = 5000000000ULL; /* > 4 GB */
3695 U64 compressed = 0;
3696 while (compressed < toCompress) {
3697 size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
3698 assert(blockCSize != 0);
3699 if (ZSTD_isError(blockCSize)) goto _output_error;
3700 compressed += blockCSize;
3701 } }
3702 DISPLAYLEVEL(3, "OK \n");
3703
3704 /* dictionary block compression */
3705 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
3706 CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
3707 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
3708 RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
3709 { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
3710 assert(r == 0); /* non-compressible block */ }
3711 memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */
3712 CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
3713 (char*)CNBuffer+dictSize+2*blockSize, blockSize));
3714 DISPLAYLEVEL(3, "OK \n");
3715
3716 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
3717 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
3718 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
3719 if (r != blockSize) {
3720 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3721 goto _output_error;
3722 } }
3723 memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize);
3724 ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
3725 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
3726 if (r != blockSize) {
3727 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3728 goto _output_error;
3729 } }
3730 assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */
3731 DISPLAYLEVEL(3, "OK \n");
3732
3733 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
3734 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
3735 if (cdict==NULL) goto _output_error;
3736 CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
3737 CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
3738 ZSTD_freeCDict(cdict);
3739 }
3740 DISPLAYLEVEL(3, "OK \n");
3741
3742 ZSTD_freeCCtx(cctx);
3743 ZSTD_freeDCtx(dctx);
3744 }
3745
3746 /* rle detection test: must compress better blocks with a single identical byte repeated */
3747 { size_t sampleSize = 0;
3748 size_t maxCompressedSize = 46; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
3749 DISPLAYLEVEL(3, "test%3i : RLE detection test : ", testNb++);
3750 memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 2);
3751 sampleSize += 256 KB - 2;
3752 memset((char*)CNBuffer+sampleSize, 'A', 100 KB);
3753 sampleSize += 100 KB;
3754 cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
3755 if (ZSTD_isError(cSize) || cSize > maxCompressedSize) {
3756 DISPLAYLEVEL(4, "error: cSize %u > %u expected ! \n", (unsigned)cSize, (unsigned)maxCompressedSize);
3757 goto _output_error;
3758 }
3759 { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
3760 if (regenSize!=sampleSize) goto _output_error; }
3761 DISPLAYLEVEL(3, "OK \n");
3762 }
3763
3764 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++);
3765 {
3766 size_t srcSize = 150 KB;
3767 BYTE* src = (BYTE*)CNBuffer;
3768 BYTE* decoded = (BYTE*)compressedBuffer;
3769
3770 ZSTD_CCtx* cctx = ZSTD_createCCtx();
3771 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3772 size_t seqsSize;
3773
3774 if (seqs == NULL) goto _output_error;
3775 assert(cctx != NULL);
3776 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
3777 /* Populate src with random data */
3778 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
3779
3780 /* Test with block delimiters roundtrip */
3781 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3782 CHECK_Z(seqsSize);
3783 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters);
3784 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3785
3786 /* Test no block delimiters roundtrip */
3787 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
3788 CHECK_Z(seqsSize);
3789 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters);
3790 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3791
3792 ZSTD_freeCCtx(cctx);
3793 free(seqs);
3794 }
3795 DISPLAYLEVEL(3, "OK \n");
3796
3797 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences too small output buffer : ", testNb++);
3798 {
3799 const size_t seqsCapacity = 10;
3800 const size_t srcSize = 150 KB;
3801 const BYTE* src = (BYTE*)CNBuffer;
3802
3803 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3804 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(seqsCapacity * sizeof(ZSTD_Sequence));
3805
3806 if (seqs == NULL) goto _output_error;
3807 if (cctx == NULL) goto _output_error;
3808 /* Populate src with random data */
3809 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
3810
3811 /* Test with block delimiters roundtrip */
3812 {
3813 size_t const seqsSize = ZSTD_generateSequences(cctx, seqs, seqsCapacity, src, srcSize);
3814 if (!ZSTD_isError(seqsSize)) goto _output_error;
3815 }
3816
3817 ZSTD_freeCCtx(cctx);
3818 free(seqs);
3819 }
3820 DISPLAYLEVEL(3, "OK \n");
3821
3822 DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
3823 {
3824 const size_t srcSize = 500 KB;
3825 const BYTE* const src = (BYTE*)CNBuffer;
3826 BYTE* const dst = (BYTE*)compressedBuffer;
3827 const size_t dstCapacity = ZSTD_compressBound(srcSize);
3828 const size_t decompressSize = srcSize;
3829 char* const decompressBuffer = (char*)malloc(decompressSize);
3830 size_t compressedSize;
3831
3832 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3833 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3834 size_t nbSeqs;
3835
3836 if (seqs == NULL) goto _output_error;
3837 assert(cctx != NULL);
3838
3839 /* Populate src with compressible random data */
3840 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
3841
3842 /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */
3843 nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3844 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3845 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
3846 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize);
3847 if (ZSTD_isError(compressedSize)) {
3848 DISPLAY("Error in sequence compression with block delims\n");
3849 goto _output_error;
3850 }
3851 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3852 if (ZSTD_isError(dSize)) {
3853 DISPLAY("Error in sequence compression roundtrip with block delims\n");
3854 goto _output_error;
3855 } }
3856 assert(!memcmp(decompressBuffer, src, srcSize));
3857
3858 /* Roundtrip Test with no block delimiters */
3859 { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs);
3860 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3861 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
3862 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize);
3863 }
3864 if (ZSTD_isError(compressedSize)) {
3865 DISPLAY("Error in sequence compression with no block delims\n");
3866 goto _output_error;
3867 }
3868 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3869 if (ZSTD_isError(dSize)) {
3870 DISPLAY("Error in sequence compression roundtrip with no block delims\n");
3871 goto _output_error;
3872 } }
3873 assert(!memcmp(decompressBuffer, src, srcSize));
3874
3875 ZSTD_freeCCtx(cctx);
3876 free(decompressBuffer);
3877 free(seqs);
3878 }
3879 DISPLAYLEVEL(3, "OK \n");
3880
3881 DISPLAYLEVEL(3, "test%3i : ZSTD_compressSequencesAndLiterals : ", testNb++);
3882 {
3883 const size_t srcSize = 497000;
3884 const BYTE* const src = (BYTE*)CNBuffer;
3885 BYTE* const dst = (BYTE*)compressedBuffer;
3886 const size_t dstCapacity = ZSTD_compressBound(srcSize);
3887 const size_t decompressSize = srcSize;
3888 char* const decompressBuffer = (char*)malloc(decompressSize);
3889 char* const litBuffer = (char*)malloc(decompressSize);
3890 size_t compressedSize;
3891
3892 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3893 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3894 size_t nbSeqs;
3895
3896 if (litBuffer == NULL) goto _output_error;
3897 if (decompressBuffer == NULL) goto _output_error;
3898 if (seqs == NULL) goto _output_error;
3899 assert(cctx != NULL);
3900
3901 /* Populate src with compressible random data */
3902 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
3903
3904 /* Roundtrip Test using the AndLiterals() variant */
3905 nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3906 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3907 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
3908 { size_t const litSize = FUZ_getLitSize(seqs, nbSeqs);
3909 FUZ_transferLiterals(litBuffer, decompressSize, CNBuffer, srcSize, seqs, nbSeqs);
3910
3911 /* not enough literals: must fail */
3912 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, decompressSize, srcSize);
3913 if (!ZSTD_isError(compressedSize)) {
3914 DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: not enough literals provided\n");
3915 goto _output_error;
3916 }
3917
3918 /* too many literals: must fail */
3919 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, decompressSize, srcSize);
3920 if (!ZSTD_isError(compressedSize)) {
3921 DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: too many literals provided\n");
3922 goto _output_error;
3923 }
3924
3925 /* srcSize too large: must fail */
3926 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize+1);
3927 if (!ZSTD_isError(compressedSize)) {
3928 DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too large\n");
3929 goto _output_error;
3930 }
3931
3932 /* srcSize too small: must fail */
3933 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize-1);
3934 if (!ZSTD_isError(compressedSize)) {
3935 DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too small\n");
3936 goto _output_error;
3937 }
3938
3939 /* correct amount of literals: should compress successfully */
3940 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize);
3941 if (ZSTD_isError(compressedSize)) {
3942 DISPLAY("Error in ZSTD_compressSequencesAndLiterals()\n");
3943 goto _output_error;
3944 }
3945 }
3946 { ZSTD_FrameHeader zfh;
3947 size_t const zfhStatus = ZSTD_getFrameHeader(&zfh, dst, compressedSize);
3948 if (zfhStatus != 0) {
3949 DISPLAY("Error reading frame header\n");
3950 goto _output_error;
3951 }
3952 if (zfh.frameContentSize != srcSize) {
3953 DISPLAY("Error: ZSTD_compressSequencesAndLiterals() did not report srcSize in the frame header\n");
3954 goto _output_error;
3955 }
3956 if (zfh.windowSize > srcSize) {
3957 DISPLAY("Error: ZSTD_compressSequencesAndLiterals() did not resized window size to smaller contentSize\n");
3958 goto _output_error;
3959 }
3960 }
3961 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3962 if (ZSTD_isError(dSize)) {
3963 DISPLAY("Error during decompression of frame produced by ZSTD_compressSequencesAndLiterals()\n");
3964 goto _output_error;
3965 }
3966 if (dSize != srcSize) {
3967 DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() has different size\n");
3968 goto _output_error;
3969 }
3970 if (memcmp(decompressBuffer, src, srcSize)) {
3971 DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() produces a different content (of same size)\n");
3972 goto _output_error;
3973 }
3974 }
3975
3976 ZSTD_freeCCtx(cctx);
3977 free(litBuffer);
3978 free(decompressBuffer);
3979 free(seqs);
3980 }
3981 DISPLAYLEVEL(3, "OK \n");
3982
3983 /* Multiple blocks of zeros test */
3984 #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
3985 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3986 memset(CNBuffer, 0, LONGZEROSLENGTH);
3987 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) );
3988 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100);
3989
3990 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3991 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) );
3992 if (r != LONGZEROSLENGTH) goto _output_error; }
3993 DISPLAYLEVEL(3, "OK \n");
3994
3995 /* All zeroes test (test bug #137) */
3996 #define ZEROESLENGTH 100
3997 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
3998 memset(CNBuffer, 0, ZEROESLENGTH);
3999 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
4000 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100);
4001
4002 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
4003 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
4004 if (r != ZEROESLENGTH) goto _output_error; }
4005 DISPLAYLEVEL(3, "OK \n");
4006
4007 /* nbSeq limit test */
4008 #define _3BYTESTESTLENGTH 131000
4009 #define NB3BYTESSEQLOG 9
4010 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
4011 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
4012 /* creates a buffer full of 3-bytes sequences */
4013 { BYTE _3BytesSeqs[NB3BYTESSEQ][3];
4014 U32 rSeed = 1;
4015
4016 /* create batch of 3-bytes sequences */
4017 { int i;
4018 for (i=0; i < NB3BYTESSEQ; i++) {
4019 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
4020 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
4021 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
4022 } }
4023
4024 /* randomly fills CNBuffer with prepared 3-bytes sequences */
4025 { int i;
4026 for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
4027 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
4028 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
4029 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
4030 ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
4031 } } }
4032 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++);
4033 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4034 size_t const maxNbSeq = _3BYTESTESTLENGTH / 3;
4035 size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
4036 size_t nbSeq = 1;
4037 while (nbSeq <= maxNbSeq) {
4038 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
4039 /* Check every sequence for the first 100, then skip more rapidly. */
4040 if (nbSeq < 100) {
4041 ++nbSeq;
4042 } else {
4043 nbSeq += (nbSeq >> 2);
4044 }
4045 }
4046 ZSTD_freeCCtx(cctx);
4047 }
4048 DISPLAYLEVEL(3, "OK \n");
4049
4050 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
4051 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
4052 CNBuffer, _3BYTESTESTLENGTH, 19) );
4053 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
4054
4055 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
4056 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
4057 if (r != _3BYTESTESTLENGTH) goto _output_error; }
4058 DISPLAYLEVEL(3, "OK \n");
4059
4060
4061 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++);
4062 RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
4063 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4064 size_t const bound = ZSTD_compressBound(CNBuffSize);
4065 size_t size = 1;
4066 while (size <= CNBuffSize) {
4067 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
4068 /* Check every size for the first 100, then skip more rapidly. */
4069 if (size < 100) {
4070 ++size;
4071 } else {
4072 size += (size >> 2);
4073 }
4074 }
4075 ZSTD_freeCCtx(cctx);
4076 }
4077 DISPLAYLEVEL(3, "OK \n");
4078
4079 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
4080 { /* Train a dictionary on low characters */
4081 size_t dictSize = 16 KB;
4082 void* const dictBuffer = malloc(dictSize);
4083 size_t const totalSampleSize = 1 MB;
4084 size_t const sampleUnitSize = 8 KB;
4085 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
4086 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
4087 if (!dictBuffer || !samplesSizes) goto _output_error;
4088 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
4089 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
4090 if (ZDICT_isError(dictSize)) goto _output_error;
4091 /* Reverse the characters to make the dictionary ill suited */
4092 { U32 u;
4093 for (u = 0; u < CNBuffSize; ++u) {
4094 ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
4095 } }
4096 { /* Compress the data */
4097 size_t const inputSize = 500;
4098 size_t const outputSize = ZSTD_compressBound(inputSize);
4099 void* const outputBuffer = malloc(outputSize);
4100 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4101 if (!outputBuffer || !cctx) goto _output_error;
4102 CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
4103 free(outputBuffer);
4104 ZSTD_freeCCtx(cctx);
4105 }
4106
4107 free(dictBuffer);
4108 free(samplesSizes);
4109 }
4110 DISPLAYLEVEL(3, "OK \n");
4111
4112
4113 /* frame operations on skippable frames */
4114 { const char skippableFrame[] = "\x52\x2a\x4d\x18\x05\x0\x0\0abcde";
4115 size_t const skippableFrameSize = sizeof(skippableFrame) - 1 /* remove the terminating /0 */;
4116
4117 DISPLAYLEVEL(3, "test%3i : ZSTD_findFrameCompressedSize on skippable frame : ", testNb++);
4118 CHECK(ZSTD_findFrameCompressedSize(skippableFrame, skippableFrameSize) == skippableFrameSize);
4119 DISPLAYLEVEL(3, "OK \n");
4120
4121 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize on skippable frame : ", testNb++);
4122 CHECK(ZSTD_getFrameContentSize(skippableFrame, skippableFrameSize) == 0);
4123 DISPLAYLEVEL(3, "OK \n");
4124
4125 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameHeader on skippable frame : ", testNb++);
4126 { ZSTD_FrameHeader zfh;
4127 size_t const s = ZSTD_getFrameHeader(&zfh, skippableFrame, skippableFrameSize);
4128 CHECK_Z(s);
4129 CHECK(s == 0); /* success */
4130 CHECK(zfh.frameType == ZSTD_skippableFrame);
4131 CHECK(zfh.headerSize == ZSTD_SKIPPABLEHEADERSIZE);
4132 CHECK(zfh.dictID == 2); /* magic variant */
4133 assert(skippableFrameSize >= ZSTD_SKIPPABLEHEADERSIZE);
4134 CHECK(zfh.frameContentSize == skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE);
4135 }
4136 DISPLAYLEVEL(3, "OK \n");
4137 }
4138
4139 /* error string tests */
4140 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
4141 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
4142 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
4143 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
4144 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
4145 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
4146 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
4147 DISPLAYLEVEL(3, "OK \n");
4148
4149 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
4150 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
4151 {
4152 size_t const size = MIN(128 KB, CNBuffSize);
4153 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4154 ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
4155 ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
4156 ZSTD_FrameHeader lgHeader;
4157 ZSTD_FrameHeader smHeader;
4158
4159 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
4160 CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
4161 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
4162 CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
4163
4164 if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
4165
4166 ZSTD_freeCDict(smCDict);
4167 ZSTD_freeCDict(lgCDict);
4168 ZSTD_freeCCtx(cctx);
4169 }
4170 DISPLAYLEVEL(3, "OK \n");
4171
4172 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++);
4173 {
4174 short norm[32];
4175 unsigned count[32];
4176 unsigned const tableLog = 5;
4177 size_t const nbSeq = 32;
4178 unsigned const maxSymbolValue = 31;
4179 size_t i;
4180
4181 for (i = 0; i < 32; ++i)
4182 count[i] = 1;
4183 /* Calling FSE_normalizeCount() on a uniform distribution should not
4184 * cause a division by zero.
4185 */
4186 FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1);
4187 }
4188 DISPLAYLEVEL(3, "OK \n");
4189
4190 DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++);
4191 {
4192 size_t const outBufSize = 9;
4193 short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18};
4194 unsigned const tableLog = 5;
4195 unsigned const maxSymbolValue = 10;
4196 BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE));
4197
4198 /* Ensure that this write doesn't write out of bounds, and that
4199 * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1.
4200 */
4201 FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog);
4202 free(outBuf);
4203 }
4204 DISPLAYLEVEL(3, "OK \n");
4205
4206 DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
4207 {
4208 U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
4209 U32 rand32 = FUZ_rand(&seed_copy);
4210 U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
4211 U32 lowbit_only_32 = 1;
4212 U64 lowbit_only_64 = 1;
4213 U32 highbit_only_32 = (U32)1 << 31;
4214 U64 highbit_only_64 = (U64)1 << 63;
4215 U32 i;
4216 if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
4217 if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
4218
4219 /* Test ZSTD_countTrailingZeros32 */
4220 CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
4221 CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u);
4222 CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32));
4223
4224 /* Test ZSTD_countLeadingZeros32 */
4225 CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u);
4226 CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u);
4227 CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32));
4228
4229 /* Test ZSTD_countTrailingZeros64 */
4230 CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u);
4231 CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u);
4232
4233 /* Test ZSTD_countLeadingZeros64 */
4234 CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u);
4235 CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u);
4236
4237 /* Test ZSTD_highbit32 */
4238 CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u);
4239 CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u);
4240
4241 /* Test ZSTD_NbCommonBytes */
4242 if (MEM_isLittleEndian()) {
4243 if (MEM_64bits()) {
4244 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
4245 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
4246 } else {
4247 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
4248 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
4249 }
4250 } else {
4251 if (MEM_64bits()) {
4252 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u);
4253 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u);
4254 } else {
4255 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u);
4256 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u);
4257 }
4258 }
4259
4260 /* Test MEM_ intrinsics */
4261 CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32));
4262 CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64));
4263
4264 /* Test fallbacks vs intrinsics on a range of small integers */
4265 for (i=1; i <= 1000; i++) {
4266 CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i));
4267 CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i));
4268 CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i));
4269 CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i));
4270 }
4271 }
4272 DISPLAYLEVEL(3, "OK \n");
4273
4274 #ifdef ZSTD_MULTITHREAD
4275 DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++);
4276 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4277 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4278 size_t const dstSize = ZSTD_compressBound(srcSize);
4279 void* const src = CNBuffer;
4280 void* const dst = compressedBuffer;
4281 void* dict = (void*)malloc(srcSize);
4282
4283 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4284 RDG_genBuffer(dict, srcSize, compressibility, 0., seed);
4285
4286 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4287 memset(dict, 0, sizeof(U32));
4288
4289 /* something more than 1 */
4290 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4291 /* lie and claim this is a full dict */
4292 CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict));
4293
4294 { ZSTD_outBuffer out = {dst, dstSize, 0};
4295 ZSTD_inBuffer in = {src, srcSize, 0};
4296 /* should fail because its not a full dict like we said it was */
4297 assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)));
4298 }
4299
4300 ZSTD_freeCCtx(cctx);
4301 free(dict);
4302 }
4303 DISPLAYLEVEL(3, "OK \n");
4304
4305 DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++);
4306 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4307 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4308 size_t const dictSize = 10;
4309 size_t const dstSize = ZSTD_compressBound(srcSize);
4310 void* const src = CNBuffer;
4311 void* const dst = compressedBuffer;
4312 void* dict = (void*)malloc(dictSize);
4313
4314 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4315 RDG_genBuffer(dict, dictSize, compressibility, 0., seed);
4316
4317 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4318 memset(dict, 0, sizeof(U32));
4319
4320 /* Enable MT, LDM, and use refPrefix() for a small dict */
4321 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4322 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4323 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
4324
4325 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
4326
4327 ZSTD_freeCCtx(cctx);
4328 free(dict);
4329 }
4330 DISPLAYLEVEL(3, "OK \n");
4331
4332 DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++);
4333 {
4334 ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0);
4335 ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0);
4336 ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400);
4337 ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000);
4338 ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000);
4339
4340 assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict)));
4341 assert(!memcmp(&medium, &mediumDict, sizeof(medium)));
4342 assert(!memcmp(&large, &largeDict, sizeof(large)));
4343 }
4344 DISPLAYLEVEL(3, "OK \n");
4345
4346 DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++);
4347 {
4348 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0);
4349 ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400);
4350 ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400);
4351
4352 assert(smallSrcAndDict.windowLog == 10);
4353 assert(!memcmp(&cParams, &smallDict, sizeof(cParams)));
4354 }
4355 DISPLAYLEVEL(3, "OK \n");
4356
4357 DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
4358 {
4359 int level = 1;
4360 size_t prevSize = 0;
4361 for (; level < ZSTD_maxCLevel(); ++level) {
4362 size_t const currSize = ZSTD_estimateCCtxSize(level);
4363 if (prevSize > currSize) {
4364 DISPLAYLEVEL(3, "Error! previous cctx size: %u at level: %d is larger than current cctx size: %u at level: %d",
4365 (unsigned)prevSize, level-1, (unsigned)currSize, level);
4366 goto _output_error;
4367 }
4368 prevSize = currSize;
4369 }
4370 }
4371 DISPLAYLEVEL(3, "OK \n");
4372
4373 DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
4374 {
4375 size_t const kSizeIncrement = 2 KB;
4376 int level = -3;
4377
4378 for (; level <= ZSTD_maxCLevel(); ++level) {
4379 size_t dictSize = 0;
4380 for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
4381 size_t srcSize = 2 KB;
4382 for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
4383 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
4384 size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
4385 size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
4386 if (cctxSizeUsingLevel < cctxSizeUsingCParams
4387 || ZSTD_isError(cctxSizeUsingCParams)
4388 || ZSTD_isError(cctxSizeUsingLevel)) {
4389 DISPLAYLEVEL(3, "error! l: %d dict: %u srcSize: %u cctx size cpar: %u, cctx size level: %u\n",
4390 level, (unsigned)dictSize, (unsigned)srcSize, (unsigned)cctxSizeUsingCParams, (unsigned)cctxSizeUsingLevel);
4391 goto _output_error;
4392 } } } } }
4393 DISPLAYLEVEL(3, "OK \n");
4394
4395 DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++)
4396 {
4397 int const threadPoolTestResult = threadPoolTests();
4398 if (threadPoolTestResult) {
4399 goto _output_error;
4400 }
4401 }
4402 DISPLAYLEVEL(3, "thread pool tests OK \n");
4403
4404 #endif /* ZSTD_MULTITHREAD */
4405
4406 _end:
4407 free(CNBuffer);
4408 free(compressedBuffer);
4409 free(decodedBuffer);
4410 return testResult;
4411
4412 _output_error:
4413 testResult = 1;
4414 DISPLAY("Error detected in Unit tests ! \n");
4415 goto _end;
4416 }
4417
longUnitTests(U32 const seed,double compressibility)4418 static int longUnitTests(U32 const seed, double compressibility)
4419 {
4420 size_t const CNBuffSize = 5 MB;
4421 void* const CNBuffer = malloc(CNBuffSize);
4422 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
4423 void* const compressedBuffer = malloc(compressedBufferSize);
4424 void* const decodedBuffer = malloc(CNBuffSize);
4425 int testResult = 0;
4426 unsigned testNb=0;
4427 size_t cSize;
4428
4429 /* Create compressible noise */
4430 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
4431 DISPLAY("Not enough memory, aborting\n");
4432 testResult = 1;
4433 goto _end;
4434 }
4435 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
4436
4437 /* note : this test is rather long, it would be great to find a way to speed up its execution */
4438 DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++);
4439 { int cLevel;
4440 size_t approxIndex = 0;
4441 size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */
4442
4443 /* Provision enough space in a static context so that we can do all
4444 * this without ever reallocating, which would reset the indices. */
4445 size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22);
4446 void* const staticCCtxBuffer = malloc(staticCCtxSize);
4447 ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
4448
4449 /* bump the indices so the following compressions happen at high
4450 * indices. */
4451 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4452 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4453 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4454 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4455 while (approxIndex <= (maxIndex / 4) * 3) {
4456 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4457 approxIndex += in.pos;
4458 CHECK_Z(in.pos == in.size);
4459 in.pos = 0;
4460 out.pos = 0;
4461 }
4462 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4463 }
4464
4465 /* spew a bunch of stuff into the table area */
4466 for (cLevel = 1; cLevel <= 22; cLevel++) {
4467 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4468 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4469 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4470 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4471 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4472 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4473 approxIndex += in.pos;
4474 }
4475
4476 /* now crank the indices so we overflow */
4477 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4478 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4479 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4480 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4481 while (approxIndex <= maxIndex) {
4482 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4483 approxIndex += in.pos;
4484 CHECK_Z(in.pos == in.size);
4485 in.pos = 0;
4486 out.pos = 0;
4487 }
4488 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4489 }
4490
4491 /* do a bunch of compressions again in low indices and ensure we don't
4492 * hit untracked invalid indices */
4493 for (cLevel = 1; cLevel <= 22; cLevel++) {
4494 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4495 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4496 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4497 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4498 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4499 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4500 approxIndex += in.pos;
4501 }
4502
4503 free(staticCCtxBuffer);
4504 }
4505 DISPLAYLEVEL(3, "OK \n");
4506
4507 DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++);
4508 { size_t cSizeLdm;
4509 size_t cSizeNoLdm;
4510 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4511
4512 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed);
4513
4514 /* Enable checksum to verify round trip. */
4515 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4516 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4517 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4518
4519 /* Round trip once with ldm. */
4520 cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4521 CHECK_Z(cSizeLdm);
4522 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm));
4523
4524 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4525 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4526 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
4527 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4528
4529 /* Round trip once without ldm. */
4530 cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4531 CHECK_Z(cSizeNoLdm);
4532 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm));
4533
4534 if (cSizeLdm > cSizeNoLdm) {
4535 DISPLAY("Using long mode should not cause regressions for btopt+\n");
4536 testResult = 1;
4537 goto _end;
4538 }
4539
4540 ZSTD_freeCCtx(cctx);
4541 }
4542 DISPLAYLEVEL(3, "OK \n");
4543
4544 DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++);
4545 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4546 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4547 size_t dictSize = CNBuffSize;
4548 void* dict = (void*)malloc(dictSize);
4549 ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams();
4550 ZSTD_dictAttachPref_e const attachPrefs[] = {
4551 ZSTD_dictDefaultAttach,
4552 ZSTD_dictForceAttach,
4553 ZSTD_dictForceCopy,
4554 ZSTD_dictForceLoad,
4555 ZSTD_dictDefaultAttach,
4556 ZSTD_dictForceAttach,
4557 ZSTD_dictForceCopy,
4558 ZSTD_dictForceLoad
4559 };
4560 int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1};
4561 int cLevel;
4562 int i;
4563
4564 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
4565 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
4566
4567 CHECK_Z(cctx_params != NULL);
4568
4569 for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
4570 DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
4571 for (cLevel = 4; cLevel < 13; cLevel++) {
4572 for (i = 0; i < 8; ++i) {
4573 ZSTD_dictAttachPref_e const attachPref = attachPrefs[i];
4574 int const enableDDS = enableDedicatedDictSearch[i];
4575 ZSTD_CDict* cdict;
4576
4577 DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i);
4578
4579 ZSTD_CCtxParams_init(cctx_params, cLevel);
4580 CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS));
4581
4582 cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem);
4583 CHECK(cdict != NULL);
4584
4585 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
4586 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref));
4587
4588 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4589 CHECK_Z(cSize);
4590 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
4591
4592 DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize);
4593
4594 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
4595 ZSTD_freeCDict(cdict);
4596 } } }
4597
4598 ZSTD_freeCCtx(cctx);
4599 ZSTD_freeDCtx(dctx);
4600 ZSTD_freeCCtxParams(cctx_params);
4601 free(dict);
4602 }
4603 DISPLAYLEVEL(3, "OK \n");
4604
4605 _end:
4606 free(CNBuffer);
4607 free(compressedBuffer);
4608 free(decodedBuffer);
4609 return testResult;
4610 }
4611
4612
findDiff(const void * buf1,const void * buf2,size_t max)4613 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
4614 {
4615 const BYTE* b1 = (const BYTE*)buf1;
4616 const BYTE* b2 = (const BYTE*)buf2;
4617 size_t u;
4618 for (u=0; u<max; u++) {
4619 if (b1[u] != b2[u]) break;
4620 }
4621 return u;
4622 }
4623
4624
FUZ_makeParams(ZSTD_compressionParameters cParams,ZSTD_frameParameters fParams)4625 static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
4626 {
4627 ZSTD_parameters params;
4628 params.cParams = cParams;
4629 params.fParams = fParams;
4630 return params;
4631 }
4632
FUZ_rLogLength(U32 * seed,U32 logLength)4633 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
4634 {
4635 size_t const lengthMask = ((size_t)1 << logLength) - 1;
4636 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
4637 }
4638
FUZ_randomLength(U32 * seed,U32 maxLog)4639 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
4640 {
4641 U32 const logLength = FUZ_rand(seed) % maxLog;
4642 return FUZ_rLogLength(seed, logLength);
4643 }
4644
4645 #undef CHECK
4646 #define CHECK(cond, ...) { \
4647 if (cond) { \
4648 DISPLAY("Error => "); \
4649 DISPLAY(__VA_ARGS__); \
4650 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4651 goto _output_error; \
4652 } }
4653
4654 #undef CHECK_Z
4655 #define CHECK_Z(f) { \
4656 size_t const err = f; \
4657 if (ZSTD_isError(err)) { \
4658 DISPLAY("Error => %s : %s ", \
4659 #f, ZSTD_getErrorName(err)); \
4660 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4661 goto _output_error; \
4662 } }
4663
4664
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,U32 const maxDurationS,double compressibility,int bigTests)4665 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
4666 {
4667 static const U32 maxSrcLog = 23;
4668 static const U32 maxSampleLog = 22;
4669 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
4670 size_t const dstBufferSize = (size_t)1<<maxSampleLog;
4671 size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
4672 BYTE* cNoiseBuffer[5];
4673 BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
4674 BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
4675 BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
4676 ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
4677 ZSTD_CCtx* const ctx = ZSTD_createCCtx();
4678 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4679 U32 result = 0;
4680 unsigned testNb = 0;
4681 U32 coreSeed = seed;
4682 UTIL_time_t const startClock = UTIL_getTime();
4683 U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
4684 int const cLevelLimiter = bigTests ? 3 : 2;
4685
4686 /* allocation */
4687 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
4688 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
4689 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
4690 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
4691 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
4692 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
4693 || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
4694 "Not enough memory, fuzzer tests cancelled");
4695
4696 /* Create initial samples */
4697 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
4698 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
4699 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
4700 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
4701 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
4702
4703 /* catch up testNb */
4704 for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
4705
4706 /* main test loop */
4707 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
4708 BYTE* srcBuffer; /* jumping pointer */
4709 U32 lseed;
4710 size_t sampleSize, maxTestSize, totalTestSize;
4711 size_t cSize, totalCSize, totalGenSize;
4712 U64 crcOrig;
4713 BYTE* sampleBuffer;
4714 const BYTE* dict;
4715 size_t dictSize;
4716
4717 /* notification */
4718 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
4719 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
4720
4721 FUZ_rand(&coreSeed);
4722 { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
4723
4724 /* srcBuffer selection [0-4] */
4725 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
4726 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
4727 else {
4728 buffNb >>= 3;
4729 if (buffNb & 7) {
4730 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
4731 buffNb = tnb[buffNb >> 3];
4732 } else {
4733 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
4734 buffNb = tnb[buffNb >> 3];
4735 } }
4736 srcBuffer = cNoiseBuffer[buffNb];
4737 }
4738
4739 /* select src segment */
4740 sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
4741
4742 /* create sample buffer (to catch read error with valgrind & sanitizers) */
4743 sampleBuffer = (BYTE*)malloc(sampleSize);
4744 CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
4745 { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
4746 memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
4747 crcOrig = XXH64(sampleBuffer, sampleSize, 0);
4748
4749 /* compression tests */
4750 { int const cLevelPositive = (int)
4751 ( FUZ_rand(&lseed) %
4752 ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) )
4753 + 1;
4754 int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ?
4755 - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */
4756 cLevelPositive;
4757 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel);
4758 cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
4759 CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
4760
4761 /* compression failure test : too small dest buffer */
4762 assert(cSize > 3);
4763 { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;
4764 const size_t tooSmallSize = cSize - missing;
4765 const unsigned endMark = 0x4DC2B1A9;
4766 memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark));
4767 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
4768 testNb, (unsigned)tooSmallSize, (unsigned)missing);
4769 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
4770 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
4771 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck));
4772 CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); }
4773 } }
4774
4775 /* frame header decompression test */
4776 { ZSTD_FrameHeader zfh;
4777 CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
4778 CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
4779 }
4780
4781 /* Decompressed size test */
4782 { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
4783 CHECK(rSize != sampleSize, "decompressed size incorrect");
4784 }
4785
4786 /* successful decompression test */
4787 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb);
4788 { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
4789 size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
4790 CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize);
4791 { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
4792 CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize);
4793 } }
4794
4795 free(sampleBuffer); /* no longer useful after this point */
4796
4797 /* truncated src decompression test */
4798 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb);
4799 { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4800 size_t const tooSmallSize = cSize - missing;
4801 void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
4802 CHECK(cBufferTooSmall == NULL, "not enough memory !");
4803 memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
4804 { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
4805 CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
4806 free(cBufferTooSmall);
4807 }
4808
4809 /* too small dst decompression test */
4810 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb);
4811 if (sampleSize > 3) {
4812 size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4813 size_t const tooSmallSize = sampleSize - missing;
4814 static const BYTE token = 0xA9;
4815 dstBuffer[tooSmallSize] = token;
4816 { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
4817 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
4818 CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
4819 }
4820
4821 /* noisy src decompression test */
4822 if (cSize > 6) {
4823 /* insert noise into src */
4824 { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
4825 size_t pos = 4; /* preserve magic number (too easy to detect) */
4826 for (;;) {
4827 /* keep some original src */
4828 { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
4829 size_t const mask = (1<<nbBits) - 1;
4830 size_t const skipLength = FUZ_rand(&lseed) & mask;
4831 pos += skipLength;
4832 }
4833 if (pos >= cSize) break;
4834 /* add noise */
4835 { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
4836 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
4837 size_t const mask = (1<<nbBits) - 1;
4838 size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
4839 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
4840 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
4841 memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
4842 pos += noiseLength;
4843 } } }
4844
4845 /* decompress noisy source */
4846 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb);
4847 { U32 const endMark = 0xA9B1C3D6;
4848 memcpy(dstBuffer+sampleSize, &endMark, 4);
4849 { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
4850 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
4851 CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
4852 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize);
4853 }
4854 { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
4855 CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
4856 } } } /* noisy src decompression test */
4857
4858 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
4859 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
4860 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
4861 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
4862 int const cLevel = (int)(FUZ_rand(&lseed) %
4863 ((U32)ZSTD_maxCLevel() -
4864 (MAX(testLog, dictLog) / (U32)cLevelLimiter))) +
4865 1;
4866 maxTestSize = FUZ_rLogLength(&lseed, testLog);
4867 if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
4868
4869 dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
4870 dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
4871
4872 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
4873 testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize);
4874
4875 if (FUZ_rand(&lseed) & 0xF) {
4876 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
4877 } else {
4878 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
4879 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
4880 !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
4881 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
4882 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
4883 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
4884 }
4885 CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
4886 }
4887
4888 { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
4889 U32 n;
4890 XXH64_state_t xxhState;
4891 XXH64_reset(&xxhState, 0);
4892 for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
4893 size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
4894 size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
4895
4896 if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
4897 if (totalTestSize+segmentSize > maxTestSize) break;
4898
4899 { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
4900 CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
4901 cSize += compressResult;
4902 }
4903 XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
4904 memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
4905 totalTestSize += segmentSize;
4906 }
4907
4908 { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
4909 CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
4910 cSize += flushResult;
4911 }
4912 crcOrig = XXH64_digest(&xxhState);
4913 }
4914
4915 /* streaming decompression test */
4916 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
4917 /* ensure memory requirement is good enough (should always be true) */
4918 { ZSTD_FrameHeader zfh;
4919 CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
4920 "ZSTD_getFrameHeader(): error retrieving frame information");
4921 { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
4922 CHECK_Z(roundBuffSize);
4923 CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN),
4924 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)",
4925 (unsigned)roundBuffSize, (unsigned)totalTestSize );
4926 } }
4927 if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
4928 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
4929 totalCSize = 0;
4930 totalGenSize = 0;
4931 while (totalCSize < cSize) {
4932 size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
4933 size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
4934 CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize));
4935 totalGenSize += genSize;
4936 totalCSize += inSize;
4937 }
4938 CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
4939 CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
4940 CHECK (totalCSize != cSize, "compressed data should be fully read")
4941 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
4942 CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
4943 (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize);
4944 }
4945 } /* for ( ; (testNb <= nbTests) */
4946 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
4947
4948 _cleanup:
4949 ZSTD_freeCCtx(refCtx);
4950 ZSTD_freeCCtx(ctx);
4951 ZSTD_freeDCtx(dctx);
4952 free(cNoiseBuffer[0]);
4953 free(cNoiseBuffer[1]);
4954 free(cNoiseBuffer[2]);
4955 free(cNoiseBuffer[3]);
4956 free(cNoiseBuffer[4]);
4957 free(cBuffer);
4958 free(dstBuffer);
4959 free(mirrorBuffer);
4960 return (int)result;
4961
4962 _output_error:
4963 result = 1;
4964 goto _cleanup;
4965 }
4966
4967
4968 /*_*******************************************************
4969 * Command line
4970 *********************************************************/
FUZ_usage(const char * programName)4971 static int FUZ_usage(const char* programName)
4972 {
4973 DISPLAY( "Usage :\n");
4974 DISPLAY( " %s [args]\n", programName);
4975 DISPLAY( "\n");
4976 DISPLAY( "Arguments :\n");
4977 DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault);
4978 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
4979 DISPLAY( " -s# : Select seed (default:prompt user)\n");
4980 DISPLAY( " -t# : Select starting test number (default:0)\n");
4981 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default);
4982 DISPLAY( " -v : verbose\n");
4983 DISPLAY( " -p : pause at the end\n");
4984 DISPLAY( " -h : display help and exit\n");
4985 return 0;
4986 }
4987
4988 /*! readU32FromChar() :
4989 @return : unsigned integer value read from input in `char` format
4990 allows and interprets K, KB, KiB, M, MB and MiB suffix.
4991 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
4992 Note : function result can overflow if digit string > MAX_UINT */
readU32FromChar(const char ** stringPtr)4993 static unsigned readU32FromChar(const char** stringPtr)
4994 {
4995 unsigned result = 0;
4996 while ((**stringPtr >='0') && (**stringPtr <='9'))
4997 result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ;
4998 if ((**stringPtr=='K') || (**stringPtr=='M')) {
4999 result <<= 10;
5000 if (**stringPtr=='M') result <<= 10;
5001 (*stringPtr)++ ;
5002 if (**stringPtr=='i') (*stringPtr)++;
5003 if (**stringPtr=='B') (*stringPtr)++;
5004 }
5005 return result;
5006 }
5007
5008 /** longCommandWArg() :
5009 * check if *stringPtr is the same as longCommand.
5010 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
5011 * @return 0 and doesn't modify *stringPtr otherwise.
5012 */
longCommandWArg(const char ** stringPtr,const char * longCommand)5013 static int longCommandWArg(const char** stringPtr, const char* longCommand)
5014 {
5015 size_t const comSize = strlen(longCommand);
5016 int const result = !strncmp(*stringPtr, longCommand, comSize);
5017 if (result) *stringPtr += comSize;
5018 return result;
5019 }
5020
main(int argc,const char ** argv)5021 int main(int argc, const char** argv)
5022 {
5023 U32 seed = 0;
5024 int seedset = 0;
5025 int argNb;
5026 int nbTests = nbTestsDefault;
5027 int testNb = 0;
5028 int proba = FUZ_compressibility_default;
5029 double probfloat;
5030 int result = 0;
5031 U32 mainPause = 0;
5032 U32 maxDuration = 0;
5033 int bigTests = 1;
5034 int longTests = 0;
5035 U32 memTestsOnly = 0;
5036 const char* const programName = argv[0];
5037
5038 /* Check command line */
5039 for (argNb=1; argNb<argc; argNb++) {
5040 const char* argument = argv[argNb];
5041 if(!argument) continue; /* Protection if argument empty */
5042
5043 /* Handle commands. Aggregated commands are allowed */
5044 if (argument[0]=='-') {
5045
5046 if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; }
5047
5048 if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
5049 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
5050 if (!strcmp(argument, "--long-tests")) { longTests=1; continue; }
5051 if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; }
5052
5053 argument++;
5054 while (*argument!=0) {
5055 switch(*argument)
5056 {
5057 case 'h':
5058 return FUZ_usage(programName);
5059
5060 case 'v':
5061 argument++;
5062 g_displayLevel++;
5063 break;
5064
5065 case 'q':
5066 argument++;
5067 g_displayLevel--;
5068 break;
5069
5070 case 'p': /* pause at the end */
5071 argument++;
5072 mainPause = 1;
5073 break;
5074
5075 case 'i':
5076 argument++; maxDuration = 0;
5077 nbTests = (int)readU32FromChar(&argument);
5078 break;
5079
5080 case 'T':
5081 argument++;
5082 nbTests = 0;
5083 maxDuration = readU32FromChar(&argument);
5084 if (*argument=='s') argument++; /* seconds */
5085 if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
5086 if (*argument=='n') argument++;
5087 break;
5088
5089 case 's':
5090 argument++;
5091 seedset = 1;
5092 seed = readU32FromChar(&argument);
5093 break;
5094
5095 case 't':
5096 argument++;
5097 testNb = (int)readU32FromChar(&argument);
5098 break;
5099
5100 case 'P': /* compressibility % */
5101 argument++;
5102 proba = (int)readU32FromChar(&argument);
5103 if (proba>100) proba = 100;
5104 break;
5105
5106 default:
5107 return (FUZ_usage(programName), 1);
5108 } } } } /* for (argNb=1; argNb<argc; argNb++) */
5109
5110 /* Get Seed */
5111 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
5112
5113 if (!seedset) {
5114 time_t const t = time(NULL);
5115 U32 const h = XXH32(&t, sizeof(t), 1);
5116 seed = h % 10000;
5117 }
5118
5119 DISPLAY("Seed = %u\n", (unsigned)seed);
5120 if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba);
5121
5122 probfloat = ((double)proba) / 100;
5123
5124 if (memTestsOnly) {
5125 g_displayLevel = MAX(3, g_displayLevel);
5126 return FUZ_mallocTests(seed, probfloat, memTestsOnly);
5127 }
5128
5129 if (nbTests < testNb) nbTests = testNb;
5130
5131 if (testNb==0) {
5132 result = basicUnitTests(0, probfloat); /* constant seed for predictability */
5133
5134 if (!result && longTests) {
5135 result = longUnitTests(0, probfloat);
5136 }
5137 }
5138 if (!result)
5139 result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests);
5140 if (mainPause) {
5141 int unused;
5142 DISPLAY("Press Enter \n");
5143 unused = getchar();
5144 (void)unused;
5145 }
5146 return result;
5147 }
5148