1 /*
2 frameTest - test tool for lz4frame
3 Copyright (C) Yann Collet 2014-2020
4
5 GPL v2 License
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License along
18 with this program; if not, write to the Free Software Foundation, Inc.,
19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21 You can contact the author at :
22 - LZ4 homepage : http://www.lz4.org
23 - LZ4 source repository : https://github.com/lz4/lz4
24 */
25
26 /*-************************************
27 * Compiler specific
28 **************************************/
29 #ifdef _MSC_VER /* Visual Studio */
30 # pragma warning(disable : 26451) /* disable: Arithmetic overflow */
31 #endif
32
33
34 /*-************************************
35 * Includes
36 **************************************/
37 #include "util.h" /* U32 */
38 #include <stdlib.h> /* malloc, free */
39 #include <stdio.h> /* fprintf */
40 #include <string.h> /* strcmp */
41 #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
42 #include <assert.h>
43 #include "lz4frame.h" /* included multiple times to test correctness/safety */
44 #include "lz4frame.h"
45 #define LZ4F_STATIC_LINKING_ONLY
46 #include "lz4frame.h"
47 #include "lz4frame.h"
48 #define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */
49 #include "lz4.h" /* LZ4_VERSION_STRING */
50 #define XXH_STATIC_LINKING_ONLY
51 #include "xxhash.h" /* XXH64 */
52
53
54 /* unoptimized version; solves endianness & alignment issues */
FUZ_writeLE32(void * dstVoidPtr,U32 value32)55 static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
56 {
57 BYTE* const dstPtr = (BYTE*)dstVoidPtr;
58 dstPtr[0] = (BYTE) value32;
59 dstPtr[1] = (BYTE)(value32 >> 8);
60 dstPtr[2] = (BYTE)(value32 >> 16);
61 dstPtr[3] = (BYTE)(value32 >> 24);
62 }
63
64
65 /*-************************************
66 * Constants
67 **************************************/
68 #define KB *(1U<<10)
69 #define MB *(1U<<20)
70 #define GB *(1U<<30)
71
72 static const U32 nbTestsDefault = 256 KB;
73 #define FUZ_COMPRESSIBILITY_DEFAULT 50
74 static const U32 prime1 = 2654435761U;
75 static const U32 prime2 = 2246822519U;
76
77
78 /*-************************************
79 * Macros
80 **************************************/
81 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
82 #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
83 #define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
84 if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \
85 { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \
86 if (displayLevel>=4) fflush(stdout); } }
87 static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
88 static clock_t g_clockTime = 0;
89
90
91 /*-***************************************
92 * Local Parameters
93 *****************************************/
94 static U32 no_prompt = 0;
95 static U32 displayLevel = 2;
96 static U32 use_pause = 0;
97
98
99 /*-*******************************************************
100 * Fuzzer functions
101 *********************************************************/
102 #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
103 #define MAX(a,b) ( (a) > (b) ? (a) : (b) )
104
105 typedef struct {
106 int nbAllocs;
107 } Test_alloc_state;
108 static Test_alloc_state g_testAllocState = { 0 };
109
dummy_malloc(void * state,size_t s)110 static void* dummy_malloc(void* state, size_t s)
111 {
112 Test_alloc_state* const t = (Test_alloc_state*)state;
113 void* const p = malloc(s);
114 if (p==NULL) return NULL;
115 assert(t != NULL);
116 t->nbAllocs += 1;
117 DISPLAYLEVEL(6, "Allocating %zu bytes at address %p \n", s, p);
118 DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs);
119 return p;
120 }
121
dummy_calloc(void * state,size_t s)122 static void* dummy_calloc(void* state, size_t s)
123 {
124 Test_alloc_state* const t = (Test_alloc_state*)state;
125 void* const p = calloc(1, s);
126 if (p==NULL) return NULL;
127 assert(t != NULL);
128 t->nbAllocs += 1;
129 DISPLAYLEVEL(6, "Allocating and zeroing %zu bytes at address %p \n", s, p);
130 DISPLAYLEVEL(5, "nb allocated memory segments : %i \n", t->nbAllocs);
131 return p;
132 }
133
dummy_free(void * state,void * p)134 static void dummy_free(void* state, void* p)
135 {
136 Test_alloc_state* const t = (Test_alloc_state*)state;
137 if (p==NULL) {
138 DISPLAYLEVEL(5, "free() on NULL \n");
139 return;
140 }
141 DISPLAYLEVEL(6, "freeing memory at address %p \n", p);
142 free(p);
143 assert(t != NULL);
144 t->nbAllocs -= 1;
145 DISPLAYLEVEL(5, "nb of allocated memory segments after this free : %i \n", t->nbAllocs);
146 assert(t->nbAllocs >= 0);
147 }
148
149 static const LZ4F_CustomMem lz4f_cmem_test = {
150 dummy_malloc,
151 dummy_calloc,
152 dummy_free,
153 &g_testAllocState
154 };
155
156
FUZ_GetClockSpan(clock_t clockStart)157 static clock_t FUZ_GetClockSpan(clock_t clockStart)
158 {
159 return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */
160 }
161
162 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(unsigned int * src)163 unsigned int FUZ_rand(unsigned int* src)
164 {
165 U32 rand32 = *src;
166 rand32 *= prime1;
167 rand32 += prime2;
168 rand32 = FUZ_rotl32(rand32, 13);
169 *src = rand32;
170 return rand32 >> 5;
171 }
172
173 #define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF)
174 #define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
FUZ_fillCompressibleNoiseBuffer(void * buffer,size_t bufferSize,double proba,U32 * seed)175 static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed)
176 {
177 BYTE* BBuffer = (BYTE*)buffer;
178 size_t pos = 0;
179 U32 P32 = (U32)(32768 * proba);
180
181 /* First Byte */
182 BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
183
184 while (pos < bufferSize) {
185 /* Select : Literal (noise) or copy (within 64K) */
186 if (FUZ_RAND15BITS < P32) {
187 /* Copy (within 64K) */
188 size_t const lengthRand = FUZ_RANDLENGTH + 4;
189 size_t const length = MIN(lengthRand, bufferSize - pos);
190 size_t const end = pos + length;
191 size_t const offsetRand = FUZ_RAND15BITS + 1;
192 size_t const offset = MIN(offsetRand, pos);
193 size_t match = pos - offset;
194 while (pos < end) BBuffer[pos++] = BBuffer[match++];
195 } else {
196 /* Literal (noise) */
197 size_t const lengthRand = FUZ_RANDLENGTH + 4;
198 size_t const length = MIN(lengthRand, bufferSize - pos);
199 size_t const end = pos + length;
200 while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
201 } }
202 }
203
204
FUZ_highbit(U32 v32)205 static unsigned FUZ_highbit(U32 v32)
206 {
207 unsigned nbBits = 0;
208 if (v32==0) return 0;
209 while (v32) {v32 >>= 1; nbBits ++;}
210 return nbBits;
211 }
212
213
214 /*-*******************************************************
215 * Tests
216 *********************************************************/
217 #define CHECK_V(v,f) v = f; if (LZ4F_isError(v)) { fprintf(stderr, "%s \n", LZ4F_getErrorName(v)); goto _output_error; }
218 #define CHECK(f) { LZ4F_errorCode_t const CHECK_V(err_ , f); }
219
basicTests(U32 seed,double compressibility)220 int basicTests(U32 seed, double compressibility)
221 {
222 #define COMPRESSIBLE_NOISE_LENGTH (2 MB)
223 void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
224 size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL);
225 void* const compressedBuffer = malloc(cBuffSize);
226 void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
227 U32 randState = seed;
228 size_t cSize, testSize;
229 LZ4F_decompressionContext_t dCtx = NULL;
230 LZ4F_compressionContext_t cctx = NULL;
231 U64 crcOrig;
232 int basicTests_error = 0;
233 LZ4F_preferences_t prefs;
234 memset(&prefs, 0, sizeof(prefs));
235
236 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
237 DISPLAY("allocation error, not enough memory to start fuzzer tests \n");
238 goto _output_error;
239 }
240 FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
241 crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
242
243 /* LZ4F_compressBound() : special case : srcSize == 0 */
244 DISPLAYLEVEL(3, "LZ4F_compressBound(0) = ");
245 { size_t const cBound = LZ4F_compressBound(0, NULL);
246 if (cBound < 64 KB) goto _output_error;
247 DISPLAYLEVEL(3, " %u \n", (U32)cBound);
248 }
249
250 /* LZ4F_compressBound() : special case : automatic flushing enabled */
251 DISPLAYLEVEL(3, "LZ4F_compressBound(1 KB, autoFlush=1) = ");
252 { size_t cBound;
253 LZ4F_preferences_t autoFlushPrefs;
254 memset(&autoFlushPrefs, 0, sizeof(autoFlushPrefs));
255 autoFlushPrefs.autoFlush = 1;
256 cBound = LZ4F_compressBound(1 KB, &autoFlushPrefs);
257 if (cBound > 64 KB) goto _output_error;
258 DISPLAYLEVEL(3, " %u \n", (U32)cBound);
259 }
260
261 /* LZ4F_compressBound() : special case : automatic flushing disabled */
262 DISPLAYLEVEL(3, "LZ4F_compressBound(1 KB, autoFlush=0) = ");
263 { size_t const cBound = LZ4F_compressBound(1 KB, &prefs);
264 if (cBound < 64 KB) goto _output_error;
265 DISPLAYLEVEL(3, " %u \n", (U32)cBound);
266 }
267
268 /* Special case : null-content frame */
269 testSize = 0;
270 DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : ");
271 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
272 DISPLAYLEVEL(3, "null content encoded into a %u bytes frame \n", (unsigned)cSize);
273
274 DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
275 CHECK ( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
276
277 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n");
278 assert(cSize >= LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH);
279 { LZ4F_frameInfo_t frame_info;
280 size_t const fhs = LZ4F_headerSize(compressedBuffer, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH);
281 size_t avail_in = fhs;
282 CHECK( fhs );
283 CHECK( LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in) );
284 if (avail_in != fhs) goto _output_error; /* must consume all, since header size is supposed to be exact */
285 }
286
287 DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n");
288 CHECK( LZ4F_freeDecompressionContext(dCtx) );
289 dCtx = NULL;
290
291 /* test one-pass frame compression */
292 testSize = COMPRESSIBLE_NOISE_LENGTH;
293
294 DISPLAYLEVEL(3, "LZ4F_compressFrame, using fast level -3 : ");
295 { LZ4F_preferences_t fastCompressPrefs;
296 memset(&fastCompressPrefs, 0, sizeof(fastCompressPrefs));
297 fastCompressPrefs.compressionLevel = -3;
298 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, &fastCompressPrefs));
299 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
300 }
301
302 DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : ");
303 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL));
304 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
305
306 DISPLAYLEVEL(3, "Decompression test : \n");
307 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
308 size_t compressedBufferSize = cSize;
309
310 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
311
312 DISPLAYLEVEL(3, "Single Pass decompression : ");
313 CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) );
314 { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
315 if (crcDest != crcOrig) goto _output_error; }
316 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
317
318 DISPLAYLEVEL(3, "Reusing decompression context \n");
319 { size_t const missingBytes = 4;
320 size_t iSize = compressedBufferSize - missingBytes;
321 const BYTE* cBuff = (const BYTE*) compressedBuffer;
322 BYTE* const ostart = (BYTE*)decodedBuffer;
323 BYTE* op = ostart;
324 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
325 size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH;
326 DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes);
327 CHECK_V(decResult, LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL));
328 if (decResult != missingBytes) {
329 DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult);
330 goto _output_error;
331 }
332 DISPLAYLEVEL(3, "indeed, requests %u bytes \n", (unsigned)decResult);
333 cBuff += iSize;
334 iSize = decResult;
335 op += oSize;
336 oSize = (size_t)(oend-op);
337 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL);
338 if (decResult != 0) goto _output_error; /* should finish now */
339 op += oSize;
340 if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; }
341 { U64 const crcDest = XXH64(decodedBuffer, (size_t)(op-ostart), 1);
342 if (crcDest != crcOrig) goto _output_error;
343 } }
344
345 { size_t oSize = 0;
346 size_t iSize = 0;
347 LZ4F_frameInfo_t fi;
348 const BYTE* ip = (BYTE*)compressedBuffer;
349
350 DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
351 CHECK( LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL) );
352 //DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
353 DISPLAYLEVEL(3, " OK \n");
354
355 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on zero-size input : ");
356 { size_t nullSize = 0;
357 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &nullSize);
358 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
359 DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n",
360 LZ4F_getErrorName(fiError));
361 goto _output_error;
362 }
363 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
364 }
365
366 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on not enough input : ");
367 { size_t inputSize = 6;
368 size_t const fiError = LZ4F_getFrameInfo(dCtx, &fi, ip, &inputSize);
369 if (LZ4F_getErrorCode(fiError) != LZ4F_ERROR_frameHeader_incomplete) {
370 DISPLAYLEVEL(3, "incorrect error : %s != ERROR_frameHeader_incomplete \n", LZ4F_getErrorName(fiError));
371 goto _output_error;
372 }
373 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(fiError));
374 }
375
376 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on enough input : ");
377 iSize = LZ4F_headerSize(ip, LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH);
378 CHECK( iSize );
379 CHECK( LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize) );
380 DISPLAYLEVEL(3, " correctly decoded \n");
381 }
382
383 DISPLAYLEVEL(3, "Decode a buggy input : ");
384 assert(COMPRESSIBLE_NOISE_LENGTH > 64);
385 assert(cSize > 48);
386 memcpy(decodedBuffer, (char*)compressedBuffer+16, 32); /* save correct data */
387 memcpy((char*)compressedBuffer+16, (const char*)decodedBuffer+32, 32); /* insert noise */
388 { size_t dbSize = COMPRESSIBLE_NOISE_LENGTH;
389 size_t cbSize = cSize;
390 size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &dbSize,
391 compressedBuffer, &cbSize,
392 NULL);
393 if (!LZ4F_isError(decompressError)) goto _output_error;
394 DISPLAYLEVEL(3, "error detected : %s \n", LZ4F_getErrorName(decompressError));
395 }
396 memcpy((char*)compressedBuffer+16, decodedBuffer, 32); /* restore correct data */
397
398 DISPLAYLEVEL(3, "Reset decompression context, since it's left in error state \n");
399 LZ4F_resetDecompressionContext(dCtx); /* always successful */
400
401 DISPLAYLEVEL(3, "Byte after byte : ");
402 { BYTE* const ostart = (BYTE*)decodedBuffer;
403 BYTE* op = ostart;
404 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
405 const BYTE* ip = (const BYTE*) compressedBuffer;
406 const BYTE* const iend = ip + cSize;
407 while (ip < iend) {
408 size_t oSize = (size_t)(oend-op);
409 size_t iSize = 1;
410 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
411 op += oSize;
412 ip += iSize;
413 }
414 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
415 if (crcDest != crcOrig) goto _output_error;
416 }
417 DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), (unsigned)COMPRESSIBLE_NOISE_LENGTH);
418 }
419 }
420
421 DISPLAYLEVEL(3, "Using 64 KB block : ");
422 prefs.frameInfo.blockSizeID = LZ4F_max64KB;
423 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
424 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
425 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
426
427 DISPLAYLEVEL(3, "without checksum : ");
428 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
429 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
430 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
431
432 DISPLAYLEVEL(3, "Using 256 KB block : ");
433 prefs.frameInfo.blockSizeID = LZ4F_max256KB;
434 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
435 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs));
436 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
437
438 DISPLAYLEVEL(3, "Decompression test : \n");
439 { size_t const decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
440 unsigned const maxBits = FUZ_highbit((U32)decodedBufferSize);
441 BYTE* const ostart = (BYTE*)decodedBuffer;
442 BYTE* op = ostart;
443 BYTE* const oend = ostart + COMPRESSIBLE_NOISE_LENGTH;
444 const BYTE* ip = (const BYTE*)compressedBuffer;
445 const BYTE* const iend = (const BYTE*)compressedBuffer + cSize;
446
447 DISPLAYLEVEL(3, "random segment sizes : ");
448 while (ip < iend) {
449 unsigned const nbBits = FUZ_rand(&randState) % maxBits;
450 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
451 size_t oSize = (size_t)(oend-op);
452 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
453 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
454 op += oSize;
455 ip += iSize;
456 }
457 { size_t const decodedSize = (size_t)(op - ostart);
458 U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
459 if (crcDest != crcOrig) goto _output_error;
460 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
461 }
462
463 CHECK( LZ4F_freeDecompressionContext(dCtx) );
464 dCtx = NULL;
465 }
466
467 DISPLAYLEVEL(3, "without checksum : ");
468 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
469 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
470 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
471
472 DISPLAYLEVEL(3, "Using 1 MB block : ");
473 prefs.frameInfo.blockSizeID = LZ4F_max1MB;
474 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
475 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
476 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
477
478 DISPLAYLEVEL(3, "without frame checksum : ");
479 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
480 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
481 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
482
483 DISPLAYLEVEL(3, "Using 4 MB block : ");
484 prefs.frameInfo.blockSizeID = LZ4F_max4MB;
485 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
486 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
487 DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
488 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
489 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
490 }
491
492 DISPLAYLEVEL(3, "without frame checksum : ");
493 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
494 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
495 DISPLAYLEVEL(4, "dstCapacity = %u ; ", (U32)dstCapacity)
496 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs) );
497 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
498 }
499
500 DISPLAYLEVEL(3, "LZ4F_compressFrame with block checksum : ");
501 memset(&prefs, 0, sizeof(prefs));
502 prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled;
503 CHECK_V(cSize, LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs) );
504 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
505
506 DISPLAYLEVEL(3, "Decompress with block checksum : ");
507 { size_t iSize = cSize;
508 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
509 LZ4F_decompressionContext_t dctx;
510 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
511 CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) );
512 if (decodedSize != testSize) goto _output_error;
513 if (iSize != cSize) goto _output_error;
514 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
515 U64 const crcSrc = XXH64(CNBuffer, testSize, 1);
516 if (crcDest != crcSrc) goto _output_error;
517 }
518 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
519
520 CHECK( LZ4F_freeDecompressionContext(dctx) );
521 }
522
523 /* frame content size tests */
524 { size_t cErr;
525 BYTE* const ostart = (BYTE*)compressedBuffer;
526 BYTE* op = ostart;
527 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
528
529 DISPLAYLEVEL(3, "compress without frameSize : ");
530 memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
531 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
532 op += cErr;
533 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
534 op += cErr;
535 CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
536 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
537
538 DISPLAYLEVEL(3, "compress with frameSize : ");
539 prefs.frameInfo.contentSize = testSize;
540 op = ostart;
541 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
542 op += cErr;
543 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
544 op += cErr;
545 CHECK( LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL) );
546 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
547
548 DISPLAYLEVEL(3, "compress with wrong frameSize : ");
549 prefs.frameInfo.contentSize = testSize+1;
550 op = ostart;
551 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
552 op += cErr;
553 CHECK_V(cErr, LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL));
554 op += cErr;
555 cErr = LZ4F_compressEnd(cctx, op, testSize, NULL);
556 if (!LZ4F_isError(cErr)) goto _output_error;
557 DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(cErr));
558
559 CHECK( LZ4F_freeCompressionContext(cctx) );
560 cctx = NULL;
561 }
562
563 /* dictID tests */
564 { size_t cErr;
565 U32 const dictID = 0x99;
566 /* test advanced variant with custom allocator functions */
567 cctx = LZ4F_createCompressionContext_advanced(lz4f_cmem_test, LZ4F_VERSION);
568 if (cctx==NULL) goto _output_error;
569
570 DISPLAYLEVEL(3, "insert a dictID : ");
571 memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
572 prefs.frameInfo.dictID = dictID;
573 CHECK_V(cErr, LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs));
574 DISPLAYLEVEL(3, "created frame header of size %i bytes \n", (int)cErr);
575
576 DISPLAYLEVEL(3, "read a dictID : ");
577 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
578 memset(&prefs.frameInfo, 0, sizeof(prefs.frameInfo));
579 CHECK( LZ4F_getFrameInfo(dCtx, &prefs.frameInfo, compressedBuffer, &cErr) );
580 if (prefs.frameInfo.dictID != dictID) goto _output_error;
581 DISPLAYLEVEL(3, "%u \n", (U32)prefs.frameInfo.dictID);
582
583 CHECK( LZ4F_freeDecompressionContext(dCtx) ); dCtx = NULL;
584 CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
585 }
586
587 /* Dictionary compression test */
588 { size_t const dictSize = 7 KB; /* small enough for LZ4_MEMORY_USAGE == 10 */
589 size_t const srcSize = 65 KB; /* must be > 64 KB to avoid short-size optimizations */
590 size_t const dstCapacity = LZ4F_compressFrameBound(srcSize, NULL);
591 size_t cSizeNoDict, cSizeWithDict;
592 LZ4F_CDict* const cdict = LZ4F_createCDict(CNBuffer, dictSize);
593 if (cdict == NULL) goto _output_error;
594 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
595
596 DISPLAYLEVEL(3, "Testing LZ4F_createCDict_advanced : ");
597 { LZ4F_CDict* const cda = LZ4F_createCDict_advanced(lz4f_cmem_test, CNBuffer, dictSize);
598 if (cda == NULL) goto _output_error;
599 LZ4F_freeCDict(cda);
600 }
601 DISPLAYLEVEL(3, "OK \n");
602
603 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with NULL dict : ");
604 CHECK_V(cSizeNoDict,
605 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
606 CNBuffer, srcSize,
607 NULL, NULL) );
608 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeNoDict);
609
610 CHECK( LZ4F_freeCompressionContext(cctx) );
611 CHECK( LZ4F_createCompressionContext(&cctx, LZ4F_VERSION) );
612 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict : ");
613 CHECK_V(cSizeWithDict,
614 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
615 CNBuffer, srcSize,
616 cdict, NULL) );
617 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
618 (unsigned)srcSize, (unsigned)cSizeWithDict);
619 if (cSizeWithDict > cSizeNoDict) {
620 DISPLAYLEVEL(3, "cSizeWithDict (%zu) should have been more compact than cSizeNoDict(%zu) \n", cSizeWithDict, cSizeNoDict);
621 goto _output_error; /* must be more efficient */
622 }
623 crcOrig = XXH64(CNBuffer, srcSize, 0);
624
625 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict : ");
626 { LZ4F_dctx* dctx;
627 size_t decodedSize = srcSize;
628 size_t compressedSize = cSizeWithDict;
629 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
630 CHECK( LZ4F_decompress_usingDict(dctx,
631 decodedBuffer, &decodedSize,
632 compressedBuffer, &compressedSize,
633 CNBuffer, dictSize,
634 NULL) );
635 if (compressedSize != cSizeWithDict) goto _output_error;
636 if (decodedSize != srcSize) goto _output_error;
637 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
638 if (crcDest != crcOrig) goto _output_error; }
639 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
640 CHECK( LZ4F_freeDecompressionContext(dctx) );
641 }
642
643 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, negative level : ");
644 { size_t cSizeLevelMax;
645 LZ4F_preferences_t cParams;
646 memset(&cParams, 0, sizeof(cParams));
647 cParams.compressionLevel = -3;
648 CHECK_V(cSizeLevelMax,
649 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
650 CNBuffer, dictSize,
651 cdict, &cParams) );
652 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
653 }
654
655 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, with dict, level max : ");
656 { size_t cSizeLevelMax;
657 LZ4F_preferences_t cParams;
658 memset(&cParams, 0, sizeof(cParams));
659 cParams.compressionLevel = LZ4F_compressionLevel_max();
660 CHECK_V(cSizeLevelMax,
661 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, dstCapacity,
662 CNBuffer, dictSize,
663 cdict, &cParams) );
664 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cSizeLevelMax);
665 }
666
667 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple linked blocks : ");
668 { size_t cSizeContiguous;
669 size_t const inSize = dictSize * 3;
670 size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
671 LZ4F_preferences_t cParams;
672 memset(&cParams, 0, sizeof(cParams));
673 cParams.frameInfo.blockMode = LZ4F_blockLinked;
674 cParams.frameInfo.blockSizeID = LZ4F_max64KB;
675 CHECK_V(cSizeContiguous,
676 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
677 CNBuffer, inSize,
678 cdict, &cParams) );
679 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
680 (unsigned)inSize, (unsigned)cSizeContiguous);
681
682 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple linked blocks : ");
683 { LZ4F_dctx* dctx;
684 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
685 size_t compressedSize = cSizeContiguous;
686 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
687 CHECK( LZ4F_decompress_usingDict(dctx,
688 decodedBuffer, &decodedSize,
689 compressedBuffer, &compressedSize,
690 CNBuffer, dictSize,
691 NULL) );
692 if (compressedSize != cSizeContiguous) goto _output_error;
693 if (decodedSize != inSize) goto _output_error;
694 crcOrig = XXH64(CNBuffer, inSize, 0);
695 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
696 if (crcDest != crcOrig) goto _output_error; }
697 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
698 CHECK( LZ4F_freeDecompressionContext(dctx) );
699 }
700 }
701
702
703 DISPLAYLEVEL(3, "LZ4F_compressFrame_usingCDict, multiple independent blocks : ");
704 { size_t cSizeIndep;
705 size_t const inSize = dictSize * 3;
706 size_t const outCapacity = LZ4F_compressFrameBound(inSize, NULL);
707 LZ4F_preferences_t cParams;
708 memset(&cParams, 0, sizeof(cParams));
709 cParams.frameInfo.blockMode = LZ4F_blockIndependent;
710 cParams.frameInfo.blockSizeID = LZ4F_max64KB;
711 CHECK_V(cSizeIndep,
712 LZ4F_compressFrame_usingCDict(cctx, compressedBuffer, outCapacity,
713 CNBuffer, inSize,
714 cdict, &cParams) );
715 DISPLAYLEVEL(3, "compressed %u bytes into %u bytes \n",
716 (unsigned)inSize, (unsigned)cSizeIndep);
717
718 DISPLAYLEVEL(3, "LZ4F_decompress_usingDict on multiple independent blocks : ");
719 { LZ4F_dctx* dctx;
720 size_t decodedSize = COMPRESSIBLE_NOISE_LENGTH;
721 size_t compressedSize = cSizeIndep;
722 CHECK( LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION) );
723 CHECK( LZ4F_decompress_usingDict(dctx,
724 decodedBuffer, &decodedSize,
725 compressedBuffer, &compressedSize,
726 CNBuffer, dictSize,
727 NULL) );
728 if (compressedSize != cSizeIndep) goto _output_error;
729 if (decodedSize != inSize) goto _output_error;
730 crcOrig = XXH64(CNBuffer, inSize, 0);
731 { U64 const crcDest = XXH64(decodedBuffer, decodedSize, 0);
732 if (crcDest != crcOrig) goto _output_error; }
733 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
734 CHECK( LZ4F_freeDecompressionContext(dctx) );
735 }
736 }
737
738 LZ4F_freeCDict(cdict);
739 CHECK( LZ4F_freeCompressionContext(cctx) ); cctx = NULL;
740 }
741
742 DISPLAYLEVEL(3, "getBlockSize test: \n");
743 { size_t result;
744 unsigned blockSizeID;
745 for (blockSizeID = 4; blockSizeID < 8; ++blockSizeID) {
746 result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)blockSizeID);
747 CHECK(result);
748 DISPLAYLEVEL(3, "Returned block size of %u bytes for blockID %u \n",
749 (unsigned)result, blockSizeID);
750 }
751
752 /* Test an invalid input that's too large */
753 result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)8);
754 if(!LZ4F_isError(result) ||
755 LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
756 goto _output_error;
757
758 /* Test an invalid input that's too small */
759 result = LZ4F_getBlockSize((LZ4F_blockSizeID_t)3);
760 if(!LZ4F_isError(result) ||
761 LZ4F_getErrorCode(result) != LZ4F_ERROR_maxBlockSize_invalid)
762 goto _output_error;
763 }
764
765
766 DISPLAYLEVEL(3, "Skippable frame test : \n");
767 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
768 unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
769 BYTE* op = (BYTE*)decodedBuffer;
770 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
771 BYTE* ip = (BYTE*)compressedBuffer;
772 BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
773
774 CHECK( LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION) );
775
776 /* generate skippable frame */
777 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
778 FUZ_writeLE32(ip+4, (U32)cSize);
779
780 DISPLAYLEVEL(3, "random segment sizes : \n");
781 while (ip < iend) {
782 unsigned nbBits = FUZ_rand(&randState) % maxBits;
783 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
784 size_t oSize = (size_t)(oend-op);
785 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
786 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
787 op += oSize;
788 ip += iSize;
789 }
790 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize);
791
792 /* generate zero-size skippable frame */
793 DISPLAYLEVEL(3, "zero-size skippable frame\n");
794 ip = (BYTE*)compressedBuffer;
795 op = (BYTE*)decodedBuffer;
796 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1);
797 FUZ_writeLE32(ip+4, 0);
798 iend = ip+8;
799
800 while (ip < iend) {
801 unsigned const nbBits = FUZ_rand(&randState) % maxBits;
802 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
803 size_t oSize = (size_t)(oend-op);
804 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
805 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
806 op += oSize;
807 ip += iSize;
808 }
809 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
810
811 DISPLAYLEVEL(3, "Skippable frame header complete in first call \n");
812 ip = (BYTE*)compressedBuffer;
813 op = (BYTE*)decodedBuffer;
814 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2);
815 FUZ_writeLE32(ip+4, 10);
816 iend = ip+18;
817 while (ip < iend) {
818 size_t iSize = 10;
819 size_t oSize = 10;
820 if (iSize > (size_t)(iend-ip)) iSize = (size_t)(iend-ip);
821 CHECK( LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL) );
822 op += oSize;
823 ip += iSize;
824 }
825 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
826 }
827
828 DISPLAY("Basic tests completed \n");
829 _end:
830 free(CNBuffer);
831 free(compressedBuffer);
832 free(decodedBuffer);
833 LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
834 LZ4F_freeCompressionContext(cctx); cctx = NULL;
835 return basicTests_error;
836
837 _output_error:
838 basicTests_error = 1;
839 DISPLAY("Error detected ! \n");
840 goto _end;
841 }
842
843
844 typedef enum { o_contiguous, o_noncontiguous, o_overwrite } o_scenario_e;
845
locateBuffDiff(const void * buff1,const void * buff2,size_t size,o_scenario_e o_scenario)846 static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_scenario_e o_scenario)
847 {
848 if (displayLevel >= 2) {
849 size_t p=0;
850 const BYTE* b1=(const BYTE*)buff1;
851 const BYTE* b2=(const BYTE*)buff2;
852 DISPLAY("locateBuffDiff: looking for error position \n");
853 if (o_scenario != o_contiguous) {
854 DISPLAY("mode %i: non-contiguous output (%u bytes), cannot search \n",
855 (int)o_scenario, (unsigned)size);
856 return;
857 }
858 while (p < size && b1[p]==b2[p]) p++;
859 if (p != size) {
860 DISPLAY("Error at pos %i/%i : %02X != %02X \n", (int)p, (int)size, b1[p], b2[p]);
861 }
862 }
863 }
864
865 # define EXIT_MSG(...) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
866 DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); exit(1); }
867 # undef CHECK
868 # define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
869
870
test_lz4f_decompression_wBuffers(const void * cSrc,size_t cSize,void * dst,size_t dstCapacity,o_scenario_e o_scenario,const void * srcRef,size_t decompressedSize,U64 crcOrig,U32 * const randState,LZ4F_dctx * const dCtx,U32 seed,U32 testNb,int findErrorPos)871 size_t test_lz4f_decompression_wBuffers(
872 const void* cSrc, size_t cSize,
873 void* dst, size_t dstCapacity, o_scenario_e o_scenario,
874 const void* srcRef, size_t decompressedSize,
875 U64 crcOrig,
876 U32* const randState,
877 LZ4F_dctx* const dCtx,
878 U32 seed, U32 testNb,
879 int findErrorPos)
880 {
881 const BYTE* ip = (const BYTE*)cSrc;
882 const BYTE* const iend = ip + cSize;
883
884 BYTE* op = (BYTE*)dst;
885 BYTE* const oend = op + dstCapacity;
886
887 unsigned const suggestedBits = FUZ_highbit((U32)cSize);
888 unsigned const maxBits = MAX(3, suggestedBits);
889 size_t totalOut = 0;
890 size_t moreToFlush = 0;
891 XXH64_state_t xxh64;
892 XXH64_reset(&xxh64, 1);
893 assert(ip < iend);
894 while (ip < iend) {
895 unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1;
896 unsigned const nbBitsO = (FUZ_rand(randState) % (maxBits)) + 1;
897 size_t const iSizeCand = (FUZ_rand(randState) & ((1<<nbBitsI)-1)) + 1;
898 size_t const iSizeMax = MIN(iSizeCand, (size_t)(iend-ip));
899 size_t iSize = iSizeMax;
900 size_t const oSizeCand = (FUZ_rand(randState) & ((1<<nbBitsO)-1)) + 2;
901 size_t const oSizeMax = MIN(oSizeCand, (size_t)(oend-op));
902 int const sentinelTest = (op + oSizeMax < oend);
903 size_t oSize = oSizeMax;
904 BYTE const mark = (BYTE)(FUZ_rand(randState) & 255);
905 LZ4F_decompressOptions_t dOptions;
906 memset(&dOptions, 0, sizeof(dOptions));
907 dOptions.stableDst = FUZ_rand(randState) & 1;
908 if (o_scenario == o_overwrite) dOptions.stableDst = 0; /* overwrite mode */
909 dOptions.skipChecksums = FUZ_rand(randState) & 127;
910 if (sentinelTest) op[oSizeMax] = mark;
911
912 DISPLAYLEVEL(7, "dstCapacity=%u, presentedInput=%u \n", (unsigned)oSize, (unsigned)iSize);
913
914 /* read data from byte-exact buffer to catch out-of-bound reads */
915 { void* const iBuffer = malloc(iSizeMax);
916 void* const tmpop = (FUZ_rand(randState) & (oSize == 0)) ? NULL : op;
917 const void* const tmpip = (FUZ_rand(randState) & (iSize == 0)) ? NULL : iBuffer;
918 assert(iBuffer != NULL);
919 memcpy(iBuffer, ip, iSizeMax);
920 moreToFlush = LZ4F_decompress(dCtx, tmpop, &oSize, tmpip, &iSize, &dOptions);
921 free(iBuffer);
922 }
923 DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize);
924
925 if (sentinelTest) {
926 CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : "
927 "Decompression overwrites beyond assigned dst size",
928 op[oSizeMax], mark);
929 }
930 if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid) {
931 if (findErrorPos) DISPLAYLEVEL(2, "checksum error detected \n");
932 if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
933 }
934 if (LZ4F_isError(moreToFlush)) return moreToFlush;
935
936 XXH64_update(&xxh64, op, oSize);
937 totalOut += oSize;
938 op += oSize;
939 ip += iSize;
940 if (o_scenario == o_noncontiguous) {
941 if (op == oend) return LZ4F_ERROR_GENERIC; /* can theoretically happen with bogus data */
942 op++; /* create a gap between consecutive output */
943 }
944 if (o_scenario==o_overwrite) op = (BYTE*)dst; /* overwrite destination */
945 if ( (op == oend) /* no more room for output; can happen with bogus input */
946 && (iSize == 0)) /* no input consumed */
947 break;
948 }
949 if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed;
950 if (totalOut) { /* otherwise, it's a skippable frame */
951 U64 const crcDecoded = XXH64_digest(&xxh64);
952 if (crcDecoded != crcOrig) {
953 if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
954 return LZ4F_ERROR_contentChecksum_invalid;
955 } }
956 return 0;
957 }
958
959
test_lz4f_decompression(const void * cSrc,size_t cSize,const void * srcRef,size_t decompressedSize,U64 crcOrig,U32 * const randState,LZ4F_dctx * const dCtx,U32 seed,U32 testNb,int findErrorPos)960 size_t test_lz4f_decompression(const void* cSrc, size_t cSize,
961 const void* srcRef, size_t decompressedSize,
962 U64 crcOrig,
963 U32* const randState,
964 LZ4F_dctx* const dCtx,
965 U32 seed, U32 testNb,
966 int findErrorPos)
967 {
968 o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3); /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */
969 /* tighten dst buffer conditions */
970 size_t const dstCapacity = (o_scenario == o_noncontiguous) ?
971 (decompressedSize * 2) + 128 :
972 decompressedSize;
973 size_t result;
974 void* const dstBuffer = malloc(dstCapacity);
975 assert(dstBuffer != NULL);
976
977 result = test_lz4f_decompression_wBuffers(cSrc, cSize,
978 dstBuffer, dstCapacity, o_scenario,
979 srcRef, decompressedSize,
980 crcOrig,
981 randState,
982 dCtx,
983 seed, testNb, findErrorPos);
984
985 free(dstBuffer);
986 return result;
987 }
988
989
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,double compressibility,U32 duration_s)990 int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s)
991 {
992 unsigned testNb = 0;
993 size_t const CNBufferLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
994 void* CNBuffer = NULL;
995 size_t const compressedBufferSize = LZ4F_compressFrameBound(CNBufferLength, NULL) + 4 MB; /* needs some margin */
996 void* compressedBuffer = NULL;
997 void* decodedBuffer = NULL;
998 U32 coreRand = seed;
999 LZ4F_decompressionContext_t dCtx = NULL;
1000 LZ4F_decompressionContext_t dCtxNoise = NULL;
1001 LZ4F_compressionContext_t cCtx = NULL;
1002 clock_t const startClock = clock();
1003 clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
1004
1005 /* Create states & buffers */
1006 { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
1007 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
1008 { size_t const creationStatus = LZ4F_createDecompressionContext(&dCtxNoise, LZ4F_VERSION);
1009 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
1010 { size_t const creationStatus = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
1011 CHECK(LZ4F_isError(creationStatus), "Allocation failed (error %i)", (int)creationStatus); }
1012 CNBuffer = malloc(CNBufferLength);
1013 CHECK(CNBuffer==NULL, "CNBuffer Allocation failed");
1014 compressedBuffer = malloc(compressedBufferSize);
1015 CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
1016 decodedBuffer = calloc(1, CNBufferLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
1017 CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
1018 FUZ_fillCompressibleNoiseBuffer(CNBuffer, CNBufferLength, compressibility, &coreRand);
1019
1020 /* jump to requested testNb */
1021 for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */
1022
1023 /* main fuzzer test loop */
1024 for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) {
1025 U32 randState = coreRand ^ prime1;
1026 unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(CNBufferLength-1)) - 1)) + 1;
1027 size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1;
1028 size_t const srcStartId = FUZ_rand(&randState) % (CNBufferLength - srcSize);
1029 const BYTE* const srcStart = (const BYTE*)CNBuffer + srcStartId;
1030 unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1;
1031 U64 const crcOrig = XXH64(srcStart, srcSize, 1);
1032 LZ4F_preferences_t prefs;
1033 const LZ4F_preferences_t* prefsPtr = &prefs;
1034 size_t cSize;
1035
1036 (void)FUZ_rand(&coreRand); /* update seed */
1037 memset(&prefs, 0, sizeof(prefs));
1038 prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1);
1039 prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3));
1040 prefs.frameInfo.blockChecksumFlag = (LZ4F_blockChecksum_t)(FUZ_rand(&randState) & 1);
1041 prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1);
1042 prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
1043 prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2;
1044 prefs.compressionLevel = -5 + (int)(FUZ_rand(&randState) % 11);
1045 if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
1046
1047 DISPLAYUPDATE(2, "\r%5u ", testNb);
1048
1049 if ((FUZ_rand(&randState) & 0xFFF) == 0) {
1050 /* create a skippable frame (rare case) */
1051 BYTE* op = (BYTE*)compressedBuffer;
1052 FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15));
1053 FUZ_writeLE32(op+4, (U32)srcSize);
1054 cSize = srcSize+8;
1055
1056 } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */
1057 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr);
1058 CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
1059
1060 } else { /* multi-segments compression */
1061 const BYTE* ip = srcStart;
1062 const BYTE* const iend = srcStart + srcSize;
1063 BYTE* op = (BYTE*)compressedBuffer;
1064 BYTE* const oend = op + (neverFlush ? LZ4F_compressFrameBound(srcSize, prefsPtr) : compressedBufferSize); /* when flushes are possible, can't guarantee a max compressed size */
1065 unsigned const maxBits = FUZ_highbit((U32)srcSize);
1066 LZ4F_compressOptions_t cOptions;
1067 memset(&cOptions, 0, sizeof(cOptions));
1068 { size_t const fhSize = LZ4F_compressBegin(cCtx, op, (size_t)(oend-op), prefsPtr);
1069 CHECK(LZ4F_isError(fhSize), "Compression header failed (error %i)",
1070 (int)fhSize);
1071 op += fhSize;
1072 }
1073 while (ip < iend) {
1074 unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits;
1075 size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
1076 size_t iSize = MIN(sampleMax, (size_t)(iend-ip));
1077 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
1078 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
1079 DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n",
1080 (unsigned)iSize, cOptions.stableSrc);
1081
1082 #if 1
1083 /* insert uncompressed segment */
1084 if ( (iSize>0)
1085 && !neverFlush /* do not mess with compressBound when neverFlush is set */
1086 && prefsPtr != NULL /* prefs are set */
1087 && prefs.frameInfo.blockMode == LZ4F_blockIndependent /* uncompressedUpdate is only valid with blockMode==independent */
1088 && (FUZ_rand(&randState) & 15) == 1 ) {
1089 size_t const uSize = FUZ_rand(&randState) % iSize;
1090 size_t const flushedSize = LZ4F_uncompressedUpdate(cCtx, op, (size_t)(oend-op), ip, uSize, &cOptions);
1091 CHECK(LZ4F_isError(flushedSize), "Insert uncompressed data failed (error %i : %s)",
1092 (int)flushedSize, LZ4F_getErrorName(flushedSize));
1093 op += flushedSize;
1094 ip += uSize;
1095 iSize -= uSize;
1096 }
1097 #endif
1098
1099 { size_t const flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
1100 CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)",
1101 (int)flushedSize, LZ4F_getErrorName(flushedSize));
1102 op += flushedSize;
1103 ip += iSize;
1104 }
1105
1106 { unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1);
1107 if (forceFlush) {
1108 size_t const flushSize = LZ4F_flush(cCtx, op, (size_t)(oend-op), &cOptions);
1109 DISPLAYLEVEL(6,"flushing %u bytes \n", (unsigned)flushSize);
1110 CHECK(LZ4F_isError(flushSize), "Compression failed (error %i)", (int)flushSize);
1111 op += flushSize;
1112 if ((FUZ_rand(&randState) % 1024) == 3) {
1113 /* add an empty block (requires uncompressed flag) */
1114 op[0] = op[1] = op[2] = 0;
1115 op[3] = 0x80; /* 0x80000000U in little-endian format */
1116 op += 4;
1117 if ((prefsPtr!= NULL) && prefsPtr->frameInfo.blockChecksumFlag) {
1118 /* add block checksum (even for empty blocks) */
1119 FUZ_writeLE32(op, XXH32(op, 0, 0));
1120 op += 4;
1121 } } } }
1122 } /* while (ip<iend) */
1123 CHECK(op>=oend, "LZ4F_compressFrameBound overflow");
1124 { size_t const dstEndSafeSize = LZ4F_compressBound(0, prefsPtr);
1125 int const tooSmallDstEnd = ((FUZ_rand(&randState) & 31) == 3);
1126 size_t const dstEndTooSmallSize = (FUZ_rand(&randState) % dstEndSafeSize) + 1;
1127 size_t const dstEndSize = tooSmallDstEnd ? dstEndTooSmallSize : dstEndSafeSize;
1128 BYTE const canaryByte = (BYTE)(FUZ_rand(&randState) & 255);
1129 size_t flushedSize;
1130 DISPLAYLEVEL(7,"canaryByte at pos %u / %u \n",
1131 (unsigned)((size_t)(op - (BYTE*)compressedBuffer) + dstEndSize),
1132 (unsigned)compressedBufferSize);
1133 assert(op + dstEndSize < (BYTE*)compressedBuffer + compressedBufferSize);
1134 op[dstEndSize] = canaryByte;
1135 flushedSize = LZ4F_compressEnd(cCtx, op, dstEndSize, &cOptions);
1136 CHECK(op[dstEndSize] != canaryByte, "LZ4F_compressEnd writes beyond dstCapacity !");
1137 if (LZ4F_isError(flushedSize)) {
1138 if (tooSmallDstEnd) /* failure is allowed */ continue;
1139 CHECK(!tooSmallDstEnd, "Compression completion failed (error %i : %s)",
1140 (int)flushedSize, LZ4F_getErrorName(flushedSize));
1141 }
1142 op += flushedSize;
1143 }
1144 cSize = (size_t)(op - (BYTE*)compressedBuffer);
1145 DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize);
1146 }
1147
1148
1149 /* multi-segments decompression */
1150 DISPLAYLEVEL(6, "normal decompression \n");
1151 { size_t result = test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtx, seed, testNb, 1 /*findError*/ );
1152 CHECK (LZ4F_isError(result), "multi-segment decompression failed (error %i => %s)",
1153 (int)result, LZ4F_getErrorName(result));
1154 }
1155
1156 #if 1
1157 /* insert noise into src */
1158 { U32 const maxNbBits = FUZ_highbit((U32)cSize);
1159 size_t pos = 0;
1160 for (;;) {
1161 /* keep some original src */
1162 { U32 const nbBits = FUZ_rand(&randState) % maxNbBits;
1163 size_t const mask = (1<<nbBits) - 1;
1164 size_t const skipLength = FUZ_rand(&randState) & mask;
1165 pos += skipLength;
1166 }
1167 if (pos >= cSize) break;
1168 /* add noise */
1169 { U32 const nbBitsCodes = FUZ_rand(&randState) % maxNbBits;
1170 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
1171 size_t const mask = (1<<nbBits) - 1;
1172 size_t const rNoiseLength = (FUZ_rand(&randState) & mask) + 1;
1173 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
1174 size_t const noiseStart = FUZ_rand(&randState) % (CNBufferLength - noiseLength);
1175 memcpy((BYTE*)compressedBuffer + pos, (const char*)CNBuffer + noiseStart, noiseLength);
1176 pos += noiseLength;
1177 } } }
1178
1179 /* test decompression on noisy src */
1180 DISPLAYLEVEL(6, "noisy decompression \n");
1181 test_lz4f_decompression(compressedBuffer, cSize, srcStart, srcSize, crcOrig, &randState, dCtxNoise, seed, testNb, 0 /*don't search error Pos*/ );
1182 /* note : we don't analyze result here : it probably failed, which is expected.
1183 * The sole purpose is to catch potential out-of-bound reads and writes. */
1184 LZ4F_resetDecompressionContext(dCtxNoise); /* context must be reset after an error */
1185 #endif
1186
1187 } /* for ( ; (testNb < nbTests) ; ) */
1188
1189 DISPLAYLEVEL(2, "\rAll tests completed \n");
1190
1191 LZ4F_freeDecompressionContext(dCtx);
1192 LZ4F_freeDecompressionContext(dCtxNoise);
1193 LZ4F_freeCompressionContext(cCtx);
1194 free(CNBuffer);
1195 free(compressedBuffer);
1196 free(decodedBuffer);
1197
1198 if (use_pause) {
1199 DISPLAY("press enter to finish \n");
1200 (void)getchar();
1201 }
1202 return 0;
1203 }
1204
1205
FUZ_usage(const char * programName)1206 int FUZ_usage(const char* programName)
1207 {
1208 DISPLAY( "Usage :\n");
1209 DISPLAY( " %s [args]\n", programName);
1210 DISPLAY( "\n");
1211 DISPLAY( "Arguments :\n");
1212 DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
1213 DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
1214 DISPLAY( " -s# : Select seed (default:prompt user)\n");
1215 DISPLAY( " -t# : Select starting test number (default:0)\n");
1216 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
1217 DISPLAY( " -v : verbose\n");
1218 DISPLAY( " -h : display help and exit\n");
1219 return 0;
1220 }
1221
1222
main(int argc,const char ** argv)1223 int main(int argc, const char** argv)
1224 {
1225 U32 seed=0;
1226 int seedset=0;
1227 int argNb;
1228 unsigned nbTests = nbTestsDefault;
1229 unsigned testNb = 0;
1230 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
1231 int result=0;
1232 U32 duration=0;
1233 const char* const programName = argv[0];
1234
1235 /* Check command line */
1236 for (argNb=1; argNb<argc; argNb++) {
1237 const char* argument = argv[argNb];
1238
1239 if(!argument) continue; /* Protection if argument empty */
1240
1241 /* Decode command (note : aggregated short commands are allowed) */
1242 if (argument[0]=='-') {
1243 if (!strcmp(argument, "--no-prompt")) {
1244 no_prompt=1;
1245 seedset=1;
1246 displayLevel=1;
1247 continue;
1248 }
1249 argument++;
1250
1251 while (*argument!=0) {
1252 switch(*argument)
1253 {
1254 case 'h':
1255 return FUZ_usage(programName);
1256 case 'v':
1257 argument++;
1258 displayLevel++;
1259 break;
1260 case 'q':
1261 argument++;
1262 displayLevel--;
1263 break;
1264 case 'p': /* pause at the end */
1265 argument++;
1266 use_pause = 1;
1267 break;
1268
1269 case 'i':
1270 argument++;
1271 nbTests=0; duration=0;
1272 while ((*argument>='0') && (*argument<='9')) {
1273 nbTests *= 10;
1274 nbTests += (unsigned)(*argument - '0');
1275 argument++;
1276 }
1277 break;
1278
1279 case 'T':
1280 argument++;
1281 nbTests = 0; duration = 0;
1282 for (;;) {
1283 switch(*argument)
1284 {
1285 case 'm': duration *= 60; argument++; continue;
1286 case 's':
1287 case 'n': argument++; continue;
1288 case '0':
1289 case '1':
1290 case '2':
1291 case '3':
1292 case '4':
1293 case '5':
1294 case '6':
1295 case '7':
1296 case '8':
1297 case '9': duration *= 10; duration += (U32)(*argument++ - '0'); continue;
1298 }
1299 break;
1300 }
1301 break;
1302
1303 case 's':
1304 argument++;
1305 seed=0;
1306 seedset=1;
1307 while ((*argument>='0') && (*argument<='9')) {
1308 seed *= 10;
1309 seed += (U32)(*argument - '0');
1310 argument++;
1311 }
1312 break;
1313 case 't':
1314 argument++;
1315 testNb=0;
1316 while ((*argument>='0') && (*argument<='9')) {
1317 testNb *= 10;
1318 testNb += (unsigned)(*argument - '0');
1319 argument++;
1320 }
1321 break;
1322 case 'P': /* compressibility % */
1323 argument++;
1324 proba=0;
1325 while ((*argument>='0') && (*argument<='9')) {
1326 proba *= 10;
1327 proba += *argument - '0';
1328 argument++;
1329 }
1330 if (proba<0) proba=0;
1331 if (proba>100) proba=100;
1332 break;
1333 default:
1334 ;
1335 return FUZ_usage(programName);
1336 }
1337 }
1338 }
1339 }
1340
1341 /* Get Seed */
1342 DISPLAY("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING);
1343
1344 if (!seedset) {
1345 time_t const t = time(NULL);
1346 U32 const h = XXH32(&t, sizeof(t), 1);
1347 seed = h % 10000;
1348 }
1349 DISPLAY("Seed = %u\n", seed);
1350 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
1351
1352 nbTests += (nbTests==0); /* avoid zero */
1353
1354 if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
1355 if (result) return 1;
1356 return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration);
1357 }
1358