1 /*
2 frameTest - test tool for lz4frame
3 Copyright (C) Yann Collet 2014-2016
4
5 GPL v2 License
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License along
18 with this program; if not, write to the Free Software Foundation, Inc.,
19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21 You can contact the author at :
22 - LZ4 homepage : http://www.lz4.org
23 - LZ4 source repository : https://github.com/lz4/lz4
24 */
25
26 /*-************************************
27 * Compiler specific
28 **************************************/
29 #ifdef _MSC_VER /* Visual Studio */
30 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
31 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
32 #endif
33
34
35 /*-************************************
36 * Includes
37 **************************************/
38 #include "util.h" /* U32 */
39 #include <stdlib.h> /* malloc, free */
40 #include <stdio.h> /* fprintf */
41 #include <string.h> /* strcmp */
42 #include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
43 #include "lz4frame_static.h"
44 #include "lz4.h" /* LZ4_VERSION_STRING */
45 #define XXH_STATIC_LINKING_ONLY
46 #include "xxhash.h" /* XXH64 */
47
48
49 /* unoptimized version; solves endianess & alignment issues */
FUZ_writeLE32(void * dstVoidPtr,U32 value32)50 static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
51 {
52 BYTE* dstPtr = (BYTE*)dstVoidPtr;
53 dstPtr[0] = (BYTE)value32;
54 dstPtr[1] = (BYTE)(value32 >> 8);
55 dstPtr[2] = (BYTE)(value32 >> 16);
56 dstPtr[3] = (BYTE)(value32 >> 24);
57 }
58
59
60 /*-************************************
61 * Constants
62 **************************************/
63 #define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
64
65 #define KB *(1U<<10)
66 #define MB *(1U<<20)
67 #define GB *(1U<<30)
68
69 static const U32 nbTestsDefault = 256 KB;
70 #define COMPRESSIBLE_NOISE_LENGTH (2 MB)
71 #define FUZ_COMPRESSIBILITY_DEFAULT 50
72 static const U32 prime1 = 2654435761U;
73 static const U32 prime2 = 2246822519U;
74
75
76
77 /*-************************************
78 * Macros
79 **************************************/
80 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
81 #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
82 #define DISPLAYUPDATE(l, ...) if (displayLevel>=l) { \
83 if ((FUZ_GetClockSpan(g_clockTime) > refreshRate) || (displayLevel>=4)) \
84 { g_clockTime = clock(); DISPLAY(__VA_ARGS__); \
85 if (displayLevel>=4) fflush(stdout); } }
86 static const clock_t refreshRate = CLOCKS_PER_SEC / 6;
87 static clock_t g_clockTime = 0;
88
89
90 /*-***************************************
91 * Local Parameters
92 *****************************************/
93 static U32 no_prompt = 0;
94 static U32 displayLevel = 2;
95 static U32 use_pause = 0;
96
97
98 /*-*******************************************************
99 * Fuzzer functions
100 *********************************************************/
101 #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
102 #define MAX(a,b) ( (a) > (b) ? (a) : (b) )
103
FUZ_GetClockSpan(clock_t clockStart)104 static clock_t FUZ_GetClockSpan(clock_t clockStart)
105 {
106 return clock() - clockStart; /* works even if overflow; max span ~ 30 mn */
107 }
108
109
110 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(unsigned int * src)111 unsigned int FUZ_rand(unsigned int* src)
112 {
113 U32 rand32 = *src;
114 rand32 *= prime1;
115 rand32 += prime2;
116 rand32 = FUZ_rotl32(rand32, 13);
117 *src = rand32;
118 return rand32 >> 5;
119 }
120
121
122 #define FUZ_RAND15BITS (FUZ_rand(seed) & 0x7FFF)
123 #define FUZ_RANDLENGTH ( (FUZ_rand(seed) & 3) ? (FUZ_rand(seed) % 15) : (FUZ_rand(seed) % 510) + 15)
FUZ_fillCompressibleNoiseBuffer(void * buffer,size_t bufferSize,double proba,U32 * seed)124 static void FUZ_fillCompressibleNoiseBuffer(void* buffer, size_t bufferSize, double proba, U32* seed)
125 {
126 BYTE* BBuffer = (BYTE*)buffer;
127 size_t pos = 0;
128 U32 P32 = (U32)(32768 * proba);
129
130 /* First Byte */
131 BBuffer[pos++] = (BYTE)(FUZ_rand(seed));
132
133 while (pos < bufferSize) {
134 /* Select : Literal (noise) or copy (within 64K) */
135 if (FUZ_RAND15BITS < P32) {
136 /* Copy (within 64K) */
137 size_t const lengthRand = FUZ_RANDLENGTH + 4;
138 size_t const length = MIN(lengthRand, bufferSize - pos);
139 size_t const end = pos + length;
140 size_t const offsetRand = FUZ_RAND15BITS + 1;
141 size_t const offset = MIN(offsetRand, pos);
142 size_t match = pos - offset;
143 while (pos < end) BBuffer[pos++] = BBuffer[match++];
144 } else {
145 /* Literal (noise) */
146 size_t const lengthRand = FUZ_RANDLENGTH + 4;
147 size_t const length = MIN(lengthRand, bufferSize - pos);
148 size_t const end = pos + length;
149 while (pos < end) BBuffer[pos++] = (BYTE)(FUZ_rand(seed) >> 5);
150 }
151 }
152 }
153
154
FUZ_highbit(U32 v32)155 static unsigned FUZ_highbit(U32 v32)
156 {
157 unsigned nbBits = 0;
158 if (v32==0) return 0;
159 while (v32) v32 >>= 1, nbBits ++;
160 return nbBits;
161 }
162
163
164 /*-*******************************************************
165 * Tests
166 *********************************************************/
basicTests(U32 seed,double compressibility)167 int basicTests(U32 seed, double compressibility)
168 {
169 int testResult = 0;
170 void* const CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
171 size_t const cBuffSize = LZ4F_compressFrameBound(COMPRESSIBLE_NOISE_LENGTH, NULL);
172 void* const compressedBuffer = malloc(cBuffSize);
173 void* const decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
174 U32 randState = seed;
175 size_t cSize, testSize;
176 LZ4F_decompressionContext_t dCtx = NULL;
177 LZ4F_compressionContext_t cctx = NULL;
178 U64 crcOrig;
179
180 LZ4F_preferences_t prefs;
181 memset(&prefs, 0, sizeof(prefs));
182 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
183 DISPLAY("allocation error, not enough memory to start fuzzer tests \n");
184 goto _output_error;
185 }
186 FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, &randState);
187 crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
188
189 /* LZ4F_compressBound() : special case : srcSize == 0 */
190 DISPLAYLEVEL(3, "LZ4F_compressBound(0) = ");
191 { size_t const cBound = LZ4F_compressBound(0, NULL);
192 if (cBound < 64 KB) goto _output_error;
193 DISPLAYLEVEL(3, " %u \n", (U32)cBound);
194 }
195
196 /* Special case : null-content frame */
197 testSize = 0;
198 DISPLAYLEVEL(3, "LZ4F_compressFrame, compress null content : \n");
199 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
200 if (LZ4F_isError(cSize)) goto _output_error;
201 DISPLAYLEVEL(3, "Compressed null content into a %i bytes frame \n", (int)cSize);
202
203 DISPLAYLEVEL(3, "LZ4F_createDecompressionContext \n");
204 { LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
205 if (LZ4F_isError(errorCode)) goto _output_error; }
206
207 DISPLAYLEVEL(3, "LZ4F_getFrameInfo on null-content frame (#157) \n");
208 { size_t avail_in = cSize;
209 LZ4F_frameInfo_t frame_info;
210 LZ4F_errorCode_t const errorCode = LZ4F_getFrameInfo(dCtx, &frame_info, compressedBuffer, &avail_in);
211 if (LZ4F_isError(errorCode)) goto _output_error;
212 }
213
214 DISPLAYLEVEL(3, "LZ4F_freeDecompressionContext \n");
215 { LZ4F_errorCode_t const errorCode = LZ4F_freeDecompressionContext(dCtx);
216 if (LZ4F_isError(errorCode)) goto _output_error; }
217 dCtx = NULL;
218
219 /* test one-pass frame compression */
220 testSize = COMPRESSIBLE_NOISE_LENGTH;
221 DISPLAYLEVEL(3, "LZ4F_compressFrame, using default preferences : \n");
222 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, NULL), CNBuffer, testSize, NULL);
223 if (LZ4F_isError(cSize)) goto _output_error;
224 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
225
226 DISPLAYLEVEL(3, "Decompression test : \n");
227 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
228 size_t compressedBufferSize = cSize;
229 BYTE* ip = (BYTE*)compressedBuffer;
230 BYTE* const iend = (BYTE*)compressedBuffer + cSize;
231
232 LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
233 if (LZ4F_isError(errorCode)) goto _output_error;
234
235 DISPLAYLEVEL(3, "Single Pass decompression : \n");
236 { size_t const decompressError = LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL);
237 if (LZ4F_isError(decompressError)) goto _output_error; }
238 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
239 if (crcDest != crcOrig) goto _output_error; }
240 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
241
242 DISPLAYLEVEL(3, "Reusing decompression context \n");
243 { size_t const missingBytes = 4;
244 size_t iSize = compressedBufferSize - missingBytes;
245 const BYTE* cBuff = (const BYTE*) compressedBuffer;
246 BYTE* const ostart = (BYTE*)decodedBuffer;
247 BYTE* op = ostart;
248 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
249 size_t decResult, oSize = COMPRESSIBLE_NOISE_LENGTH;
250 DISPLAYLEVEL(3, "Missing last %u bytes : ", (U32)missingBytes);
251 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL);
252 if (LZ4F_isError(decResult)) goto _output_error;
253 if (decResult != missingBytes) {
254 DISPLAY("%u bytes missing != %u bytes requested \n", (U32)missingBytes, (U32)decResult);
255 goto _output_error;
256 }
257 DISPLAYLEVEL(3, "indeed, requests %u bytes \n", (unsigned)decResult);
258 cBuff += iSize;
259 iSize = decResult;
260 op += oSize;
261 oSize = (size_t)(oend-op);
262 decResult = LZ4F_decompress(dCtx, op, &oSize, cBuff, &iSize, NULL);
263 if (decResult != 0) goto _output_error; /* should finish now */
264 op += oSize;
265 if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; }
266 { U64 const crcDest = XXH64(decodedBuffer, op-ostart, 1);
267 if (crcDest != crcOrig) goto _output_error;
268 } }
269
270 { size_t oSize = 0;
271 size_t iSize = 0;
272 LZ4F_frameInfo_t fi;
273
274 DISPLAYLEVEL(3, "Start by feeding 0 bytes, to get next input size : ");
275 errorCode = LZ4F_decompress(dCtx, NULL, &oSize, ip, &iSize, NULL);
276 if (LZ4F_isError(errorCode)) goto _output_error;
277 DISPLAYLEVEL(3, " %u \n", (unsigned)errorCode);
278
279 DISPLAYLEVEL(3, "get FrameInfo on null input : ");
280 errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
281 if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
282 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
283
284 DISPLAYLEVEL(3, "get FrameInfo on not enough input : ");
285 iSize = 6;
286 errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
287 if (errorCode != (size_t)-LZ4F_ERROR_frameHeader_incomplete) goto _output_error;
288 DISPLAYLEVEL(3, " correctly failed : %s \n", LZ4F_getErrorName(errorCode));
289 ip += iSize;
290
291 DISPLAYLEVEL(3, "get FrameInfo on enough input : ");
292 iSize = 15 - iSize;
293 errorCode = LZ4F_getFrameInfo(dCtx, &fi, ip, &iSize);
294 if (LZ4F_isError(errorCode)) goto _output_error;
295 DISPLAYLEVEL(3, " correctly decoded \n");
296 ip += iSize;
297 }
298
299 DISPLAYLEVEL(3, "Byte after byte : \n");
300 { BYTE* const ostart = (BYTE*)decodedBuffer;
301 BYTE* op = ostart;
302 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
303 while (ip < iend) {
304 size_t oSize = oend-op;
305 size_t iSize = 1;
306 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
307 if (LZ4F_isError(errorCode)) goto _output_error;
308 op += oSize;
309 ip += iSize;
310 }
311 { U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
312 if (crcDest != crcOrig) goto _output_error; }
313 DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), COMPRESSIBLE_NOISE_LENGTH);
314 }
315
316 errorCode = LZ4F_freeDecompressionContext(dCtx);
317 if (LZ4F_isError(errorCode)) goto _output_error;
318 dCtx = NULL;
319 }
320
321 DISPLAYLEVEL(3, "Using 64 KB block : \n");
322 prefs.frameInfo.blockSizeID = LZ4F_max64KB;
323 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
324 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
325 if (LZ4F_isError(cSize)) goto _output_error;
326 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
327
328 DISPLAYLEVEL(3, "without checksum : \n");
329 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
330 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
331 if (LZ4F_isError(cSize)) goto _output_error;
332 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
333
334 DISPLAYLEVEL(3, "Using 256 KB block : \n");
335 prefs.frameInfo.blockSizeID = LZ4F_max256KB;
336 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
337 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
338 if (LZ4F_isError(cSize)) goto _output_error;
339 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
340
341 DISPLAYLEVEL(3, "Decompression test : \n");
342 { size_t const decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
343 unsigned const maxBits = FUZ_highbit((U32)decodedBufferSize);
344 BYTE* const ostart = (BYTE*)decodedBuffer;
345 BYTE* op = ostart;
346 BYTE* const oend = ostart + COMPRESSIBLE_NOISE_LENGTH;
347 const BYTE* ip = (const BYTE*)compressedBuffer;
348 const BYTE* const iend = (const BYTE*)compressedBuffer + cSize;
349
350 { LZ4F_errorCode_t const createError = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
351 if (LZ4F_isError(createError)) goto _output_error; }
352
353 DISPLAYLEVEL(3, "random segment sizes : \n");
354 while (ip < iend) {
355 unsigned const nbBits = FUZ_rand(&randState) % maxBits;
356 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
357 size_t oSize = oend-op;
358 if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
359 { size_t const decompressError = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
360 if (LZ4F_isError(decompressError)) goto _output_error; }
361 op += oSize;
362 ip += iSize;
363 }
364 { size_t const decodedSize = (size_t)(op - ostart);
365 U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
366 if (crcDest != crcOrig) goto _output_error;
367 DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
368 }
369
370 { LZ4F_errorCode_t const freeError = LZ4F_freeDecompressionContext(dCtx);
371 if (LZ4F_isError(freeError)) goto _output_error; }
372 dCtx = NULL;
373 }
374
375 DISPLAYLEVEL(3, "without checksum : \n");
376 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
377 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
378 if (LZ4F_isError(cSize)) goto _output_error;
379 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
380
381 DISPLAYLEVEL(3, "Using 1 MB block : \n");
382 prefs.frameInfo.blockSizeID = LZ4F_max1MB;
383 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
384 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
385 if (LZ4F_isError(cSize)) goto _output_error;
386 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
387
388 DISPLAYLEVEL(3, "without checksum : \n");
389 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
390 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(testSize, &prefs), CNBuffer, testSize, &prefs);
391 if (LZ4F_isError(cSize)) goto _output_error;
392 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)cSize);
393
394 DISPLAYLEVEL(3, "Using 4 MB block : \n");
395 prefs.frameInfo.blockSizeID = LZ4F_max4MB;
396 prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
397 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
398 DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity)
399 cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs);
400 if (LZ4F_isError(cSize)) goto _output_error;
401 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
402 }
403
404 DISPLAYLEVEL(3, "without checksum : \n");
405 prefs.frameInfo.contentChecksumFlag = LZ4F_noContentChecksum;
406 { size_t const dstCapacity = LZ4F_compressFrameBound(testSize, &prefs);
407 DISPLAYLEVEL(4, "dstCapacity = %u \n", (U32)dstCapacity)
408 cSize = LZ4F_compressFrame(compressedBuffer, dstCapacity, CNBuffer, testSize, &prefs);
409 if (LZ4F_isError(cSize)) goto _output_error;
410 DISPLAYLEVEL(3, "Compressed %u bytes into a %u bytes frame \n", (U32)testSize, (U32)cSize);
411 }
412
413 { size_t errorCode;
414 BYTE* const ostart = (BYTE*)compressedBuffer;
415 BYTE* op = ostart;
416 errorCode = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
417 if (LZ4F_isError(errorCode)) goto _output_error;
418
419 DISPLAYLEVEL(3, "compress without frameSize : \n");
420 memset(&(prefs.frameInfo), 0, sizeof(prefs.frameInfo));
421 errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
422 if (LZ4F_isError(errorCode)) goto _output_error;
423 op += errorCode;
424 errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
425 if (LZ4F_isError(errorCode)) goto _output_error;
426 op += errorCode;
427 errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
428 if (LZ4F_isError(errorCode)) goto _output_error;
429 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
430
431 DISPLAYLEVEL(3, "compress with frameSize : \n");
432 prefs.frameInfo.contentSize = testSize;
433 op = ostart;
434 errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
435 if (LZ4F_isError(errorCode)) goto _output_error;
436 op += errorCode;
437 errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
438 if (LZ4F_isError(errorCode)) goto _output_error;
439 op += errorCode;
440 errorCode = LZ4F_compressEnd(cctx, compressedBuffer, testSize, NULL);
441 if (LZ4F_isError(errorCode)) goto _output_error;
442 DISPLAYLEVEL(3, "Compressed %i bytes into a %i bytes frame \n", (int)testSize, (int)(op-ostart));
443
444 DISPLAYLEVEL(3, "compress with wrong frameSize : \n");
445 prefs.frameInfo.contentSize = testSize+1;
446 op = ostart;
447 errorCode = LZ4F_compressBegin(cctx, compressedBuffer, testSize, &prefs);
448 if (LZ4F_isError(errorCode)) goto _output_error;
449 op += errorCode;
450 errorCode = LZ4F_compressUpdate(cctx, op, LZ4F_compressBound(testSize, &prefs), CNBuffer, testSize, NULL);
451 if (LZ4F_isError(errorCode)) goto _output_error;
452 op += errorCode;
453 errorCode = LZ4F_compressEnd(cctx, op, testSize, NULL);
454 if (LZ4F_isError(errorCode)) { DISPLAYLEVEL(3, "Error correctly detected : %s \n", LZ4F_getErrorName(errorCode)); }
455 else
456 goto _output_error;
457
458 errorCode = LZ4F_freeCompressionContext(cctx);
459 if (LZ4F_isError(errorCode)) goto _output_error;
460 cctx = NULL;
461 }
462
463 DISPLAYLEVEL(3, "Skippable frame test : \n");
464 { size_t decodedBufferSize = COMPRESSIBLE_NOISE_LENGTH;
465 unsigned maxBits = FUZ_highbit((U32)decodedBufferSize);
466 BYTE* op = (BYTE*)decodedBuffer;
467 BYTE* const oend = (BYTE*)decodedBuffer + COMPRESSIBLE_NOISE_LENGTH;
468 BYTE* ip = (BYTE*)compressedBuffer;
469 BYTE* iend = (BYTE*)compressedBuffer + cSize + 8;
470
471 LZ4F_errorCode_t errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
472 if (LZ4F_isError(errorCode)) goto _output_error;
473
474 /* generate skippable frame */
475 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START);
476 FUZ_writeLE32(ip+4, (U32)cSize);
477
478 DISPLAYLEVEL(3, "random segment sizes : \n");
479 while (ip < iend) {
480 unsigned nbBits = FUZ_rand(&randState) % maxBits;
481 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
482 size_t oSize = oend-op;
483 if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
484 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
485 if (LZ4F_isError(errorCode)) goto _output_error;
486 op += oSize;
487 ip += iSize;
488 }
489 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)decodedBufferSize);
490
491 /* generate zero-size skippable frame */
492 DISPLAYLEVEL(3, "zero-size skippable frame\n");
493 ip = (BYTE*)compressedBuffer;
494 op = (BYTE*)decodedBuffer;
495 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+1);
496 FUZ_writeLE32(ip+4, 0);
497 iend = ip+8;
498
499 while (ip < iend) {
500 unsigned nbBits = FUZ_rand(&randState) % maxBits;
501 size_t iSize = (FUZ_rand(&randState) & ((1<<nbBits)-1)) + 1;
502 size_t oSize = oend-op;
503 if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
504 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
505 if (LZ4F_isError(errorCode)) goto _output_error;
506 op += oSize;
507 ip += iSize;
508 }
509 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
510
511 DISPLAYLEVEL(3, "Skippable frame header complete in first call \n");
512 ip = (BYTE*)compressedBuffer;
513 op = (BYTE*)decodedBuffer;
514 FUZ_writeLE32(ip, LZ4F_MAGIC_SKIPPABLE_START+2);
515 FUZ_writeLE32(ip+4, 10);
516 iend = ip+18;
517 while (ip < iend) {
518 size_t iSize = 10;
519 size_t oSize = 10;
520 if (iSize > (size_t)(iend-ip)) iSize = iend-ip;
521 errorCode = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, NULL);
522 if (LZ4F_isError(errorCode)) goto _output_error;
523 op += oSize;
524 ip += iSize;
525 }
526 DISPLAYLEVEL(3, "Skipped %i bytes \n", (int)(ip - (BYTE*)compressedBuffer - 8));
527 }
528
529 DISPLAY("Basic tests completed \n");
530 _end:
531 free(CNBuffer);
532 free(compressedBuffer);
533 free(decodedBuffer);
534 LZ4F_freeDecompressionContext(dCtx); dCtx = NULL;
535 LZ4F_freeCompressionContext(cctx); cctx = NULL;
536 return testResult;
537
538 _output_error:
539 testResult = 1;
540 DISPLAY("Error detected ! \n");
541 goto _end;
542 }
543
544
locateBuffDiff(const void * buff1,const void * buff2,size_t size,unsigned nonContiguous)545 static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, unsigned nonContiguous)
546 {
547 int p=0;
548 const BYTE* b1=(const BYTE*)buff1;
549 const BYTE* b2=(const BYTE*)buff2;
550 if (nonContiguous) {
551 DISPLAY("Non-contiguous output test (%i bytes)\n", (int)size);
552 return;
553 }
554 while (b1[p]==b2[p]) p++;
555 DISPLAY("Error at pos %i/%i : %02X != %02X \n", p, (int)size, b1[p], b2[p]);
556 }
557
558
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,double compressibility,U32 duration_s)559 int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, U32 duration_s)
560 {
561 unsigned testResult = 0;
562 unsigned testNb = 0;
563 size_t const srcDataLength = 9 MB; /* needs to be > 2x4MB to test large blocks */
564 void* srcBuffer = NULL;
565 size_t const compressedBufferSize = LZ4F_compressFrameBound(srcDataLength, NULL);
566 void* compressedBuffer = NULL;
567 void* decodedBuffer = NULL;
568 U32 coreRand = seed;
569 LZ4F_decompressionContext_t dCtx = NULL;
570 LZ4F_compressionContext_t cCtx = NULL;
571 size_t result;
572 clock_t const startClock = clock();
573 clock_t const clockDuration = duration_s * CLOCKS_PER_SEC;
574 # define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
575 DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
576
577 /* Create buffers */
578 result = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
579 CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
580 result = LZ4F_createCompressionContext(&cCtx, LZ4F_VERSION);
581 CHECK(LZ4F_isError(result), "Allocation failed (error %i)", (int)result);
582 srcBuffer = malloc(srcDataLength);
583 CHECK(srcBuffer==NULL, "srcBuffer Allocation failed");
584 compressedBuffer = malloc(compressedBufferSize);
585 CHECK(compressedBuffer==NULL, "compressedBuffer Allocation failed");
586 decodedBuffer = calloc(1, srcDataLength); /* calloc avoids decodedBuffer being considered "garbage" by scan-build */
587 CHECK(decodedBuffer==NULL, "decodedBuffer Allocation failed");
588 FUZ_fillCompressibleNoiseBuffer(srcBuffer, srcDataLength, compressibility, &coreRand);
589
590 /* jump to requested testNb */
591 for (testNb =0; (testNb < startTest); testNb++) (void)FUZ_rand(&coreRand); /* sync randomizer */
592
593 /* main fuzzer test loop */
594 for ( ; (testNb < nbTests) || (clockDuration > FUZ_GetClockSpan(startClock)) ; testNb++) {
595 U32 randState = coreRand ^ prime1;
596 unsigned const srcBits = (FUZ_rand(&randState) % (FUZ_highbit((U32)(srcDataLength-1)) - 1)) + 1;
597 size_t const srcSize = (FUZ_rand(&randState) & ((1<<srcBits)-1)) + 1;
598 size_t const srcStartId = FUZ_rand(&randState) % (srcDataLength - srcSize);
599 const BYTE* const srcStart = (const BYTE*)srcBuffer + srcStartId;
600 unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1;
601 U64 const crcOrig = XXH64(srcStart, srcSize, 1);
602 LZ4F_preferences_t prefs;
603 const LZ4F_preferences_t* prefsPtr = &prefs;
604 size_t cSize;
605
606 (void)FUZ_rand(&coreRand); /* update seed */
607 memset(&prefs, 0, sizeof(prefs));
608 prefs.frameInfo.blockMode = (LZ4F_blockMode_t)(FUZ_rand(&randState) & 1);
609 prefs.frameInfo.blockSizeID = (LZ4F_blockSizeID_t)(4 + (FUZ_rand(&randState) & 3));
610 prefs.frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)(FUZ_rand(&randState) & 1);
611 prefs.frameInfo.contentSize = ((FUZ_rand(&randState) & 0xF) == 1) ? srcSize : 0;
612 prefs.autoFlush = neverFlush ? 0 : (FUZ_rand(&randState) & 7) == 2;
613 prefs.compressionLevel = FUZ_rand(&randState) % 5;
614 if ((FUZ_rand(&randState) & 0xF) == 1) prefsPtr = NULL;
615
616 DISPLAYUPDATE(2, "\r%5u ", testNb);
617
618 if ((FUZ_rand(&randState) & 0xFFF) == 0) {
619 /* create a skippable frame (rare case) */
620 BYTE* op = (BYTE*)compressedBuffer;
621 FUZ_writeLE32(op, LZ4F_MAGIC_SKIPPABLE_START + (FUZ_rand(&randState) & 15));
622 FUZ_writeLE32(op+4, (U32)srcSize);
623 cSize = srcSize+8;
624 } else if ((FUZ_rand(&randState) & 0xF) == 2) { /* single pass compression (simple) */
625 cSize = LZ4F_compressFrame(compressedBuffer, LZ4F_compressFrameBound(srcSize, prefsPtr), srcStart, srcSize, prefsPtr);
626 CHECK(LZ4F_isError(cSize), "LZ4F_compressFrame failed : error %i (%s)", (int)cSize, LZ4F_getErrorName(cSize));
627 } else { /* multi-segments compression */
628 const BYTE* ip = srcStart;
629 const BYTE* const iend = srcStart + srcSize;
630 BYTE* op = (BYTE*)compressedBuffer;
631 BYTE* const oend = op + (neverFlush ? LZ4F_compressFrameBound(srcSize, prefsPtr) : compressedBufferSize); /* when flushes are possible, can't guarantee a max compressed size */
632 unsigned const maxBits = FUZ_highbit((U32)srcSize);
633 LZ4F_compressOptions_t cOptions;
634 memset(&cOptions, 0, sizeof(cOptions));
635 result = LZ4F_compressBegin(cCtx, op, oend-op, prefsPtr);
636 CHECK(LZ4F_isError(result), "Compression header failed (error %i)", (int)result);
637 op += result;
638 while (ip < iend) {
639 unsigned const nbBitsSeg = FUZ_rand(&randState) % maxBits;
640 size_t const sampleMax = (FUZ_rand(&randState) & ((1<<nbBitsSeg)-1)) + 1;
641 size_t const iSize = MIN(sampleMax, (size_t)(iend-ip));
642 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
643 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
644
645 result = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
646 CHECK(LZ4F_isError(result), "Compression failed (error %i : %s)", (int)result, LZ4F_getErrorName(result));
647 op += result;
648 ip += iSize;
649
650 { unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1);
651 if (forceFlush) {
652 result = LZ4F_flush(cCtx, op, oend-op, &cOptions);
653 CHECK(LZ4F_isError(result), "Compression failed (error %i)", (int)result);
654 op += result;
655 } }
656 }
657 CHECK(op>=oend, "LZ4F_compressFrameBound overflow");
658 result = LZ4F_compressEnd(cCtx, op, oend-op, &cOptions);
659 CHECK(LZ4F_isError(result), "Compression completion failed (error %i : %s)", (int)result, LZ4F_getErrorName(result));
660 op += result;
661 cSize = op-(BYTE*)compressedBuffer;
662 DISPLAYLEVEL(5, "\nCompressed %u bytes into %u \n", (U32)srcSize, (U32)cSize);
663 }
664
665 /* multi-segments decompression */
666 { const BYTE* ip = (const BYTE*)compressedBuffer;
667 const BYTE* const iend = ip + cSize;
668 BYTE* op = (BYTE*)decodedBuffer;
669 BYTE* const oend = op + srcDataLength;
670 unsigned const suggestedBits = FUZ_highbit((U32)cSize);
671 unsigned const maxBits = MAX(3, suggestedBits);
672 unsigned const nonContiguousDst = FUZ_rand(&randState) % 3; /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */
673 size_t totalOut = 0;
674 XXH64_state_t xxh64;
675 XXH64_reset(&xxh64, 1);
676 while (ip < iend) {
677 unsigned const nbBitsI = (FUZ_rand(&randState) % (maxBits-1)) + 1;
678 unsigned const nbBitsO = (FUZ_rand(&randState) % (maxBits)) + 1;
679 size_t const iSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsI)-1)) + 1;
680 size_t iSize = MIN(iSizeMax, (size_t)(iend-ip));
681 size_t const oSizeMax = (FUZ_rand(&randState) & ((1<<nbBitsO)-1)) + 2;
682 size_t oSize = MIN(oSizeMax, (size_t)(oend-op));
683 LZ4F_decompressOptions_t dOptions;
684 memset(&dOptions, 0, sizeof(dOptions));
685 dOptions.stableDst = FUZ_rand(&randState) & 1;
686 if (nonContiguousDst==2) dOptions.stableDst = 0; /* overwrite mode */
687 result = LZ4F_decompress(dCtx, op, &oSize, ip, &iSize, &dOptions);
688 if (LZ4F_getErrorCode(result) == LZ4F_ERROR_contentChecksum_invalid) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
689 CHECK(LZ4F_isError(result), "Decompression failed (error %i:%s)", (int)result, LZ4F_getErrorName(result));
690 XXH64_update(&xxh64, op, (U32)oSize);
691 totalOut += oSize;
692 op += oSize;
693 ip += iSize;
694 op += nonContiguousDst;
695 if (nonContiguousDst==2) op = (BYTE*)decodedBuffer; /* overwritten destination */
696 }
697 CHECK(result != 0, "Frame decompression failed (error %i)", (int)result);
698 if (totalOut) { /* otherwise, it's a skippable frame */
699 U64 const crcDecoded = XXH64_digest(&xxh64);
700 if (crcDecoded != crcOrig) locateBuffDiff(srcStart, decodedBuffer, srcSize, nonContiguousDst);
701 CHECK(crcDecoded != crcOrig, "Decompression corruption");
702 }
703 }
704 }
705
706 DISPLAYLEVEL(2, "\rAll tests completed \n");
707
708 _end:
709 LZ4F_freeDecompressionContext(dCtx);
710 LZ4F_freeCompressionContext(cCtx);
711 free(srcBuffer);
712 free(compressedBuffer);
713 free(decodedBuffer);
714
715 if (use_pause) {
716 DISPLAY("press enter to finish \n");
717 (void)getchar();
718 }
719 return testResult;
720
721 _output_error:
722 testResult = 1;
723 goto _end;
724 }
725
726
FUZ_usage(const char * programName)727 int FUZ_usage(const char* programName)
728 {
729 DISPLAY( "Usage :\n");
730 DISPLAY( " %s [args]\n", programName);
731 DISPLAY( "\n");
732 DISPLAY( "Arguments :\n");
733 DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
734 DISPLAY( " -T# : Duration of tests, in seconds (default: use Nb of tests) \n");
735 DISPLAY( " -s# : Select seed (default:prompt user)\n");
736 DISPLAY( " -t# : Select starting test number (default:0)\n");
737 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
738 DISPLAY( " -v : verbose\n");
739 DISPLAY( " -h : display help and exit\n");
740 return 0;
741 }
742
743
main(int argc,const char ** argv)744 int main(int argc, const char** argv)
745 {
746 U32 seed=0;
747 int seedset=0;
748 int argNb;
749 int nbTests = nbTestsDefault;
750 int testNb = 0;
751 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
752 int result=0;
753 U32 duration=0;
754 const char* const programName = argv[0];
755
756 /* Check command line */
757 for (argNb=1; argNb<argc; argNb++) {
758 const char* argument = argv[argNb];
759
760 if(!argument) continue; /* Protection if argument empty */
761
762 /* Decode command (note : aggregated short commands are allowed) */
763 if (argument[0]=='-') {
764 if (!strcmp(argument, "--no-prompt")) {
765 no_prompt=1;
766 seedset=1;
767 displayLevel=1;
768 continue;
769 }
770 argument++;
771
772 while (*argument!=0) {
773 switch(*argument)
774 {
775 case 'h':
776 return FUZ_usage(programName);
777 case 'v':
778 argument++;
779 displayLevel++;
780 break;
781 case 'q':
782 argument++;
783 displayLevel--;
784 break;
785 case 'p': /* pause at the end */
786 argument++;
787 use_pause = 1;
788 break;
789
790 case 'i':
791 argument++;
792 nbTests=0; duration=0;
793 while ((*argument>='0') && (*argument<='9')) {
794 nbTests *= 10;
795 nbTests += *argument - '0';
796 argument++;
797 }
798 break;
799
800 case 'T':
801 argument++;
802 nbTests = 0; duration = 0;
803 for (;;) {
804 switch(*argument)
805 {
806 case 'm': duration *= 60; argument++; continue;
807 case 's':
808 case 'n': argument++; continue;
809 case '0':
810 case '1':
811 case '2':
812 case '3':
813 case '4':
814 case '5':
815 case '6':
816 case '7':
817 case '8':
818 case '9': duration *= 10; duration += *argument++ - '0'; continue;
819 }
820 break;
821 }
822 break;
823
824 case 's':
825 argument++;
826 seed=0;
827 seedset=1;
828 while ((*argument>='0') && (*argument<='9')) {
829 seed *= 10;
830 seed += *argument - '0';
831 argument++;
832 }
833 break;
834 case 't':
835 argument++;
836 testNb=0;
837 while ((*argument>='0') && (*argument<='9')) {
838 testNb *= 10;
839 testNb += *argument - '0';
840 argument++;
841 }
842 break;
843 case 'P': /* compressibility % */
844 argument++;
845 proba=0;
846 while ((*argument>='0') && (*argument<='9')) {
847 proba *= 10;
848 proba += *argument - '0';
849 argument++;
850 }
851 if (proba<0) proba=0;
852 if (proba>100) proba=100;
853 break;
854 default:
855 ;
856 return FUZ_usage(programName);
857 }
858 }
859 }
860 }
861
862 /* Get Seed */
863 DISPLAY("Starting lz4frame tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING);
864
865 if (!seedset) {
866 time_t const t = time(NULL);
867 U32 const h = XXH32(&t, sizeof(t), 1);
868 seed = h % 10000;
869 }
870 DISPLAY("Seed = %u\n", seed);
871 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
872
873 if (nbTests<=0) nbTests=1;
874
875 if (testNb==0) result = basicTests(seed, ((double)proba) / 100);
876 if (result) return 1;
877 return fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, duration);
878 }
879