1 /* LZ4frame API example : compress a file
2 * Modified from an example code by Zbigniew Jędrzejewski-Szmek
3 *
4 * This example streams an input file into an output file
5 * using a bounded memory budget.
6 * Input is read in chunks of IN_CHUNK_SIZE */
7
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <errno.h>
12 #include <assert.h>
13
14 #include <getopt.h>
15 #include <lz4frame.h>
16 #include <lz4frame_static.h>
17
18 #define IN_CHUNK_SIZE (16*1024)
19
20 static const LZ4F_preferences_t kPrefs = {
21 { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame,
22 0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum },
23 0, /* compression level; 0 == default */
24 0, /* autoflush */
25 0, /* favor decompression speed */
26 { 0, 0, 0 }, /* reserved, must be set to 0 */
27 };
28
29
30 /* safe_fwrite() :
31 * performs fwrite(), ensure operation success, or immediately exit() */
safe_fwrite(void * buf,size_t eltSize,size_t nbElt,FILE * f)32 static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f)
33 {
34 size_t const writtenSize = fwrite(buf, eltSize, nbElt, f);
35 size_t const expectedSize = eltSize * nbElt;
36 if (nbElt>0) assert(expectedSize / nbElt == eltSize); /* check overflow */
37 if (writtenSize < expectedSize) {
38 if (ferror(f)) /* note : ferror() must follow fwrite */
39 fprintf(stderr, "Write failed \n");
40 else
41 fprintf(stderr, "Write too short \n");
42 exit(1);
43 }
44 }
45
46
47 /* ================================================= */
48 /* Streaming Compression example */
49 /* ================================================= */
50
51 typedef struct {
52 int error;
53 unsigned long long size_in;
54 unsigned long long size_out;
55 } compressResult_t;
56
57 static compressResult_t
compress_file_internal(FILE * f_in,FILE * f_out,LZ4F_compressionContext_t ctx,void * inBuff,size_t inChunkSize,void * outBuff,size_t outCapacity,FILE * f_unc,long uncOffset)58 compress_file_internal(FILE* f_in, FILE* f_out,
59 LZ4F_compressionContext_t ctx,
60 void* inBuff, size_t inChunkSize,
61 void* outBuff, size_t outCapacity,
62 FILE* f_unc, long uncOffset)
63 {
64 compressResult_t result = { 1, 0, 0 }; /* result for an error */
65 long long count_in = 0, count_out, bytesToOffset = -1;
66
67 assert(f_in != NULL); assert(f_out != NULL);
68 assert(ctx != NULL);
69 assert(outCapacity >= LZ4F_HEADER_SIZE_MAX);
70 assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs));
71
72 /* write frame header */
73 { size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs);
74 if (LZ4F_isError(headerSize)) {
75 printf("Failed to start compression: error %u \n", (unsigned)headerSize);
76 return result;
77 }
78 count_out = headerSize;
79 printf("Buffer size is %u bytes, header size %u bytes \n",
80 (unsigned)outCapacity, (unsigned)headerSize);
81 safe_fwrite(outBuff, 1, headerSize, f_out);
82 }
83
84 /* stream file */
85 for (;;) {
86 size_t compressedSize;
87 long long inSize = IN_CHUNK_SIZE;
88 if (uncOffset >= 0) {
89 bytesToOffset = uncOffset - count_in;
90
91 /* read only remaining bytes to offset position */
92 if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) {
93 inSize = bytesToOffset;
94 }
95 }
96
97 /* input data is at uncompressed data offset */
98 if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) {
99 size_t const readSize = fread(inBuff, 1, inSize, f_unc);
100 if (readSize == 0) {
101 uncOffset = -1;
102 continue;
103 }
104 count_in += readSize;
105 compressedSize = LZ4F_uncompressedUpdate(ctx,
106 outBuff, outCapacity,
107 inBuff, readSize,
108 NULL);
109 } else {
110 size_t const readSize = fread(inBuff, 1, inSize, f_in);
111 if (readSize == 0) break; /* nothing left to read from input file */
112 count_in += readSize;
113 compressedSize = LZ4F_compressUpdate(ctx,
114 outBuff, outCapacity,
115 inBuff, readSize,
116 NULL);
117
118 }
119
120 if (LZ4F_isError(compressedSize)) {
121 printf("Compression failed: error %u \n", (unsigned)compressedSize);
122 return result;
123 }
124
125 printf("Writing %u bytes\n", (unsigned)compressedSize);
126 safe_fwrite(outBuff, 1, compressedSize, f_out);
127 count_out += compressedSize;
128 }
129
130 /* flush whatever remains within internal buffers */
131 { size_t const compressedSize = LZ4F_compressEnd(ctx,
132 outBuff, outCapacity,
133 NULL);
134 if (LZ4F_isError(compressedSize)) {
135 printf("Failed to end compression: error %u \n", (unsigned)compressedSize);
136 return result;
137 }
138
139 printf("Writing %u bytes \n", (unsigned)compressedSize);
140 safe_fwrite(outBuff, 1, compressedSize, f_out);
141 count_out += compressedSize;
142 }
143
144 result.size_in = count_in;
145 result.size_out = count_out;
146 result.error = 0;
147 return result;
148 }
149
150 static compressResult_t
compress_file(FILE * f_in,FILE * f_out,FILE * f_unc,int uncOffset)151 compress_file(FILE* f_in, FILE* f_out,
152 FILE* f_unc, int uncOffset)
153 {
154 assert(f_in != NULL);
155 assert(f_out != NULL);
156
157 /* resource allocation */
158 LZ4F_compressionContext_t ctx;
159 size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
160 void* const src = malloc(IN_CHUNK_SIZE);
161 size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs); /* large enough for any input <= IN_CHUNK_SIZE */
162 void* const outbuff = malloc(outbufCapacity);
163
164 compressResult_t result = { 1, 0, 0 }; /* == error (default) */
165 if (!LZ4F_isError(ctxCreation) && src && outbuff) {
166 result = compress_file_internal(f_in, f_out,
167 ctx,
168 src, IN_CHUNK_SIZE,
169 outbuff, outbufCapacity,
170 f_unc, uncOffset);
171 } else {
172 printf("error : resource allocation failed \n");
173 }
174
175 LZ4F_freeCompressionContext(ctx); /* supports free on NULL */
176 free(src);
177 free(outbuff);
178 return result;
179 }
180
181
182 /* ================================================= */
183 /* Streaming decompression example */
184 /* ================================================= */
185
get_block_size(const LZ4F_frameInfo_t * info)186 static size_t get_block_size(const LZ4F_frameInfo_t* info) {
187 switch (info->blockSizeID) {
188 case LZ4F_default:
189 case LZ4F_max64KB: return 1 << 16;
190 case LZ4F_max256KB: return 1 << 18;
191 case LZ4F_max1MB: return 1 << 20;
192 case LZ4F_max4MB: return 1 << 22;
193 default:
194 printf("Impossible with expected frame specification (<=v1.6.1)\n");
195 exit(1);
196 }
197 }
198
199 /* @return : 1==error, 0==success */
200 static int
decompress_file_internal(FILE * f_in,FILE * f_out,LZ4F_dctx * dctx,void * src,size_t srcCapacity,size_t filled,size_t alreadyConsumed,void * dst,size_t dstCapacity)201 decompress_file_internal(FILE* f_in, FILE* f_out,
202 LZ4F_dctx* dctx,
203 void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed,
204 void* dst, size_t dstCapacity)
205 {
206 int firstChunk = 1;
207 size_t ret = 1;
208
209 assert(f_in != NULL); assert(f_out != NULL);
210 assert(dctx != NULL);
211 assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled);
212 assert(dst != NULL); assert(dstCapacity > 0);
213
214 /* Decompression */
215 while (ret != 0) {
216 /* Load more input */
217 size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0;
218 const void* srcPtr = (const char*)src + alreadyConsumed; alreadyConsumed=0;
219 const void* const srcEnd = (const char*)srcPtr + readSize;
220 if (readSize == 0 || ferror(f_in)) {
221 printf("Decompress: not enough input or error reading file\n");
222 return 1;
223 }
224
225 /* Decompress:
226 * Continue while there is more input to read (srcPtr != srcEnd)
227 * and the frame isn't over (ret != 0)
228 */
229 while (srcPtr < srcEnd && ret != 0) {
230 /* Any data within dst has been flushed at this stage */
231 size_t dstSize = dstCapacity;
232 size_t srcSize = (const char*)srcEnd - (const char*)srcPtr;
233 ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL);
234 if (LZ4F_isError(ret)) {
235 printf("Decompression error: %s\n", LZ4F_getErrorName(ret));
236 return 1;
237 }
238 /* Flush output */
239 if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out);
240 /* Update input */
241 srcPtr = (const char*)srcPtr + srcSize;
242 }
243
244 assert(srcPtr <= srcEnd);
245
246 /* Ensure all input data has been consumed.
247 * It is valid to have multiple frames in the same file,
248 * but this example only supports one frame.
249 */
250 if (srcPtr < srcEnd) {
251 printf("Decompress: Trailing data left in file after frame\n");
252 return 1;
253 }
254 }
255
256 /* Check that there isn't trailing data in the file after the frame.
257 * It is valid to have multiple frames in the same file,
258 * but this example only supports one frame.
259 */
260 { size_t const readSize = fread(src, 1, 1, f_in);
261 if (readSize != 0 || !feof(f_in)) {
262 printf("Decompress: Trailing data left in file after frame\n");
263 return 1;
264 } }
265
266 return 0;
267 }
268
269
270 /* @return : 1==error, 0==completed */
271 static int
decompress_file_allocDst(FILE * f_in,FILE * f_out,LZ4F_dctx * dctx,void * src,size_t srcCapacity)272 decompress_file_allocDst(FILE* f_in, FILE* f_out,
273 LZ4F_dctx* dctx,
274 void* src, size_t srcCapacity)
275 {
276 assert(f_in != NULL); assert(f_out != NULL);
277 assert(dctx != NULL);
278 assert(src != NULL);
279 assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX); /* ensure LZ4F_getFrameInfo() can read enough data */
280
281 /* Read Frame header */
282 size_t const readSize = fread(src, 1, srcCapacity, f_in);
283 if (readSize == 0 || ferror(f_in)) {
284 printf("Decompress: not enough input or error reading file\n");
285 return 1;
286 }
287
288 LZ4F_frameInfo_t info;
289 size_t consumedSize = readSize;
290 { size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize);
291 if (LZ4F_isError(fires)) {
292 printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires));
293 return 1;
294 } }
295
296 /* Allocating enough space for an entire block isn't necessary for
297 * correctness, but it allows some memcpy's to be elided.
298 */
299 size_t const dstCapacity = get_block_size(&info);
300 void* const dst = malloc(dstCapacity);
301 if (!dst) { perror("decompress_file(dst)"); return 1; }
302
303 int const decompressionResult = decompress_file_internal(
304 f_in, f_out,
305 dctx,
306 src, srcCapacity, readSize-consumedSize, consumedSize,
307 dst, dstCapacity);
308
309 free(dst);
310 return decompressionResult;
311 }
312
313
314 /* @result : 1==error, 0==success */
decompress_file(FILE * f_in,FILE * f_out)315 static int decompress_file(FILE* f_in, FILE* f_out)
316 {
317 assert(f_in != NULL); assert(f_out != NULL);
318
319 /* Resource allocation */
320 void* const src = malloc(IN_CHUNK_SIZE);
321 if (!src) { perror("decompress_file(src)"); return 1; }
322
323 LZ4F_dctx* dctx;
324 { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
325 if (LZ4F_isError(dctxStatus)) {
326 printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus));
327 } }
328
329 int const result = !dctx ? 1 /* error */ :
330 decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE);
331
332 free(src);
333 LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */
334 return result;
335 }
336
337
compareFiles(FILE * fp0,FILE * fp1,FILE * fpUnc,long uncOffset)338 int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset)
339 {
340 int result = 0;
341 long bytesRead = 0;
342 long bytesToOffset = -1;
343 long b1Size = 1024;
344
345 while (result==0) {
346 char b1[b1Size];
347 size_t r1;
348 size_t bytesToRead = sizeof b1;
349 if (uncOffset >= 0) {
350 bytesToOffset = uncOffset - bytesRead;
351
352 /* read remainder to offset */
353 if (bytesToOffset < b1Size) {
354 bytesToRead = bytesToOffset;
355 }
356 }
357
358 char b0[1024];
359 size_t r0;
360 if (bytesToOffset <= 0 && fpUnc) {
361 bytesToRead = sizeof b1;
362 r0 = fread(b0, 1,bytesToRead, fpUnc);
363 } else {
364 r0 = fread(b0, 1, bytesToRead, fp0);
365 }
366
367 r1 = fread(b1, 1, r0, fp1);
368
369 result = (r0 != r1);
370 if (!r0 || !r1) break;
371 if (!result) result = memcmp(b0, b1, r0);
372
373 bytesRead += r1;
374 }
375
376 return result;
377 }
378
379
main(int argc,char ** argv)380 int main(int argc, char **argv) {
381 char inpFilename[256] = { 0 };
382 char lz4Filename[256] = { 0 };
383 char decFilename[256] = { 0 };
384
385 int uncOffset = -1;
386 char uncFilename[256] = { 0 };
387 int opt;
388
389 if (argc < 2) {
390 printf("Please specify input filename\n");
391 return EXIT_FAILURE;
392 }
393
394 snprintf(inpFilename, 256, "%s", argv[1]);
395 snprintf(lz4Filename, 256, "%s.lz4", argv[1]);
396 snprintf(decFilename, 256, "%s.lz4.dec", argv[1]);
397
398 while ((opt = getopt(argc, argv, "o:d:")) != -1) {
399 switch (opt) {
400 case 'd':
401 snprintf(uncFilename, 256, "%s", optarg);
402 break;
403 case 'o':
404 uncOffset = atoi(optarg);
405 break;
406 default:
407 printf("usage: %s <input file> [-o <offset> -d <file>]\n", argv[0]);
408 printf("-o uncompressed data offset\n");
409 printf(" inject uncompressed data at this offset into the lz4 file\n");
410 printf("-d uncompressed file\n");
411 printf(" file to inject without compression into the lz4 file\n");
412 return EXIT_FAILURE;
413 }
414 }
415
416 printf("inp = [%s]\n", inpFilename);
417 printf("lz4 = [%s]\n", lz4Filename);
418 printf("dec = [%s]\n", decFilename);
419 if (uncOffset > 0) {
420 printf("unc = [%s]\n", uncFilename);
421 printf("ofs = [%i]\n", uncOffset);
422 }
423
424 /* compress */
425 { FILE* const inpFp = fopen(inpFilename, "rb");
426 FILE* const outFp = fopen(lz4Filename, "wb");
427 FILE* const uncFp = fopen(uncFilename, "rb");
428
429 printf("compress : %s -> %s\n", inpFilename, lz4Filename);
430 compressResult_t const ret = compress_file(
431 inpFp, outFp,
432 uncFp, uncOffset);
433
434 fclose(outFp);
435 fclose(inpFp);
436 if (uncFp)
437 fclose(uncFp);
438
439 if (ret.error) {
440 printf("compress : failed with code %i\n", ret.error);
441 return ret.error;
442 }
443 printf("%s: %zu → %zu bytes, %.1f%%\n",
444 inpFilename,
445 (size_t)ret.size_in, (size_t)ret.size_out, /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */
446 (double)ret.size_out / ret.size_in * 100);
447 printf("compress : done\n");
448 }
449
450 /* decompress */
451 { FILE* const inpFp = fopen(lz4Filename, "rb");
452 FILE* const outFp = fopen(decFilename, "wb");
453
454 printf("decompress : %s -> %s\n", lz4Filename, decFilename);
455 int const ret = decompress_file(inpFp, outFp);
456
457 fclose(outFp);
458 fclose(inpFp);
459
460 if (ret) {
461 printf("decompress : failed with code %i\n", ret);
462 return ret;
463 }
464 printf("decompress : done\n");
465 }
466
467 /* verify */
468 { FILE* const inpFp = fopen(inpFilename, "rb");
469 FILE* const decFp = fopen(decFilename, "rb");
470 FILE* const uncFp = fopen(uncFilename, "rb");
471
472 printf("verify : %s <-> %s\n", inpFilename, decFilename);
473 int const cmp = compareFiles(inpFp, decFp,
474 uncFp, uncOffset);
475
476 fclose(decFp);
477 fclose(inpFp);
478 if (uncFp)
479 fclose(uncFp);
480
481 if (cmp) {
482 printf("corruption detected : decompressed file differs from original\n");
483 return cmp;
484 }
485 printf("verify : OK\n");
486 }
487
488 return 0;
489 }
490