1 // LZ4 streaming API example : line-by-line logfile compression
2 // Copyright : Takayuki Matsuoka
3
4
5 #define _CRT_SECURE_NO_WARNINGS // for MSVC
6 #include "lz4.h"
7
8 #include <stdio.h>
9 #include <stdint.h>
10 #include <stdlib.h>
11 #include <string.h>
12
write_uint16(FILE * fp,uint16_t i)13 static size_t write_uint16(FILE* fp, uint16_t i)
14 {
15 return fwrite(&i, sizeof(i), 1, fp);
16 }
17
write_bin(FILE * fp,const void * array,int arrayBytes)18 static size_t write_bin(FILE* fp, const void* array, int arrayBytes)
19 {
20 return fwrite(array, 1, arrayBytes, fp);
21 }
22
read_uint16(FILE * fp,uint16_t * i)23 static size_t read_uint16(FILE* fp, uint16_t* i)
24 {
25 return fread(i, sizeof(*i), 1, fp);
26 }
27
read_bin(FILE * fp,void * array,int arrayBytes)28 static size_t read_bin(FILE* fp, void* array, int arrayBytes)
29 {
30 return fread(array, 1, arrayBytes, fp);
31 }
32
33
test_compress(FILE * outFp,FILE * inpFp,size_t messageMaxBytes,size_t ringBufferBytes)34 static void test_compress(
35 FILE* outFp,
36 FILE* inpFp,
37 size_t messageMaxBytes,
38 size_t ringBufferBytes)
39 {
40 LZ4_stream_t* const lz4Stream = LZ4_createStream();
41 char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
42 char* const inpBuf = malloc(ringBufferBytes);
43 int inpOffset = 0;
44
45 for ( ; ; )
46 {
47 char* const inpPtr = &inpBuf[inpOffset];
48
49 #if 0
50 // Read random length data to the ring buffer.
51 const int randomLength = (rand() % messageMaxBytes) + 1;
52 const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
53 if (0 == inpBytes) break;
54 #else
55 // Read line to the ring buffer.
56 int inpBytes = 0;
57 if (!fgets(inpPtr, (int) messageMaxBytes, inpFp))
58 break;
59 inpBytes = (int) strlen(inpPtr);
60 #endif
61
62 {
63 const int cmpBytes = LZ4_compress_continue(
64 lz4Stream, inpPtr, cmpBuf, inpBytes);
65 if (cmpBytes <= 0) break;
66 write_uint16(outFp, (uint16_t) cmpBytes);
67 write_bin(outFp, cmpBuf, cmpBytes);
68
69 // Add and wraparound the ringbuffer offset
70 inpOffset += inpBytes;
71 if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0;
72 }
73 }
74 write_uint16(outFp, 0);
75
76 free(inpBuf);
77 free(cmpBuf);
78 LZ4_freeStream(lz4Stream);
79 }
80
81
test_decompress(FILE * outFp,FILE * inpFp,size_t messageMaxBytes,size_t ringBufferBytes)82 static void test_decompress(
83 FILE* outFp,
84 FILE* inpFp,
85 size_t messageMaxBytes,
86 size_t ringBufferBytes)
87 {
88 LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode();
89 char* const cmpBuf = malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
90 char* const decBuf = malloc(ringBufferBytes);
91 int decOffset = 0;
92
93 for ( ; ; )
94 {
95 uint16_t cmpBytes = 0;
96
97 if (read_uint16(inpFp, &cmpBytes) != 1) break;
98 if (cmpBytes <= 0) break;
99 if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break;
100
101 {
102 char* const decPtr = &decBuf[decOffset];
103 const int decBytes = LZ4_decompress_safe_continue(
104 lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes);
105 if (decBytes <= 0) break;
106 write_bin(outFp, decPtr, decBytes);
107
108 // Add and wraparound the ringbuffer offset
109 decOffset += decBytes;
110 if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0;
111 }
112 }
113
114 free(decBuf);
115 free(cmpBuf);
116 LZ4_freeStreamDecode(lz4StreamDecode);
117 }
118
119
compare(FILE * f0,FILE * f1)120 static int compare(FILE* f0, FILE* f1)
121 {
122 int result = 0;
123 const size_t tempBufferBytes = 65536;
124 char* const b0 = malloc(tempBufferBytes);
125 char* const b1 = malloc(tempBufferBytes);
126
127 while(0 == result)
128 {
129 const size_t r0 = fread(b0, 1, tempBufferBytes, f0);
130 const size_t r1 = fread(b1, 1, tempBufferBytes, f1);
131
132 result = (int) r0 - (int) r1;
133
134 if (0 == r0 || 0 == r1) break;
135 if (0 == result) result = memcmp(b0, b1, r0);
136 }
137
138 free(b1);
139 free(b0);
140 return result;
141 }
142
143
main(int argc,char * argv[])144 int main(int argc, char* argv[])
145 {
146 enum {
147 MESSAGE_MAX_BYTES = 1024,
148 RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES,
149 };
150
151 char inpFilename[256] = { 0 };
152 char lz4Filename[256] = { 0 };
153 char decFilename[256] = { 0 };
154
155 if (argc < 2)
156 {
157 printf("Please specify input filename\n");
158 return 0;
159 }
160
161 snprintf(inpFilename, 256, "%s", argv[1]);
162 snprintf(lz4Filename, 256, "%s.lz4s", argv[1]);
163 snprintf(decFilename, 256, "%s.lz4s.dec", argv[1]);
164
165 printf("inp = [%s]\n", inpFilename);
166 printf("lz4 = [%s]\n", lz4Filename);
167 printf("dec = [%s]\n", decFilename);
168
169 // compress
170 {
171 FILE* inpFp = fopen(inpFilename, "rb");
172 FILE* outFp = fopen(lz4Filename, "wb");
173
174 test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
175
176 fclose(outFp);
177 fclose(inpFp);
178 }
179
180 // decompress
181 {
182 FILE* inpFp = fopen(lz4Filename, "rb");
183 FILE* outFp = fopen(decFilename, "wb");
184
185 test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
186
187 fclose(outFp);
188 fclose(inpFp);
189 }
190
191 // verify
192 {
193 FILE* inpFp = fopen(inpFilename, "rb");
194 FILE* decFp = fopen(decFilename, "rb");
195
196 const int cmp = compare(inpFp, decFp);
197 if (0 == cmp)
198 printf("Verify : OK\n");
199 else
200 printf("Verify : NG\n");
201
202 fclose(decFp);
203 fclose(inpFp);
204 }
205
206 return 0;
207 }
208