1 // LZ4 streaming API example : line-by-line logfile compression
2 // Copyright : Takayuki Matsuoka
3
4
5 #ifdef _MSC_VER /* Visual Studio */
6 # define _CRT_SECURE_NO_WARNINGS
7 # define snprintf sprintf_s
8 #endif
9 #include "lz4.h"
10
11 #include <stdio.h>
12 #include <stdint.h>
13 #include <stdlib.h>
14 #include <string.h>
15
write_uint16(FILE * fp,uint16_t i)16 static size_t write_uint16(FILE* fp, uint16_t i)
17 {
18 return fwrite(&i, sizeof(i), 1, fp);
19 }
20
write_bin(FILE * fp,const void * array,int arrayBytes)21 static size_t write_bin(FILE* fp, const void* array, int arrayBytes)
22 {
23 return fwrite(array, 1, arrayBytes, fp);
24 }
25
read_uint16(FILE * fp,uint16_t * i)26 static size_t read_uint16(FILE* fp, uint16_t* i)
27 {
28 return fread(i, sizeof(*i), 1, fp);
29 }
30
read_bin(FILE * fp,void * array,int arrayBytes)31 static size_t read_bin(FILE* fp, void* array, int arrayBytes)
32 {
33 return fread(array, 1, arrayBytes, fp);
34 }
35
36
test_compress(FILE * outFp,FILE * inpFp,size_t messageMaxBytes,size_t ringBufferBytes)37 static void test_compress(
38 FILE* outFp,
39 FILE* inpFp,
40 size_t messageMaxBytes,
41 size_t ringBufferBytes)
42 {
43 LZ4_stream_t* const lz4Stream = LZ4_createStream();
44 const size_t cmpBufBytes = LZ4_COMPRESSBOUND(messageMaxBytes);
45 char* const cmpBuf = (char*) malloc(cmpBufBytes);
46 char* const inpBuf = (char*) malloc(ringBufferBytes);
47 int inpOffset = 0;
48
49 for ( ; ; )
50 {
51 char* const inpPtr = &inpBuf[inpOffset];
52
53 #if 0
54 // Read random length data to the ring buffer.
55 const int randomLength = (rand() % messageMaxBytes) + 1;
56 const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
57 if (0 == inpBytes) break;
58 #else
59 // Read line to the ring buffer.
60 int inpBytes = 0;
61 if (!fgets(inpPtr, (int) messageMaxBytes, inpFp))
62 break;
63 inpBytes = (int) strlen(inpPtr);
64 #endif
65
66 {
67 const int cmpBytes = LZ4_compress_fast_continue(
68 lz4Stream, inpPtr, cmpBuf, inpBytes, cmpBufBytes, 1);
69 if (cmpBytes <= 0) break;
70 write_uint16(outFp, (uint16_t) cmpBytes);
71 write_bin(outFp, cmpBuf, cmpBytes);
72
73 // Add and wraparound the ringbuffer offset
74 inpOffset += inpBytes;
75 if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0;
76 }
77 }
78 write_uint16(outFp, 0);
79
80 free(inpBuf);
81 free(cmpBuf);
82 LZ4_freeStream(lz4Stream);
83 }
84
85
test_decompress(FILE * outFp,FILE * inpFp,size_t messageMaxBytes,size_t ringBufferBytes)86 static void test_decompress(
87 FILE* outFp,
88 FILE* inpFp,
89 size_t messageMaxBytes,
90 size_t ringBufferBytes)
91 {
92 LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode();
93 char* const cmpBuf = (char*) malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
94 char* const decBuf = (char*) malloc(ringBufferBytes);
95 int decOffset = 0;
96
97 for ( ; ; )
98 {
99 uint16_t cmpBytes = 0;
100
101 if (read_uint16(inpFp, &cmpBytes) != 1) break;
102 if (cmpBytes <= 0) break;
103 if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break;
104
105 {
106 char* const decPtr = &decBuf[decOffset];
107 const int decBytes = LZ4_decompress_safe_continue(
108 lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes);
109 if (decBytes <= 0) break;
110 write_bin(outFp, decPtr, decBytes);
111
112 // Add and wraparound the ringbuffer offset
113 decOffset += decBytes;
114 if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0;
115 }
116 }
117
118 free(decBuf);
119 free(cmpBuf);
120 LZ4_freeStreamDecode(lz4StreamDecode);
121 }
122
123
compare(FILE * f0,FILE * f1)124 static int compare(FILE* f0, FILE* f1)
125 {
126 int result = 0;
127 const size_t tempBufferBytes = 65536;
128 char* const b0 = (char*) malloc(tempBufferBytes);
129 char* const b1 = (char*) malloc(tempBufferBytes);
130
131 while(0 == result)
132 {
133 const size_t r0 = fread(b0, 1, tempBufferBytes, f0);
134 const size_t r1 = fread(b1, 1, tempBufferBytes, f1);
135
136 result = (int) r0 - (int) r1;
137
138 if (0 == r0 || 0 == r1) break;
139 if (0 == result) result = memcmp(b0, b1, r0);
140 }
141
142 free(b1);
143 free(b0);
144 return result;
145 }
146
147
main(int argc,char * argv[])148 int main(int argc, char* argv[])
149 {
150 enum {
151 MESSAGE_MAX_BYTES = 1024,
152 RING_BUFFER_BYTES = 1024 * 256 + MESSAGE_MAX_BYTES,
153 };
154
155 char inpFilename[256] = { 0 };
156 char lz4Filename[256] = { 0 };
157 char decFilename[256] = { 0 };
158
159 if (argc < 2)
160 {
161 printf("Please specify input filename\n");
162 return 0;
163 }
164
165 snprintf(inpFilename, 256, "%s", argv[1]);
166 snprintf(lz4Filename, 256, "%s.lz4s", argv[1]);
167 snprintf(decFilename, 256, "%s.lz4s.dec", argv[1]);
168
169 printf("inp = [%s]\n", inpFilename);
170 printf("lz4 = [%s]\n", lz4Filename);
171 printf("dec = [%s]\n", decFilename);
172
173 // compress
174 {
175 FILE* inpFp = fopen(inpFilename, "rb");
176 FILE* outFp = fopen(lz4Filename, "wb");
177
178 test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
179
180 fclose(outFp);
181 fclose(inpFp);
182 }
183
184 // decompress
185 {
186 FILE* inpFp = fopen(lz4Filename, "rb");
187 FILE* outFp = fopen(decFilename, "wb");
188
189 test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
190
191 fclose(outFp);
192 fclose(inpFp);
193 }
194
195 // verify
196 {
197 FILE* inpFp = fopen(inpFilename, "rb");
198 FILE* decFp = fopen(decFilename, "rb");
199
200 const int cmp = compare(inpFp, decFp);
201 if (0 == cmp)
202 printf("Verify : OK\n");
203 else
204 printf("Verify : NG\n");
205
206 fclose(decFp);
207 fclose(inpFp);
208 }
209
210 return 0;
211 }
212