1 /*
2 * Copyright (C) 2005 Johannes E. Schindelin. All Rights Reserved.
3 *
4 * This is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This software is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this software; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
17 * USA.
18 */
19
20 #ifdef LIBVNCSERVER_HAVE_LIBZ
21
22 /*
23 * zrle.c - handle zrle encoding.
24 *
25 * This file shouldn't be compiled directly. It is included multiple times by
26 * rfbproto.c, each time with a different definition of the macro BPP. For
27 * each value of BPP, this file defines a function which handles an zrle
28 * encoded rectangle with BPP bits per pixel.
29 */
30
31 #ifndef REALBPP
32 #define REALBPP BPP
33 #endif
34
35 #if !defined(UNCOMP) || UNCOMP==0
36 #define HandleZRLE CONCAT2E(HandleZRLE,REALBPP)
37 #define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP)
38 #elif UNCOMP>0
39 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Down)
40 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Down)
41 #else
42 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Up)
43 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up)
44 #endif
45 #define CARDBPP CONCAT3E(uint,BPP,_t)
46 #define CARDREALBPP CONCAT3E(uint,REALBPP,_t)
47
48 #define ENDIAN_LITTLE 0
49 #define ENDIAN_BIG 1
50 #define ENDIAN_NO 2
51 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
52 #undef END_FIX
53 #if ZYWRLE_ENDIAN == ENDIAN_LITTLE
54 # define END_FIX LE
55 #elif ZYWRLE_ENDIAN == ENDIAN_BIG
56 # define END_FIX BE
57 #else
58 # define END_FIX NE
59 #endif
60 #define __RFB_CONCAT3E(a,b,c) CONCAT3E(a,b,c)
61 #define __RFB_CONCAT2E(a,b) CONCAT2E(a,b)
62 #undef CPIXEL
63 #if REALBPP != BPP
64 #if UNCOMP == 0
65 #define CPIXEL REALBPP
66 #elif UNCOMP>0
67 #define CPIXEL CONCAT2E(REALBPP,Down)
68 #else
69 #define CPIXEL CONCAT2E(REALBPP,Up)
70 #endif
71 #endif
72 #define PIXEL_T __RFB_CONCAT3E(uint,BPP,_t)
73 #if BPP!=8
74 #define ZYWRLE_DECODE 1
75 #include "zywrletemplate.c"
76 #endif
77 #undef CPIXEL
78
79 static int HandleZRLETile(rfbClient* client,
80 uint8_t* buffer,size_t buffer_length,
81 int x,int y,int w,int h);
82
83 static rfbBool
HandleZRLE(rfbClient * client,int rx,int ry,int rw,int rh)84 HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh)
85 {
86 rfbZRLEHeader header;
87 int remaining;
88 int inflateResult;
89 int toRead;
90 int min_buffer_size = rw * rh * (REALBPP / 8) * 2;
91
92 /* First make sure we have a large enough raw buffer to hold the
93 * decompressed data. In practice, with a fixed REALBPP, fixed frame
94 * buffer size and the first update containing the entire frame
95 * buffer, this buffer allocation should only happen once, on the
96 * first update.
97 */
98 if ( client->raw_buffer_size < min_buffer_size) {
99
100 if ( client->raw_buffer != NULL ) {
101
102 free( client->raw_buffer );
103
104 }
105
106 client->raw_buffer_size = min_buffer_size;
107 client->raw_buffer = (char*) malloc( client->raw_buffer_size );
108
109 }
110
111 if (!ReadFromRFBServer(client, (char *)&header, sz_rfbZRLEHeader))
112 return FALSE;
113
114 remaining = rfbClientSwap32IfLE(header.length);
115
116 /* Need to initialize the decompressor state. */
117 client->decompStream.next_in = ( Bytef * )client->buffer;
118 client->decompStream.avail_in = 0;
119 client->decompStream.next_out = ( Bytef * )client->raw_buffer;
120 client->decompStream.avail_out = client->raw_buffer_size;
121 client->decompStream.data_type = Z_BINARY;
122
123 /* Initialize the decompression stream structures on the first invocation. */
124 if ( client->decompStreamInited == FALSE ) {
125
126 inflateResult = inflateInit( &client->decompStream );
127
128 if ( inflateResult != Z_OK ) {
129 rfbClientLog(
130 "inflateInit returned error: %d, msg: %s\n",
131 inflateResult,
132 client->decompStream.msg);
133 return FALSE;
134 }
135
136 client->decompStreamInited = TRUE;
137
138 }
139
140 inflateResult = Z_OK;
141
142 /* Process buffer full of data until no more to process, or
143 * some type of inflater error, or Z_STREAM_END.
144 */
145 while (( remaining > 0 ) &&
146 ( inflateResult == Z_OK )) {
147
148 if ( remaining > RFB_BUFFER_SIZE ) {
149 toRead = RFB_BUFFER_SIZE;
150 }
151 else {
152 toRead = remaining;
153 }
154
155 /* Fill the buffer, obtaining data from the server. */
156 if (!ReadFromRFBServer(client, client->buffer,toRead))
157 return FALSE;
158
159 client->decompStream.next_in = ( Bytef * )client->buffer;
160 client->decompStream.avail_in = toRead;
161
162 /* Need to uncompress buffer full. */
163 inflateResult = inflate( &client->decompStream, Z_SYNC_FLUSH );
164
165 /* We never supply a dictionary for compression. */
166 if ( inflateResult == Z_NEED_DICT ) {
167 rfbClientLog("zlib inflate needs a dictionary!\n");
168 return FALSE;
169 }
170 if ( inflateResult < 0 ) {
171 rfbClientLog(
172 "zlib inflate returned error: %d, msg: %s\n",
173 inflateResult,
174 client->decompStream.msg);
175 return FALSE;
176 }
177
178 /* Result buffer allocated to be at least large enough. We should
179 * never run out of space!
180 */
181 if (( client->decompStream.avail_in > 0 ) &&
182 ( client->decompStream.avail_out <= 0 )) {
183 rfbClientLog("zlib inflate ran out of space!\n");
184 return FALSE;
185 }
186
187 remaining -= toRead;
188
189 } /* while ( remaining > 0 ) */
190
191 if ( inflateResult == Z_OK ) {
192 char* buf=client->raw_buffer;
193 int i,j;
194
195 remaining = client->raw_buffer_size-client->decompStream.avail_out;
196
197 for(j=0; j<rh; j+=rfbZRLETileHeight)
198 for(i=0; i<rw; i+=rfbZRLETileWidth) {
199 int subWidth=(i+rfbZRLETileWidth>rw)?rw-i:rfbZRLETileWidth;
200 int subHeight=(j+rfbZRLETileHeight>rh)?rh-j:rfbZRLETileHeight;
201 int result=HandleZRLETile(client,buf,remaining,rx+i,ry+j,subWidth,subHeight);
202
203 if(result<0) {
204 rfbClientLog("ZRLE decoding failed (%d)\n",result);
205 return TRUE;
206 return FALSE;
207 }
208
209 buf+=result;
210 remaining-=result;
211 }
212 }
213 else {
214
215 rfbClientLog(
216 "zlib inflate returned error: %d, msg: %s\n",
217 inflateResult,
218 client->decompStream.msg);
219 return FALSE;
220
221 }
222
223 return TRUE;
224 }
225
226 #if REALBPP!=BPP && defined(UNCOMP) && UNCOMP!=0
227 #if UNCOMP>0
228 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)>>UNCOMP)
229 #else
230 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)<<(-(UNCOMP)))
231 #endif
232 #else
233 #define UncompressCPixel(pointer) (*(CARDBPP*)pointer)
234 #endif
235
HandleZRLETile(rfbClient * client,uint8_t * buffer,size_t buffer_length,int x,int y,int w,int h)236 static int HandleZRLETile(rfbClient* client,
237 uint8_t* buffer,size_t buffer_length,
238 int x,int y,int w,int h) {
239 uint8_t* buffer_copy = buffer;
240 uint8_t* buffer_end = buffer+buffer_length;
241 uint8_t type;
242 #if BPP!=8
243 uint8_t zywrle_level = (client->appData.qualityLevel & 0x80) ?
244 0 : (3 - client->appData.qualityLevel / 3);
245 #endif
246
247 if(buffer_length<1)
248 return -2;
249
250 type = *buffer;
251 buffer++;
252 {
253 if( type == 0 ) /* raw */
254 #if BPP!=8
255 if( zywrle_level > 0 ){
256 CARDBPP* pFrame = (CARDBPP*)client->frameBuffer + y*client->width+x;
257 int ret;
258 client->appData.qualityLevel |= 0x80;
259 ret = HandleZRLETile(client, buffer, buffer_end-buffer, x, y, w, h);
260 client->appData.qualityLevel &= 0x7F;
261 if( ret < 0 ){
262 return ret;
263 }
264 ZYWRLE_SYNTHESIZE( pFrame, pFrame, w, h, client->width, zywrle_level, (int*)client->zlib_buffer );
265 buffer += ret;
266 }else
267 #endif
268 {
269 #if REALBPP!=BPP
270 int i,j;
271
272 if(1+w*h*REALBPP/8>buffer_length) {
273 rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h);
274 return -3;
275 }
276
277 for(j=y*client->width; j<(y+h)*client->width; j+=client->width)
278 for(i=x; i<x+w; i++,buffer+=REALBPP/8)
279 ((CARDBPP*)client->frameBuffer)[j+i] = UncompressCPixel(buffer);
280 #else
281 CopyRectangle(client, buffer, x, y, w, h);
282 buffer+=w*h*REALBPP/8;
283 #endif
284 }
285 else if( type == 1 ) /* solid */
286 {
287 CARDBPP color = UncompressCPixel(buffer);
288
289 if(1+REALBPP/8>buffer_length)
290 return -4;
291
292 FillRectangle(client, x, y, w, h, color);
293
294 buffer+=REALBPP/8;
295
296 }
297 else if( (type >= 2)&&(type <= 127) ) /* packed Palette */
298 {
299 CARDBPP palette[16];
300 int i,j,shift,
301 bpp=(type>4?(type>16?8:4):(type>2?2:1)),
302 mask=(1<<bpp)-1,
303 divider=(8/bpp);
304
305 if(1+type*REALBPP/8+((w+divider-1)/divider)*h>buffer_length)
306 return -5;
307
308 /* read palette */
309 for(i=0; i<type; i++,buffer+=REALBPP/8)
310 palette[i] = UncompressCPixel(buffer);
311
312 /* read palettized pixels */
313 for(j=y*client->width; j<(y+h)*client->width; j+=client->width) {
314 for(i=x,shift=8-bpp; i<x+w; i++) {
315 ((CARDBPP*)client->frameBuffer)[j+i] = palette[((*buffer)>>shift)&mask];
316 shift-=bpp;
317 if(shift<0) {
318 shift=8-bpp;
319 buffer++;
320 }
321 }
322 if(shift<8-bpp)
323 buffer++;
324 }
325
326 }
327 /* case 17 ... 127: not used, but valid */
328 else if( type == 128 ) /* plain RLE */
329 {
330 int i=0,j=0;
331 while(j<h) {
332 int color,length;
333 /* read color */
334 if(buffer+REALBPP/8+1>buffer_end)
335 return -7;
336 color = UncompressCPixel(buffer);
337 buffer+=REALBPP/8;
338 /* read run length */
339 length=1;
340 while(*buffer==0xff) {
341 if(buffer+1>=buffer_end)
342 return -8;
343 length+=*buffer;
344 buffer++;
345 }
346 length+=*buffer;
347 buffer++;
348 while(j<h && length>0) {
349 ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
350 length--;
351 i++;
352 if(i>=w) {
353 i=0;
354 j++;
355 }
356 }
357 if(length>0)
358 rfbClientLog("Warning: possible ZRLE corruption\n");
359 }
360
361 }
362 else if( type == 129 ) /* unused */
363 {
364 return -8;
365 }
366 else if( type >= 130 ) /* palette RLE */
367 {
368 CARDBPP palette[128];
369 int i,j;
370
371 if(2+(type-128)*REALBPP/8>buffer_length)
372 return -9;
373
374 /* read palette */
375 for(i=0; i<type-128; i++,buffer+=REALBPP/8)
376 palette[i] = UncompressCPixel(buffer);
377 /* read palettized pixels */
378 i=j=0;
379 while(j<h) {
380 int color,length;
381 /* read color */
382 if(buffer>=buffer_end)
383 return -10;
384 color = palette[(*buffer)&0x7f];
385 length=1;
386 if(*buffer&0x80) {
387 if(buffer+1>=buffer_end)
388 return -11;
389 buffer++;
390 /* read run length */
391 while(*buffer==0xff) {
392 if(buffer+1>=buffer_end)
393 return -8;
394 length+=*buffer;
395 buffer++;
396 }
397 length+=*buffer;
398 }
399 buffer++;
400 while(j<h && length>0) {
401 ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
402 length--;
403 i++;
404 if(i>=w) {
405 i=0;
406 j++;
407 }
408 }
409 if(length>0)
410 rfbClientLog("Warning: possible ZRLE corruption\n");
411 }
412 }
413 }
414
415 return buffer-buffer_copy;
416 }
417
418 #undef CARDBPP
419 #undef CARDREALBPP
420 #undef HandleZRLE
421 #undef HandleZRLETile
422 #undef UncompressCPixel
423 #undef REALBPP
424
425 #endif
426
427 #undef UNCOMP
428