Line 0
Link Here
|
|
|
1 |
/* bug fix on Robert Jakabosky from alphatrade.com's lighttp 1.4.10 mod_deflate path |
2 |
* |
3 |
* new module option: |
4 |
* deflate.nocompress-url = "^/nocompressurl/" # pcre regex which don't compress |
5 |
* |
6 |
* Bug fix and new features: |
7 |
* 1) fix loop bug when content-length is bigger than work-block-size*k |
8 |
* 2) prevent compress on buggy http 1.0 client with Accept Encoding: gzip, deflate |
9 |
* 3) fix bug with chunk transfer encoding (under mod_fastcgi+php environment) |
10 |
* |
11 |
* deflate.sync-flush = "enable" is buggy on chunk encoding transfer. Use it carefully, |
12 |
*/ |
13 |
#include <sys/types.h> |
14 |
#include <sys/stat.h> |
15 |
|
16 |
#include <fcntl.h> |
17 |
#include <unistd.h> |
18 |
#include <ctype.h> |
19 |
#include <stdlib.h> |
20 |
#include <string.h> |
21 |
#include <errno.h> |
22 |
#include <time.h> |
23 |
#include <assert.h> |
24 |
|
25 |
#if defined(HAVE_PCRE_H) |
26 |
#include <pcre.h> |
27 |
#endif |
28 |
|
29 |
#include "base.h" |
30 |
#include "log.h" |
31 |
#include "buffer.h" |
32 |
#include "response.h" |
33 |
#include "joblist.h" |
34 |
#include "stat_cache.h" |
35 |
|
36 |
#include "plugin.h" |
37 |
|
38 |
#include "crc32.h" |
39 |
#include "etag.h" |
40 |
#include "inet_ntop_cache.h" |
41 |
|
42 |
#if defined HAVE_ZLIB_H && defined HAVE_LIBZ |
43 |
# define USE_ZLIB |
44 |
# include <zlib.h> |
45 |
#else |
46 |
# define Z_DEFAULT_COMPRESSION 1 |
47 |
#endif |
48 |
|
49 |
#if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2 |
50 |
# define USE_BZ2LIB |
51 |
/* we don't need stdio interface */ |
52 |
# define BZ_NO_STDIO |
53 |
# include <bzlib.h> |
54 |
#endif |
55 |
|
56 |
#include "sys-mmap.h" |
57 |
|
58 |
/* request: accept-encoding */ |
59 |
#define HTTP_ACCEPT_ENCODING_IDENTITY BV(0) |
60 |
#define HTTP_ACCEPT_ENCODING_GZIP BV(1) |
61 |
#define HTTP_ACCEPT_ENCODING_DEFLATE BV(2) |
62 |
#define HTTP_ACCEPT_ENCODING_COMPRESS BV(3) |
63 |
#define HTTP_ACCEPT_ENCODING_BZIP2 BV(4) |
64 |
|
65 |
#define KByte * 1024 |
66 |
#define MByte * 1024 KByte |
67 |
#define GByte * 1024 MByte |
68 |
|
69 |
typedef struct { |
70 |
unsigned short debug; |
71 |
unsigned short enabled; |
72 |
unsigned short bzip2; |
73 |
unsigned short sync_flush; |
74 |
unsigned short output_buffer_size; |
75 |
unsigned short min_compress_size; |
76 |
unsigned short work_block_size; |
77 |
short mem_level; |
78 |
short compression_level; |
79 |
short window_size; |
80 |
array *mimetypes; |
81 |
buffer *nocompress_url; |
82 |
#if defined(HAVE_PCRE_H) |
83 |
pcre *nocompress_regex; |
84 |
#endif |
85 |
} plugin_config; |
86 |
|
87 |
typedef struct { |
88 |
PLUGIN_DATA; |
89 |
buffer *tmp_buf; |
90 |
|
91 |
plugin_config **config_storage; |
92 |
plugin_config conf; |
93 |
} plugin_data; |
94 |
|
95 |
typedef struct { |
96 |
int bytes_in; |
97 |
int bytes_out; |
98 |
chunkqueue *in_queue; |
99 |
buffer *output; |
100 |
/* compression type & state */ |
101 |
int compression_type; |
102 |
int stream_open; |
103 |
#ifdef USE_ZLIB |
104 |
unsigned long crc; |
105 |
z_stream z; |
106 |
unsigned short gzip_header; |
107 |
#endif |
108 |
#ifdef USE_BZ2LIB |
109 |
bz_stream bz; |
110 |
#endif |
111 |
plugin_data *plugin_data; |
112 |
} handler_ctx; |
113 |
|
114 |
static handler_ctx *handler_ctx_init() { |
115 |
handler_ctx *hctx; |
116 |
|
117 |
hctx = calloc(1, sizeof(*hctx)); |
118 |
hctx->in_queue = chunkqueue_init(); |
119 |
|
120 |
return hctx; |
121 |
} |
122 |
|
123 |
static void handler_ctx_free(handler_ctx *hctx) { |
124 |
chunkqueue_free(hctx->in_queue); |
125 |
free(hctx); |
126 |
} |
127 |
|
128 |
INIT_FUNC(mod_deflate_init) { |
129 |
plugin_data *p; |
130 |
|
131 |
p = calloc(1, sizeof(*p)); |
132 |
|
133 |
p->tmp_buf = buffer_init(); |
134 |
|
135 |
return p; |
136 |
} |
137 |
|
138 |
FREE_FUNC(mod_deflate_free) { |
139 |
plugin_data *p = p_d; |
140 |
|
141 |
UNUSED(srv); |
142 |
|
143 |
if (!p) return HANDLER_GO_ON; |
144 |
|
145 |
if (p->config_storage) { |
146 |
size_t i; |
147 |
for (i = 0; i < srv->config_context->used; i++) { |
148 |
plugin_config *s = p->config_storage[i]; |
149 |
|
150 |
if (!s) continue; |
151 |
|
152 |
array_free(s->mimetypes); |
153 |
buffer_free(s->nocompress_url); |
154 |
#if defined(HAVE_PCRE_H) |
155 |
if (s->nocompress_regex) pcre_free(s->nocompress_regex); |
156 |
#endif |
157 |
free(s); |
158 |
} |
159 |
free(p->config_storage); |
160 |
} |
161 |
|
162 |
buffer_free(p->tmp_buf); |
163 |
|
164 |
free(p); |
165 |
|
166 |
return HANDLER_GO_ON; |
167 |
} |
168 |
|
169 |
SETDEFAULTS_FUNC(mod_deflate_setdefaults) { |
170 |
plugin_data *p = p_d; |
171 |
size_t i = 0; |
172 |
|
173 |
config_values_t cv[] = { |
174 |
{ "deflate.output-buffer-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, |
175 |
{ "deflate.mimetypes", NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION }, |
176 |
{ "deflate.compression-level", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, |
177 |
{ "deflate.mem-level", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, |
178 |
{ "deflate.window-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, |
179 |
{ "deflate.min-compress-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, |
180 |
{ "deflate.work-block-size", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, |
181 |
{ "deflate.enabled", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, |
182 |
{ "deflate.debug", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, |
183 |
{ "deflate.bzip2", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, |
184 |
{ "deflate.sync-flush", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, |
185 |
{ "deflate.nocompress-url", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION }, |
186 |
{ NULL, NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET } |
187 |
}; |
188 |
|
189 |
p->config_storage = calloc(1, srv->config_context->used * sizeof(specific_config *)); |
190 |
|
191 |
for (i = 0; i < srv->config_context->used; i++) { |
192 |
plugin_config *s; |
193 |
#if defined(HAVE_PCRE_H) |
194 |
const char *errptr; |
195 |
int erroff; |
196 |
#endif |
197 |
|
198 |
s = calloc(1, sizeof(plugin_config)); |
199 |
s->enabled = 1; |
200 |
s->bzip2 = 1; |
201 |
s->sync_flush = 0; |
202 |
s->debug = 0; |
203 |
s->output_buffer_size = 0; |
204 |
s->mem_level = 9; |
205 |
s->window_size = 15; |
206 |
s->min_compress_size = 0; |
207 |
s->work_block_size = 2048; |
208 |
s->compression_level = Z_DEFAULT_COMPRESSION; |
209 |
s->mimetypes = array_init(); |
210 |
s->nocompress_url = buffer_init(); |
211 |
#if defined(HAVE_PCRE_H) |
212 |
s->nocompress_regex = NULL; |
213 |
#endif |
214 |
|
215 |
cv[0].destination = &(s->output_buffer_size); |
216 |
cv[1].destination = s->mimetypes; |
217 |
cv[2].destination = &(s->compression_level); |
218 |
cv[3].destination = &(s->mem_level); |
219 |
cv[4].destination = &(s->window_size); |
220 |
cv[5].destination = &(s->min_compress_size); |
221 |
cv[6].destination = &(s->work_block_size); |
222 |
cv[7].destination = &(s->enabled); |
223 |
cv[8].destination = &(s->debug); |
224 |
cv[9].destination = &(s->bzip2); |
225 |
cv[10].destination = &(s->sync_flush); |
226 |
cv[11].destination = s->nocompress_url; |
227 |
|
228 |
p->config_storage[i] = s; |
229 |
|
230 |
if (0 != config_insert_values_global(srv, ((data_config *)srv->config_context->data[i])->value, cv)) { |
231 |
return HANDLER_ERROR; |
232 |
} |
233 |
|
234 |
#if defined(HAVE_PCRE_H) |
235 |
if (!buffer_is_empty(s->nocompress_url)) { |
236 |
if (NULL == (s->nocompress_regex = pcre_compile(s->nocompress_url->ptr, |
237 |
0, &errptr, &erroff, NULL))) { |
238 |
|
239 |
log_error_write(srv, __FILE__, __LINE__, "sbss", |
240 |
"compiling regex for nocompress-url failed:", |
241 |
s->nocompress_url, "pos:", erroff); |
242 |
return HANDLER_ERROR; |
243 |
} |
244 |
} |
245 |
#endif |
246 |
if((s->compression_level < 1 || s->compression_level > 9) && |
247 |
s->compression_level != Z_DEFAULT_COMPRESSION) { |
248 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
249 |
"compression-level must be between 1 and 9:", s->compression_level); |
250 |
return HANDLER_ERROR; |
251 |
} |
252 |
|
253 |
if(s->mem_level < 1 || s->mem_level > 9) { |
254 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
255 |
"mem-level must be between 1 and 9:", s->mem_level); |
256 |
return HANDLER_ERROR; |
257 |
} |
258 |
|
259 |
if(s->window_size < 1 || s->window_size > 15) { |
260 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
261 |
"window-size must be between 1 and 15:", s->window_size); |
262 |
return HANDLER_ERROR; |
263 |
} |
264 |
s->window_size = 0 - s->window_size; |
265 |
|
266 |
if(s->sync_flush) { |
267 |
s->output_buffer_size = 0; |
268 |
} |
269 |
} |
270 |
|
271 |
return HANDLER_GO_ON; |
272 |
|
273 |
} |
274 |
|
275 |
#ifdef USE_ZLIB |
276 |
/* Copied gzip_header from apache 2.2's mod_deflate.c */ |
277 |
/* RFC 1952 Section 2.3 defines the gzip header: |
278 |
* |
279 |
* +---+---+---+---+---+---+---+---+---+---+ |
280 |
* |ID1|ID2|CM |FLG| MTIME |XFL|OS | |
281 |
* +---+---+---+---+---+---+---+---+---+---+ |
282 |
*/ |
283 |
static const char gzip_header[10] = |
284 |
{ '\037', '\213', Z_DEFLATED, 0, |
285 |
0, 0, 0, 0, /* mtime */ |
286 |
0, 0x03 /* Unix OS_CODE */ |
287 |
}; |
288 |
static int stream_deflate_init(server *srv, connection *con, handler_ctx *hctx) { |
289 |
plugin_data *p = hctx->plugin_data; |
290 |
z_stream *z; |
291 |
|
292 |
UNUSED(srv); |
293 |
UNUSED(con); |
294 |
|
295 |
z = &(hctx->z); |
296 |
z->zalloc = Z_NULL; |
297 |
z->zfree = Z_NULL; |
298 |
z->opaque = Z_NULL; |
299 |
z->total_in = 0; |
300 |
z->total_out = 0; |
301 |
z->next_out = NULL; |
302 |
z->avail_out = 0; |
303 |
|
304 |
if(p->conf.debug) { |
305 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
306 |
"output-buffer-size:", p->conf.output_buffer_size); |
307 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
308 |
"compression-level:", p->conf.compression_level); |
309 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
310 |
"mem-level:", p->conf.mem_level); |
311 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
312 |
"window-size:", p->conf.window_size); |
313 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
314 |
"min-compress-size:", p->conf.min_compress_size); |
315 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
316 |
"work-block-size:", p->conf.work_block_size); |
317 |
} |
318 |
if (Z_OK != deflateInit2(z, |
319 |
p->conf.compression_level, |
320 |
Z_DEFLATED, |
321 |
p->conf.window_size, /* supress zlib-header */ |
322 |
p->conf.mem_level, |
323 |
Z_DEFAULT_STRATEGY)) { |
324 |
return -1; |
325 |
} |
326 |
hctx->stream_open = 1; |
327 |
|
328 |
return 0; |
329 |
} |
330 |
|
331 |
static int stream_deflate_compress(server *srv, connection *con, handler_ctx *hctx, unsigned char *start, off_t st_size) { |
332 |
plugin_data *p = hctx->plugin_data; |
333 |
z_stream *z; |
334 |
int len; |
335 |
int in = 0, out = 0; |
336 |
|
337 |
UNUSED(srv); |
338 |
z = &(hctx->z); |
339 |
|
340 |
if(z->next_out == NULL) { |
341 |
z->next_out = (unsigned char *)hctx->output->ptr; |
342 |
z->avail_out = hctx->output->size; |
343 |
} |
344 |
|
345 |
if(hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) { |
346 |
if(hctx->gzip_header == 0) { |
347 |
hctx->gzip_header = 1; |
348 |
/* copy gzip header into output buffer */ |
349 |
buffer_copy_memory(hctx->output, gzip_header, sizeof(gzip_header)); |
350 |
if(p->conf.debug) { |
351 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
352 |
"gzip_header len=", sizeof(gzip_header)); |
353 |
} |
354 |
/* initialize crc32 */ |
355 |
hctx->crc = crc32(0L, Z_NULL, 0); |
356 |
z->next_out = (unsigned char *)(hctx->output->ptr + sizeof(gzip_header)); |
357 |
z->avail_out = hctx->output->size - sizeof(gzip_header); |
358 |
} |
359 |
hctx->crc = crc32(hctx->crc, start, st_size); |
360 |
} |
361 |
|
362 |
z->next_in = start; |
363 |
z->avail_in = st_size; |
364 |
hctx->bytes_in += st_size; |
365 |
|
366 |
/* compress data */ |
367 |
in = z->avail_in; |
368 |
do { |
369 |
if (Z_OK != deflate(z, Z_NO_FLUSH)) { |
370 |
deflateEnd(z); |
371 |
hctx->stream_open = 0; |
372 |
return -1; |
373 |
} |
374 |
|
375 |
if(z->avail_out == 0 || z->avail_in > 0) { |
376 |
len = hctx->output->size - z->avail_out; |
377 |
hctx->bytes_out += len; |
378 |
out += len; |
379 |
chunkqueue_append_mem(con->write_queue, hctx->output->ptr, len+1); |
380 |
z->next_out = (unsigned char *)hctx->output->ptr; |
381 |
z->avail_out = hctx->output->size; |
382 |
} |
383 |
} while (z->avail_in > 0); |
384 |
|
385 |
if(p->conf.debug) { |
386 |
log_error_write(srv, __FILE__, __LINE__, "sdsd", |
387 |
"compress: in=", in, ", out=", out); |
388 |
} |
389 |
return 0; |
390 |
} |
391 |
|
392 |
static int stream_deflate_flush(server *srv, connection *con, handler_ctx *hctx, int end) { |
393 |
plugin_data *p = hctx->plugin_data; |
394 |
z_stream *z; |
395 |
int len; |
396 |
int rc = 0; |
397 |
int done; |
398 |
int flush = 1; |
399 |
int in = 0, out = 0; |
400 |
|
401 |
UNUSED(srv); |
402 |
|
403 |
z = &(hctx->z); |
404 |
|
405 |
if(z->next_out == NULL) { |
406 |
z->next_out = (unsigned char *)hctx->output->ptr; |
407 |
z->avail_out = hctx->output->size; |
408 |
} |
409 |
/* compress data */ |
410 |
in = z->avail_in; |
411 |
do { |
412 |
done = 1; |
413 |
if(end) { |
414 |
rc = deflate(z, Z_FINISH); |
415 |
if (rc == Z_OK) { |
416 |
done = 0; |
417 |
} else if (rc != Z_STREAM_END) { |
418 |
deflateEnd(z); |
419 |
hctx->stream_open = 0; |
420 |
return -1; |
421 |
} |
422 |
} else { |
423 |
if(p->conf.sync_flush) { |
424 |
rc = deflate(z, Z_SYNC_FLUSH); |
425 |
} else if(z->avail_in > 0) { |
426 |
if(p->conf.output_buffer_size > 0) flush = 0; |
427 |
rc = deflate(z, Z_NO_FLUSH); |
428 |
} else { |
429 |
if(p->conf.output_buffer_size > 0) flush = 0; |
430 |
rc = Z_OK; |
431 |
} |
432 |
if (rc != Z_OK) { |
433 |
deflateEnd(z); |
434 |
hctx->stream_open = 0; |
435 |
return -1; |
436 |
} |
437 |
} |
438 |
|
439 |
len = hctx->output->size - z->avail_out; |
440 |
if(z->avail_out == 0 || (flush && len > 0)) { |
441 |
hctx->bytes_out += len; |
442 |
out += len; |
443 |
chunkqueue_append_mem(con->write_queue, hctx->output->ptr, len+1); |
444 |
z->next_out = (unsigned char *)hctx->output->ptr; |
445 |
z->avail_out = hctx->output->size; |
446 |
} |
447 |
} while (z->avail_in != 0 || !done); |
448 |
|
449 |
|
450 |
if(p->conf.debug) { |
451 |
log_error_write(srv, __FILE__, __LINE__, "sdsd", |
452 |
"flush: in=", in, ", out=", out); |
453 |
} |
454 |
if(p->conf.sync_flush) { |
455 |
z->next_out = NULL; |
456 |
z->avail_out = 0; |
457 |
} |
458 |
return 0; |
459 |
} |
460 |
|
461 |
static int stream_deflate_end(server *srv, connection *con, handler_ctx *hctx) { |
462 |
plugin_data *p = hctx->plugin_data; |
463 |
z_stream *z; |
464 |
int rc; |
465 |
|
466 |
UNUSED(srv); |
467 |
|
468 |
z = &(hctx->z); |
469 |
if(!hctx->stream_open) return 0; |
470 |
hctx->stream_open = 0; |
471 |
|
472 |
if(hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP && hctx->bytes_out > 0 && |
473 |
(unsigned int )hctx->bytes_out >= sizeof(gzip_header)) { |
474 |
/* write gzip footer */ |
475 |
unsigned char c[8]; |
476 |
|
477 |
c[0] = (hctx->crc >> 0) & 0xff; |
478 |
c[1] = (hctx->crc >> 8) & 0xff; |
479 |
c[2] = (hctx->crc >> 16) & 0xff; |
480 |
c[3] = (hctx->crc >> 24) & 0xff; |
481 |
c[4] = (z->total_in >> 0) & 0xff; |
482 |
c[5] = (z->total_in >> 8) & 0xff; |
483 |
c[6] = (z->total_in >> 16) & 0xff; |
484 |
c[7] = (z->total_in >> 24) & 0xff; |
485 |
/* append footer to write_queue */ |
486 |
chunkqueue_append_mem(con->write_queue, (char *)c, 9); |
487 |
hctx->bytes_out += 8; |
488 |
if(p->conf.debug) { |
489 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
490 |
"gzip_footer len=", 8); |
491 |
} |
492 |
} |
493 |
|
494 |
if ((rc = deflateEnd(z)) != Z_OK) { |
495 |
if(rc == Z_DATA_ERROR) return 0; |
496 |
if(z->msg != NULL) { |
497 |
log_error_write(srv, __FILE__, __LINE__, "sdss", |
498 |
"deflateEnd error ret=", rc, ", msg=", z->msg); |
499 |
} else { |
500 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
501 |
"deflateEnd error ret=", rc); |
502 |
} |
503 |
return -1; |
504 |
} |
505 |
return 0; |
506 |
} |
507 |
|
508 |
#endif |
509 |
|
510 |
#ifdef USE_BZ2LIB |
511 |
static int stream_bzip2_init(server *srv, connection *con, handler_ctx *hctx) { |
512 |
plugin_data *p = hctx->plugin_data; |
513 |
bz_stream *bz; |
514 |
|
515 |
UNUSED(srv); |
516 |
UNUSED(con); |
517 |
|
518 |
bz = &(hctx->bz); |
519 |
bz->bzalloc = NULL; |
520 |
bz->bzfree = NULL; |
521 |
bz->opaque = NULL; |
522 |
bz->total_in_lo32 = 0; |
523 |
bz->total_in_hi32 = 0; |
524 |
bz->total_out_lo32 = 0; |
525 |
bz->total_out_hi32 = 0; |
526 |
|
527 |
if(p->conf.debug) { |
528 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
529 |
"output-buffer-size:", p->conf.output_buffer_size); |
530 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
531 |
"compression-level:", p->conf.compression_level); |
532 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
533 |
"mem-level:", p->conf.mem_level); |
534 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
535 |
"window-size:", p->conf.window_size); |
536 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
537 |
"min-compress-size:", p->conf.min_compress_size); |
538 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
539 |
"work-block-size:", p->conf.work_block_size); |
540 |
} |
541 |
if (BZ_OK != BZ2_bzCompressInit(bz, |
542 |
p->conf.compression_level, /* blocksize = 900k */ |
543 |
0, /* no output */ |
544 |
30)) { /* workFactor: default */ |
545 |
return -1; |
546 |
} |
547 |
hctx->stream_open = 1; |
548 |
|
549 |
return 0; |
550 |
} |
551 |
|
552 |
static int stream_bzip2_compress(server *srv, connection *con, handler_ctx *hctx, unsigned char *start, off_t st_size) { |
553 |
plugin_data *p = hctx->plugin_data; |
554 |
bz_stream *bz; |
555 |
int len; |
556 |
int rc; |
557 |
int in = 0, out = 0; |
558 |
|
559 |
UNUSED(srv); |
560 |
|
561 |
bz = &(hctx->bz); |
562 |
|
563 |
if(bz->next_out == NULL) { |
564 |
bz->next_out = hctx->output->ptr; |
565 |
bz->avail_out = hctx->output->size; |
566 |
} |
567 |
|
568 |
bz->next_in = (char *)start; |
569 |
bz->avail_in = st_size; |
570 |
hctx->bytes_in += st_size; |
571 |
|
572 |
/* compress data */ |
573 |
in = bz->avail_in; |
574 |
do { |
575 |
rc = BZ2_bzCompress(bz, BZ_RUN); |
576 |
if (rc != BZ_RUN_OK) { |
577 |
BZ2_bzCompressEnd(bz); |
578 |
hctx->stream_open = 0; |
579 |
return -1; |
580 |
} |
581 |
|
582 |
if(bz->avail_out == 0 || bz->avail_in > 0) { |
583 |
len = hctx->output->size - bz->avail_out; |
584 |
hctx->bytes_out += len; |
585 |
out += len; |
586 |
chunkqueue_append_mem(con->write_queue, hctx->output->ptr, len+1); |
587 |
bz->next_out = hctx->output->ptr; |
588 |
bz->avail_out = hctx->output->size; |
589 |
} |
590 |
} while (bz->avail_in > 0); |
591 |
if(p->conf.debug) { |
592 |
log_error_write(srv, __FILE__, __LINE__, "sdsd", |
593 |
"compress: in=", in, ", out=", out); |
594 |
} |
595 |
return 0; |
596 |
} |
597 |
|
598 |
static int stream_bzip2_flush(server *srv, connection *con, handler_ctx *hctx, int end) { |
599 |
plugin_data *p = hctx->plugin_data; |
600 |
bz_stream *bz; |
601 |
int len; |
602 |
int rc; |
603 |
int done; |
604 |
int flush = 1; |
605 |
int in = 0, out = 0; |
606 |
|
607 |
UNUSED(srv); |
608 |
|
609 |
bz = &(hctx->bz); |
610 |
|
611 |
if(bz->next_out == NULL) { |
612 |
bz->next_out = hctx->output->ptr; |
613 |
bz->avail_out = hctx->output->size; |
614 |
} |
615 |
/* compress data */ |
616 |
in = bz->avail_in; |
617 |
do { |
618 |
done = 1; |
619 |
if(end) { |
620 |
rc = BZ2_bzCompress(bz, BZ_FINISH); |
621 |
if (rc == BZ_FINISH_OK) { |
622 |
done = 0; |
623 |
} else if (rc != BZ_STREAM_END) { |
624 |
BZ2_bzCompressEnd(bz); |
625 |
hctx->stream_open = 0; |
626 |
return -1; |
627 |
} |
628 |
} else if(bz->avail_in > 0) { |
629 |
rc = BZ2_bzCompress(bz, BZ_RUN); |
630 |
if (rc != BZ_RUN_OK) { |
631 |
BZ2_bzCompressEnd(bz); |
632 |
hctx->stream_open = 0; |
633 |
return -1; |
634 |
} |
635 |
if(p->conf.output_buffer_size > 0) flush = 0; |
636 |
} |
637 |
|
638 |
len = hctx->output->size - bz->avail_out; |
639 |
if(bz->avail_out == 0 || (flush && len > 0)) { |
640 |
hctx->bytes_out += len; |
641 |
out += len; |
642 |
chunkqueue_append_mem(con->write_queue, hctx->output->ptr, len+1); |
643 |
bz->next_out = hctx->output->ptr; |
644 |
bz->avail_out = hctx->output->size; |
645 |
} |
646 |
} while (bz->avail_in != 0 || !done); |
647 |
if(p->conf.debug) { |
648 |
log_error_write(srv, __FILE__, __LINE__, "sdsd", |
649 |
"flush: in=", in, ", out=", out); |
650 |
} |
651 |
if(p->conf.sync_flush) { |
652 |
bz->next_out = NULL; |
653 |
bz->avail_out = 0; |
654 |
} |
655 |
return 0; |
656 |
} |
657 |
|
658 |
static int stream_bzip2_end(server *srv, connection *con, handler_ctx *hctx) { |
659 |
plugin_data *p = hctx->plugin_data; |
660 |
bz_stream *bz; |
661 |
int rc; |
662 |
|
663 |
UNUSED(p); |
664 |
UNUSED(con); |
665 |
|
666 |
bz = &(hctx->bz); |
667 |
if(!hctx->stream_open) return 0; |
668 |
hctx->stream_open = 0; |
669 |
|
670 |
if ((rc = BZ2_bzCompressEnd(bz)) != BZ_OK) { |
671 |
if(rc == BZ_DATA_ERROR) return 0; |
672 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
673 |
"BZ2_bzCompressEnd error ret=", rc); |
674 |
return -1; |
675 |
} |
676 |
return 0; |
677 |
} |
678 |
|
679 |
#endif |
680 |
|
681 |
static int mod_deflate_compress(server *srv, connection *con, handler_ctx *hctx, unsigned char *start, off_t st_size) { |
682 |
int ret = -1; |
683 |
if(st_size == 0) return 0; |
684 |
switch(hctx->compression_type) { |
685 |
#ifdef USE_ZLIB |
686 |
case HTTP_ACCEPT_ENCODING_GZIP: |
687 |
case HTTP_ACCEPT_ENCODING_DEFLATE: |
688 |
ret = stream_deflate_compress(srv, con, hctx, start, st_size); |
689 |
break; |
690 |
#endif |
691 |
#ifdef USE_BZ2LIB |
692 |
case HTTP_ACCEPT_ENCODING_BZIP2: |
693 |
ret = stream_bzip2_compress(srv, con, hctx, start, st_size); |
694 |
break; |
695 |
#endif |
696 |
default: |
697 |
ret = -1; |
698 |
break; |
699 |
} |
700 |
|
701 |
return ret; |
702 |
} |
703 |
|
704 |
static int mod_deflate_stream_flush(server *srv, connection *con, handler_ctx *hctx, int end) { |
705 |
int ret = -1; |
706 |
if(hctx->bytes_in == 0) return 0; |
707 |
switch(hctx->compression_type) { |
708 |
#ifdef USE_ZLIB |
709 |
case HTTP_ACCEPT_ENCODING_GZIP: |
710 |
case HTTP_ACCEPT_ENCODING_DEFLATE: |
711 |
ret = stream_deflate_flush(srv, con, hctx, end); |
712 |
break; |
713 |
#endif |
714 |
#ifdef USE_BZ2LIB |
715 |
case HTTP_ACCEPT_ENCODING_BZIP2: |
716 |
ret = stream_bzip2_flush(srv, con, hctx, end); |
717 |
break; |
718 |
#endif |
719 |
default: |
720 |
ret = -1; |
721 |
break; |
722 |
} |
723 |
|
724 |
return ret; |
725 |
} |
726 |
|
727 |
static int mod_deflate_stream_end(server *srv, connection *con, handler_ctx *hctx) { |
728 |
int ret = -1; |
729 |
switch(hctx->compression_type) { |
730 |
#ifdef USE_ZLIB |
731 |
case HTTP_ACCEPT_ENCODING_GZIP: |
732 |
case HTTP_ACCEPT_ENCODING_DEFLATE: |
733 |
ret = stream_deflate_end(srv, con, hctx); |
734 |
break; |
735 |
#endif |
736 |
#ifdef USE_BZ2LIB |
737 |
case HTTP_ACCEPT_ENCODING_BZIP2: |
738 |
ret = stream_bzip2_end(srv, con, hctx); |
739 |
break; |
740 |
#endif |
741 |
default: |
742 |
ret = -1; |
743 |
break; |
744 |
} |
745 |
|
746 |
return ret; |
747 |
} |
748 |
|
749 |
static int mod_deflate_file_chunk(server *srv, connection *con, handler_ctx *hctx, chunk *c, off_t st_size) { |
750 |
plugin_data *p = hctx->plugin_data; |
751 |
off_t abs_offset; |
752 |
off_t toSend; |
753 |
stat_cache_entry *sce = NULL; |
754 |
off_t we_want_to_mmap = 2 MByte; |
755 |
off_t we_want_to_send = st_size; |
756 |
char *start = NULL; |
757 |
|
758 |
if (HANDLER_ERROR == stat_cache_get_entry(srv, con, c->file.name, &sce)) { |
759 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
760 |
strerror(errno), c->file.name); |
761 |
return -1; |
762 |
} |
763 |
|
764 |
abs_offset = c->file.start + c->offset; |
765 |
|
766 |
if (abs_offset > sce->st.st_size) { |
767 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
768 |
"file was shrinked:", c->file.name); |
769 |
|
770 |
return -1; |
771 |
} |
772 |
|
773 |
we_want_to_send = st_size; |
774 |
/* mmap the buffer |
775 |
* - first mmap |
776 |
* - new mmap as the we are at the end of the last one */ |
777 |
if (c->file.mmap.start == MAP_FAILED || |
778 |
abs_offset == (off_t)(c->file.mmap.offset + c->file.mmap.length)) { |
779 |
|
780 |
/* Optimizations for the future: |
781 |
* |
782 |
* adaptive mem-mapping |
783 |
* the problem: |
784 |
* we mmap() the whole file. If someone has alot large files and 32bit |
785 |
* machine the virtual address area will be unrun and we will have a failing |
786 |
* mmap() call. |
787 |
* solution: |
788 |
* only mmap 16M in one chunk and move the window as soon as we have finished |
789 |
* the first 8M |
790 |
* |
791 |
* read-ahead buffering |
792 |
* the problem: |
793 |
* sending out several large files in parallel trashes the read-ahead of the |
794 |
* kernel leading to long wait-for-seek times. |
795 |
* solutions: (increasing complexity) |
796 |
* 1. use madvise |
797 |
* 2. use a internal read-ahead buffer in the chunk-structure |
798 |
* 3. use non-blocking IO for file-transfers |
799 |
* */ |
800 |
|
801 |
/* all mmap()ed areas are 512kb expect the last which might be smaller */ |
802 |
size_t to_mmap; |
803 |
|
804 |
/* this is a remap, move the mmap-offset */ |
805 |
if (c->file.mmap.start != MAP_FAILED) { |
806 |
munmap(c->file.mmap.start, c->file.mmap.length); |
807 |
c->file.mmap.offset += we_want_to_mmap; |
808 |
} else { |
809 |
/* in case the range-offset is after the first mmap()ed area we skip the area */ |
810 |
c->file.mmap.offset = 0; |
811 |
|
812 |
while (c->file.mmap.offset + we_want_to_mmap < c->file.start) { |
813 |
c->file.mmap.offset += we_want_to_mmap; |
814 |
} |
815 |
} |
816 |
|
817 |
/* length is rel, c->offset too, assume there is no limit at the mmap-boundaries */ |
818 |
to_mmap = (c->file.start + c->file.length) - c->file.mmap.offset; |
819 |
if(to_mmap > we_want_to_mmap) to_mmap = we_want_to_mmap; |
820 |
/* we have more to send than we can mmap() at once */ |
821 |
if(we_want_to_send > to_mmap) we_want_to_send = to_mmap; |
822 |
|
823 |
if (-1 == c->file.fd) { /* open the file if not already open */ |
824 |
if (-1 == (c->file.fd = open(c->file.name->ptr, O_RDONLY))) { |
825 |
log_error_write(srv, __FILE__, __LINE__, "sbs", "open failed for:", c->file.name, strerror(errno)); |
826 |
|
827 |
return -1; |
828 |
} |
829 |
#ifdef FD_CLOEXEC |
830 |
fcntl(c->file.fd, F_SETFD, FD_CLOEXEC); |
831 |
#endif |
832 |
} |
833 |
|
834 |
if (MAP_FAILED == (c->file.mmap.start = mmap(0, to_mmap, PROT_READ, MAP_SHARED, c->file.fd, c->file.mmap.offset))) { |
835 |
/* close it here, otherwise we'd have to set FD_CLOEXEC */ |
836 |
|
837 |
log_error_write(srv, __FILE__, __LINE__, "ssbd", "mmap failed:", |
838 |
strerror(errno), c->file.name, c->file.fd); |
839 |
|
840 |
return -1; |
841 |
} |
842 |
|
843 |
c->file.mmap.length = to_mmap; |
844 |
#ifdef LOCAL_BUFFERING |
845 |
buffer_copy_string_len(c->mem, c->file.mmap.start, c->file.mmap.length); |
846 |
#else |
847 |
#ifdef HAVE_MADVISE |
848 |
/* don't advise files < 64Kb */ |
849 |
if (c->file.mmap.length > (64 KByte) && |
850 |
0 != madvise(c->file.mmap.start, c->file.mmap.length, MADV_WILLNEED)) { |
851 |
log_error_write(srv, __FILE__, __LINE__, "ssbd", "madvise failed:", |
852 |
strerror(errno), c->file.name, c->file.fd); |
853 |
} |
854 |
#endif |
855 |
#endif |
856 |
|
857 |
/* chunk_reset() or chunk_free() will cleanup for us */ |
858 |
} |
859 |
|
860 |
/* to_send = abs_mmap_end - abs_offset */ |
861 |
toSend = (c->file.mmap.offset + c->file.mmap.length) - (abs_offset); |
862 |
if(toSend > we_want_to_send) toSend = we_want_to_send; |
863 |
|
864 |
if (toSend < 0) { |
865 |
log_error_write(srv, __FILE__, __LINE__, "soooo", |
866 |
"toSend is negative:", |
867 |
toSend, |
868 |
c->file.mmap.length, |
869 |
abs_offset, |
870 |
c->file.mmap.offset); |
871 |
assert(toSend < 0); |
872 |
} |
873 |
|
874 |
#ifdef LOCAL_BUFFERING |
875 |
start = c->mem->ptr; |
876 |
#else |
877 |
start = c->file.mmap.start; |
878 |
#endif |
879 |
|
880 |
if(p->conf.debug) { |
881 |
log_error_write(srv, __FILE__, __LINE__, "sdsd", |
882 |
"compress file chunk: offset=", (int)c->offset, |
883 |
", toSend=", (int)toSend); |
884 |
} |
885 |
if (mod_deflate_compress(srv, con, hctx, |
886 |
(unsigned char *)start + (abs_offset - c->file.mmap.offset), toSend) < 0) { |
887 |
log_error_write(srv, __FILE__, __LINE__, "s", |
888 |
"compress failed."); |
889 |
return -1; |
890 |
} |
891 |
|
892 |
c->offset += toSend; |
893 |
if (c->offset == c->file.length) { |
894 |
/* we don't need the mmaping anymore */ |
895 |
if (c->file.mmap.start != MAP_FAILED) { |
896 |
munmap(c->file.mmap.start, c->file.mmap.length); |
897 |
c->file.mmap.start = MAP_FAILED; |
898 |
} |
899 |
} |
900 |
|
901 |
return toSend; |
902 |
} |
903 |
|
904 |
static int deflate_compress_cleanup(server *srv, connection *con, handler_ctx *hctx) { |
905 |
plugin_data *p = hctx->plugin_data; |
906 |
int rc; |
907 |
|
908 |
rc = mod_deflate_stream_end(srv, con, hctx); |
909 |
if(rc < 0) { |
910 |
log_error_write(srv, __FILE__, __LINE__, "s", "error closing stream"); |
911 |
} |
912 |
|
913 |
if (hctx->bytes_in < hctx->bytes_out) { |
914 |
log_error_write(srv, __FILE__, __LINE__, "sbsdsd", |
915 |
"uri ", con->uri.path_raw, " in=", hctx->bytes_in, " smaller than out=", hctx->bytes_out); |
916 |
} |
917 |
|
918 |
if(p->conf.debug) { |
919 |
log_error_write(srv, __FILE__, __LINE__, "sdsd", |
920 |
" in:", hctx->bytes_in, |
921 |
" out:", hctx->bytes_out); |
922 |
} |
923 |
|
924 |
/* cleanup compression state */ |
925 |
if(hctx->output != p->tmp_buf) { |
926 |
buffer_free(hctx->output); |
927 |
} |
928 |
handler_ctx_free(hctx); |
929 |
con->plugin_ctx[p->id] = NULL; |
930 |
|
931 |
return 0; |
932 |
} |
933 |
|
934 |
static handler_t deflate_compress_response(server *srv, connection *con, handler_ctx *hctx, int end) { |
935 |
plugin_data *p = hctx->plugin_data; |
936 |
chunk *c; |
937 |
size_t chunks_written = 0; |
938 |
int chunk_finished = 0; |
939 |
int rc=-1; |
940 |
int close_stream = 0, len = 0; |
941 |
unsigned int out = 0, max = 0; |
942 |
|
943 |
/* move all chunk from write_queue into our in_queue */ |
944 |
chunkqueue_append_chunkqueue(hctx->in_queue, con->write_queue); |
945 |
|
946 |
len = chunkqueue_length(hctx->in_queue); |
947 |
if(p->conf.debug) { |
948 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
949 |
"compress: in_queue len=", len); |
950 |
} |
951 |
/* calculate max bytes to compress for this call. */ |
952 |
if(!end) { |
953 |
max = p->conf.work_block_size * 1024; |
954 |
if(max == 0 || max > len) max = len; |
955 |
} else { |
956 |
max = len; |
957 |
} |
958 |
|
959 |
/* Compress chunks from in_queue into chunks for write_queue */ |
960 |
for(c = hctx->in_queue->first; c && out < max; c = c->next) { |
961 |
chunk_finished = 0; |
962 |
len = 0; |
963 |
|
964 |
switch(c->type) { |
965 |
case MEM_CHUNK: |
966 |
len = c->mem->used - 1; |
967 |
if(len > (max - out)) len = max - out; |
968 |
if (mod_deflate_compress(srv, con, hctx, (unsigned char *)c->mem->ptr, len) < 0) { |
969 |
log_error_write(srv, __FILE__, __LINE__, "s", |
970 |
"compress failed."); |
971 |
return HANDLER_ERROR; |
972 |
} |
973 |
c->offset += len; |
974 |
out += len; |
975 |
if (c->offset == c->mem->used - 1) { |
976 |
chunk_finished = 1; |
977 |
chunks_written++; |
978 |
} |
979 |
break; |
980 |
case FILE_CHUNK: |
981 |
len = c->file.length - c->offset; |
982 |
if(len > (max - out)) len = max - out; |
983 |
if ((len = mod_deflate_file_chunk(srv, con, hctx, c, len)) < 0) { |
984 |
log_error_write(srv, __FILE__, __LINE__, "s", |
985 |
"compress file chunk failed."); |
986 |
return HANDLER_ERROR; |
987 |
} |
988 |
out += len; |
989 |
if (c->offset == c->file.length) { |
990 |
chunk_finished = 1; |
991 |
chunks_written++; |
992 |
} |
993 |
break; |
994 |
default: |
995 |
|
996 |
log_error_write(srv, __FILE__, __LINE__, "ds", c, "type not known"); |
997 |
|
998 |
return HANDLER_ERROR; |
999 |
} |
1000 |
if(!chunk_finished) break; |
1001 |
} |
1002 |
if(p->conf.debug) { |
1003 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
1004 |
"compressed bytes:", out); |
1005 |
} |
1006 |
hctx->in_queue->bytes_out += out; |
1007 |
|
1008 |
if(chunks_written > 0) { |
1009 |
chunkqueue_remove_finished_chunks(hctx->in_queue); |
1010 |
} |
1011 |
|
1012 |
close_stream = (con->file_finished && chunkqueue_is_empty(hctx->in_queue)); |
1013 |
rc = mod_deflate_stream_flush(srv, con, hctx, close_stream); |
1014 |
if(rc < 0) { |
1015 |
log_error_write(srv, __FILE__, __LINE__, "s", "flush error"); |
1016 |
} |
1017 |
if(close_stream || end) { |
1018 |
deflate_compress_cleanup(srv, con, hctx); |
1019 |
if(p->conf.debug) { |
1020 |
log_error_write(srv, __FILE__, __LINE__, "sbsb", |
1021 |
"finished uri:", con->uri.path_raw, ", query:", con->uri.query); |
1022 |
} |
1023 |
return HANDLER_FINISHED; |
1024 |
} else { |
1025 |
if(!chunkqueue_is_empty(hctx->in_queue)) { |
1026 |
/* We have more data to compress. */ |
1027 |
joblist_append(srv, con); |
1028 |
} |
1029 |
return HANDLER_COMEBACK; |
1030 |
} |
1031 |
} |
1032 |
|
1033 |
#define PATCH(x) \ |
1034 |
p->conf.x = s->x; |
1035 |
static int mod_deflate_patch_connection(server *srv, connection *con, plugin_data *p) { |
1036 |
size_t i, j; |
1037 |
plugin_config *s = p->config_storage[0]; |
1038 |
|
1039 |
PATCH(output_buffer_size); |
1040 |
PATCH(mimetypes); |
1041 |
PATCH(compression_level); |
1042 |
PATCH(mem_level); |
1043 |
PATCH(window_size); |
1044 |
PATCH(min_compress_size); |
1045 |
PATCH(work_block_size); |
1046 |
PATCH(enabled); |
1047 |
PATCH(debug); |
1048 |
PATCH(bzip2); |
1049 |
PATCH(sync_flush); |
1050 |
#if defined(HAVE_PCRE_H) |
1051 |
PATCH(nocompress_regex); |
1052 |
#endif |
1053 |
|
1054 |
/* skip the first, the global context */ |
1055 |
for (i = 1; i < srv->config_context->used; i++) { |
1056 |
data_config *dc = (data_config *)srv->config_context->data[i]; |
1057 |
s = p->config_storage[i]; |
1058 |
|
1059 |
/* condition didn't match */ |
1060 |
if (!config_check_cond(srv, con, dc)) continue; |
1061 |
|
1062 |
/* merge config */ |
1063 |
for (j = 0; j < dc->value->used; j++) { |
1064 |
data_unset *du = dc->value->data[j]; |
1065 |
|
1066 |
if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.output-buffer-size"))) { |
1067 |
PATCH(output_buffer_size); |
1068 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.mimetypes"))) { |
1069 |
PATCH(mimetypes); |
1070 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.compression-level"))) { |
1071 |
PATCH(compression_level); |
1072 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.mem-level"))) { |
1073 |
PATCH(mem_level); |
1074 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.window-size"))) { |
1075 |
PATCH(window_size); |
1076 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.min-compress-size"))) { |
1077 |
PATCH(min_compress_size); |
1078 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.work-block-size"))) { |
1079 |
PATCH(work_block_size); |
1080 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.enabled"))) { |
1081 |
PATCH(enabled); |
1082 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.debug"))) { |
1083 |
PATCH(debug); |
1084 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.bzip2"))) { |
1085 |
PATCH(bzip2); |
1086 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.sync-flush"))) { |
1087 |
PATCH(sync_flush); |
1088 |
} else if (buffer_is_equal_string(du->key, CONST_STR_LEN("deflate.nocompress-url"))) { |
1089 |
#if defined(HAVE_PCRE_H) |
1090 |
PATCH(nocompress_regex); |
1091 |
#endif |
1092 |
} |
1093 |
} |
1094 |
} |
1095 |
|
1096 |
return 0; |
1097 |
} |
1098 |
#undef PATCH |
1099 |
|
1100 |
PHYSICALPATH_FUNC(mod_deflate_handle_response_start) { |
1101 |
plugin_data *p = p_d; |
1102 |
handler_ctx *hctx; |
1103 |
data_string *ds; |
1104 |
int accept_encoding = 0; |
1105 |
char *value; |
1106 |
int srv_encodings = 0; |
1107 |
int matched_encodings = 0; |
1108 |
const char *dflt_gzip = "gzip"; |
1109 |
const char *dflt_deflate = "deflate"; |
1110 |
const char *dflt_bzip2 = "bzip2"; |
1111 |
const char *compression_name = NULL; |
1112 |
int file_len=0; |
1113 |
int rc=-2; |
1114 |
int end = 0; |
1115 |
size_t m; |
1116 |
#if defined(HAVE_PCRE_H) |
1117 |
int n; |
1118 |
# define N 10 |
1119 |
int ovec[N * 3]; |
1120 |
#endif |
1121 |
|
1122 |
/* disable compression for some http status types. */ |
1123 |
switch(con->http_status) { |
1124 |
case 100: |
1125 |
case 101: |
1126 |
case 204: |
1127 |
case 205: |
1128 |
case 304: |
1129 |
/* disable compression as we have no response entity */ |
1130 |
return HANDLER_GO_ON; |
1131 |
default: |
1132 |
break; |
1133 |
} |
1134 |
|
1135 |
mod_deflate_patch_connection(srv, con, p); |
1136 |
|
1137 |
/* is compression allowed */ |
1138 |
if(!p->conf.enabled) { |
1139 |
if(p->conf.debug) { |
1140 |
log_error_write(srv, __FILE__, __LINE__, "s", "compression disabled."); |
1141 |
} |
1142 |
return HANDLER_GO_ON; |
1143 |
} |
1144 |
|
1145 |
#if defined(HAVE_PCRE_H) |
1146 |
if(p->conf.nocompress_regex) { /*check no compress regex now */ |
1147 |
if ((n = pcre_exec(p->conf.nocompress_regex, NULL, con->uri.path->ptr, con->uri.path->used - 1, 0, 0, ovec, 3 * N)) < 0) { |
1148 |
if (n != PCRE_ERROR_NOMATCH) { |
1149 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
1150 |
"execution error while matching:", n); |
1151 |
return HANDLER_ERROR; |
1152 |
} |
1153 |
} else { |
1154 |
if(p->conf.debug) { |
1155 |
log_error_write(srv, __FILE__, __LINE__, "sb", "no compress for url:", con->uri.path); |
1156 |
} |
1157 |
return HANDLER_GO_ON; |
1158 |
} |
1159 |
} |
1160 |
#endif |
1161 |
/* Check if response has a Content-Encoding. */ |
1162 |
if (NULL != (ds = (data_string *)array_get_element(con->response.headers, "Content-Encoding"))) { |
1163 |
return HANDLER_GO_ON; |
1164 |
} |
1165 |
|
1166 |
/* Check Accept-Encoding for supported encoding. */ |
1167 |
if (NULL == (ds = (data_string *)array_get_element(con->request.headers, "Accept-Encoding"))) { |
1168 |
return HANDLER_GO_ON; |
1169 |
} |
1170 |
|
1171 |
/* get client side support encodings */ |
1172 |
value = ds->value->ptr; |
1173 |
#ifdef USE_ZLIB |
1174 |
if (NULL != strstr(value, "gzip")) accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP; |
1175 |
if (NULL != strstr(value, "deflate")) accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE; |
1176 |
#endif |
1177 |
/* if (NULL != strstr(value, "compress")) accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS; */ |
1178 |
#ifdef USE_BZ2LIB |
1179 |
if(p->conf.bzip2) { |
1180 |
if (NULL != strstr(value, "bzip2")) accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2; |
1181 |
} |
1182 |
#endif |
1183 |
if (NULL != strstr(value, "identity")) accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY; |
1184 |
|
1185 |
/* get server side supported ones */ |
1186 |
#ifdef USE_BZ2LIB |
1187 |
if(p->conf.bzip2) { |
1188 |
srv_encodings |= HTTP_ACCEPT_ENCODING_BZIP2; |
1189 |
} |
1190 |
#endif |
1191 |
#ifdef USE_ZLIB |
1192 |
srv_encodings |= HTTP_ACCEPT_ENCODING_GZIP; |
1193 |
srv_encodings |= HTTP_ACCEPT_ENCODING_DEFLATE; |
1194 |
#endif |
1195 |
|
1196 |
/* find matching encodings */ |
1197 |
matched_encodings = accept_encoding & srv_encodings; |
1198 |
if (!matched_encodings) { |
1199 |
return HANDLER_GO_ON; |
1200 |
} |
1201 |
|
1202 |
if (con->request.true_http_10_client) { |
1203 |
/*disable gzip/bzip2 when we meet HTTP 1.0 client with Accept-Encoding |
1204 |
* maybe old buggy proxy server |
1205 |
*/ |
1206 |
/* most of buggy clients are Yahoo Slurp;) */ |
1207 |
if (p->conf.debug) log_error_write(srv, __FILE__, __LINE__, "ss", |
1208 |
"Buggy HTTP 1.0 client sending Accept Encoding: gzip, deflate", |
1209 |
(char *) inet_ntop_cache_get_ip(srv, &(con->dst_addr))); |
1210 |
return HANDLER_GO_ON; |
1211 |
} |
1212 |
/* check if size of response is below min-compress-size */ |
1213 |
if(con->file_finished && con->request.http_method != HTTP_METHOD_HEAD) { |
1214 |
file_len = chunkqueue_length(con->write_queue); |
1215 |
if(file_len == 0) return HANDLER_GO_ON; |
1216 |
} else { |
1217 |
file_len = 0; |
1218 |
} |
1219 |
|
1220 |
if(file_len > 0 && p->conf.min_compress_size > 0 && file_len < p->conf.min_compress_size) { |
1221 |
if(p->conf.debug) { |
1222 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
1223 |
"Content-Length smaller then min_compress_size: file_len=", file_len); |
1224 |
} |
1225 |
return HANDLER_GO_ON; |
1226 |
} |
1227 |
|
1228 |
/* Check mimetype in response header "Content-Type" */ |
1229 |
if (NULL != (ds = (data_string *)array_get_element(con->response.headers, "Content-Type"))) { |
1230 |
int found = 0; |
1231 |
if(p->conf.debug) { |
1232 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
1233 |
"Content-Type:", ds->value); |
1234 |
} |
1235 |
for (m = 0; m < p->conf.mimetypes->used; m++) { |
1236 |
data_string *mimetype = (data_string *)p->conf.mimetypes->data[m]; |
1237 |
|
1238 |
if(p->conf.debug) { |
1239 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
1240 |
"mime-type:", mimetype->value); |
1241 |
} |
1242 |
if (strncmp(mimetype->value->ptr, ds->value->ptr, mimetype->value->used-1) == 0) { |
1243 |
/* if (buffer_is_equal(mimetype->value, ds->value)) { */ |
1244 |
/* mimetype found */ |
1245 |
found = 1; |
1246 |
break; |
1247 |
} |
1248 |
} |
1249 |
if(!found && p->conf.mimetypes->used > 0) { |
1250 |
if(p->conf.debug) { |
1251 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
1252 |
"No compression for mimetype:", ds->value); |
1253 |
} |
1254 |
return HANDLER_GO_ON; |
1255 |
} |
1256 |
#if 0 |
1257 |
if(strncasecmp(ds->value->ptr, "application/x-javascript", 24) == 0) { |
1258 |
/*reset compress type to deflate for javascript |
1259 |
* prevent buggy IE6 SP1 doesn't work for js in IFrame |
1260 |
*/ |
1261 |
matched_encodings = HTTP_ACCEPT_ENCODING_DEFLATE; |
1262 |
} |
1263 |
#endif |
1264 |
} |
1265 |
|
1266 |
if(p->conf.debug) { |
1267 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
1268 |
"enable compression for ", con->uri.path); |
1269 |
} |
1270 |
|
1271 |
/* the response might change according to Accept-Encoding */ |
1272 |
if (NULL != (ds = (data_string *)array_get_element(con->response.headers, "Vary"))) { |
1273 |
/* append Accept-Encoding to Vary header */ |
1274 |
if (NULL == strstr(ds->value->ptr, "Accept-Encoding")) { |
1275 |
buffer_append_string(ds->value, ",Accept-Encoding"); |
1276 |
if (p->conf.debug) { |
1277 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
1278 |
"appending ,Accept-Encoding for ", con->uri.path); |
1279 |
} |
1280 |
} |
1281 |
} else { |
1282 |
if (p->conf.debug) { |
1283 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
1284 |
"add Vary: Accept-Encoding for ", con->uri.path); |
1285 |
} |
1286 |
response_header_insert(srv, con, CONST_STR_LEN("Vary"), |
1287 |
CONST_STR_LEN("Accept-Encoding")); |
1288 |
} |
1289 |
|
1290 |
/* enable compression */ |
1291 |
hctx = handler_ctx_init(); |
1292 |
hctx->plugin_data = p; |
1293 |
|
1294 |
/* select best matching encoding */ |
1295 |
if (matched_encodings & HTTP_ACCEPT_ENCODING_BZIP2) { |
1296 |
hctx->compression_type = HTTP_ACCEPT_ENCODING_BZIP2; |
1297 |
compression_name = dflt_bzip2; |
1298 |
rc = stream_bzip2_init(srv, con, hctx); |
1299 |
} else if (matched_encodings & HTTP_ACCEPT_ENCODING_GZIP) { |
1300 |
hctx->compression_type = HTTP_ACCEPT_ENCODING_GZIP; |
1301 |
compression_name = dflt_gzip; |
1302 |
rc = stream_deflate_init(srv, con, hctx); |
1303 |
} else if (matched_encodings & HTTP_ACCEPT_ENCODING_DEFLATE) { |
1304 |
hctx->compression_type = HTTP_ACCEPT_ENCODING_DEFLATE; |
1305 |
compression_name = dflt_deflate; |
1306 |
rc = stream_deflate_init(srv, con, hctx); |
1307 |
} |
1308 |
if(rc == -1) { |
1309 |
log_error_write(srv, __FILE__, __LINE__, "s", |
1310 |
"Failed to initialize compression."); |
1311 |
} |
1312 |
|
1313 |
if(rc < 0) { |
1314 |
handler_ctx_free(hctx); |
1315 |
return HANDLER_GO_ON; |
1316 |
} |
1317 |
|
1318 |
/* setup output buffer. */ |
1319 |
if(p->conf.sync_flush || p->conf.output_buffer_size == 0) { |
1320 |
buffer_prepare_copy(p->tmp_buf, 32 * 1024); |
1321 |
hctx->output = p->tmp_buf; |
1322 |
} else { |
1323 |
hctx->output = buffer_init(); |
1324 |
buffer_prepare_copy(hctx->output, p->conf.output_buffer_size); |
1325 |
} |
1326 |
con->plugin_ctx[p->id] = hctx; |
1327 |
|
1328 |
/* set Content-Encoding to show selected compression type. */ |
1329 |
response_header_overwrite(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name)); |
1330 |
|
1331 |
if (con->file_finished) end = 1; |
1332 |
|
1333 |
con->parsed_response &= ~(HTTP_CONTENT_LENGTH); |
1334 |
#if 0 |
1335 |
/* debug */ |
1336 |
if (con->parsed_response & HTTP_TRANSFER_ENCODING_CHUNKED) |
1337 |
log_error_write(srv, __FILE__, __LINE__, "s", |
1338 |
"deflate: response with chunked encoding"); |
1339 |
if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) |
1340 |
log_error_write(srv, __FILE__, __LINE__, "s", |
1341 |
"deflate: transfer encoding with chunked encoding"); |
1342 |
#endif |
1343 |
|
1344 |
if (con->file_finished && (p->conf.work_block_size == 0 || file_len < (p->conf.work_block_size * 1024)) |
1345 |
&& con->request.http_method != HTTP_METHOD_HEAD) { |
1346 |
/* disable chunk transfer */ |
1347 |
con->response.transfer_encoding = 0; |
1348 |
con->parsed_response &= ~(HTTP_TRANSFER_ENCODING_CHUNKED); |
1349 |
if(p->conf.debug) { |
1350 |
log_error_write(srv, __FILE__, __LINE__, "sd", |
1351 |
"Compress all content and use Content-Length header: uncompress len=", file_len); |
1352 |
} |
1353 |
return deflate_compress_response(srv, con, hctx, end); |
1354 |
} else { |
1355 |
if (con->request.http_version == HTTP_VERSION_1_1) { |
1356 |
if (p->conf.debug) |
1357 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
1358 |
"chunk transfer encoding for uri", con->uri.path); |
1359 |
/* Make sure to use chunked encoding. */ |
1360 |
con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED; |
1361 |
} else { |
1362 |
if (p->conf.debug) |
1363 |
log_error_write(srv, __FILE__, __LINE__, "sb", |
1364 |
"http 1.0 encoding for uri", con->uri.path); |
1365 |
/* We don't have to use chunked encoding because HTTP 1.0 don't support it. */ |
1366 |
con->response.transfer_encoding = 0; |
1367 |
con->parsed_response &= ~(HTTP_TRANSFER_ENCODING_CHUNKED); |
1368 |
} |
1369 |
} |
1370 |
|
1371 |
if (p->conf.debug) |
1372 |
log_error_write(srv, __FILE__, __LINE__, "sdsb", "end =", end, "for uri", con->uri.path); |
1373 |
|
1374 |
deflate_compress_response(srv, con, hctx, end); |
1375 |
return HANDLER_GO_ON; |
1376 |
} |
1377 |
|
1378 |
JOBLIST_FUNC(mod_deflate_handle_response_filter) { |
1379 |
plugin_data *p = p_d; |
1380 |
handler_ctx *hctx = con->plugin_ctx[p->id]; |
1381 |
|
1382 |
if(hctx == NULL) return HANDLER_GO_ON; |
1383 |
if(!hctx->stream_open) return HANDLER_GO_ON; |
1384 |
if(con->request.http_method == HTTP_METHOD_HEAD) return HANDLER_GO_ON; |
1385 |
|
1386 |
return deflate_compress_response(srv, con, hctx, 0); |
1387 |
} |
1388 |
|
1389 |
handler_t mod_deflate_cleanup(server *srv, connection *con, void *p_d) { |
1390 |
plugin_data *p = p_d; |
1391 |
handler_ctx *hctx = con->plugin_ctx[p->id]; |
1392 |
|
1393 |
if(hctx == NULL) return HANDLER_GO_ON; |
1394 |
|
1395 |
if(p->conf.debug && hctx->stream_open) { |
1396 |
log_error_write(srv, __FILE__, __LINE__, "sbsb", |
1397 |
"stream open at cleanup. uri=", con->uri.path_raw, ", query=", con->uri.query); |
1398 |
} |
1399 |
|
1400 |
deflate_compress_cleanup(srv, con, hctx); |
1401 |
|
1402 |
return HANDLER_GO_ON; |
1403 |
} |
1404 |
|
1405 |
int mod_deflate_plugin_init(plugin *p) { |
1406 |
p->version = LIGHTTPD_VERSION_ID; |
1407 |
p->name = buffer_init_string("deflate"); |
1408 |
|
1409 |
p->init = mod_deflate_init; |
1410 |
p->cleanup = mod_deflate_free; |
1411 |
p->set_defaults = mod_deflate_setdefaults; |
1412 |
p->connection_reset = mod_deflate_cleanup; |
1413 |
p->handle_connection_close = mod_deflate_cleanup; |
1414 |
p->handle_response_start = mod_deflate_handle_response_start; |
1415 |
p->handle_response_filter = mod_deflate_handle_response_filter; |
1416 |
|
1417 |
p->data = NULL; |
1418 |
|
1419 |
return 0; |
1420 |
} |