| 1 |
/* gzappend -- command to append to a gzip file |
| 2 |
|
| 3 |
Copyright (C) 2003, 2012 Mark Adler, all rights reserved |
| 4 |
version 1.2, 11 Oct 2012 |
| 5 |
|
| 6 |
This software is provided 'as-is', without any express or implied |
| 7 |
warranty. In no event will the author be held liable for any damages |
| 8 |
arising from the use of this software. |
| 9 |
|
| 10 |
Permission is granted to anyone to use this software for any purpose, |
| 11 |
including commercial applications, and to alter it and redistribute it |
| 12 |
freely, subject to the following restrictions: |
| 13 |
|
| 14 |
1. The origin of this software must not be misrepresented; you must not |
| 15 |
claim that you wrote the original software. If you use this software |
| 16 |
in a product, an acknowledgment in the product documentation would be |
| 17 |
appreciated but is not required. |
| 18 |
2. Altered source versions must be plainly marked as such, and must not be |
| 19 |
misrepresented as being the original software. |
| 20 |
3. This notice may not be removed or altered from any source distribution. |
| 21 |
|
| 22 |
Mark Adler madler@alumni.caltech.edu |
| 23 |
*/ |
| 24 |
|
| 25 |
/* |
| 26 |
* Change history: |
| 27 |
* |
| 28 |
* 1.0 19 Oct 2003 - First version |
| 29 |
* 1.1 4 Nov 2003 - Expand and clarify some comments and notes |
| 30 |
* - Add version and copyright to help |
| 31 |
* - Send help to stdout instead of stderr |
| 32 |
* - Add some preemptive typecasts |
| 33 |
* - Add L to constants in lseek() calls |
| 34 |
* - Remove some debugging information in error messages |
| 35 |
* - Use new data_type definition for zlib 1.2.1 |
| 36 |
* - Simplfy and unify file operations |
| 37 |
* - Finish off gzip file in gztack() |
| 38 |
* - Use deflatePrime() instead of adding empty blocks |
| 39 |
* - Keep gzip file clean on appended file read errors |
| 40 |
* - Use in-place rotate instead of auxiliary buffer |
| 41 |
* (Why you ask? Because it was fun to write!) |
| 42 |
* 1.2 11 Oct 2012 - Fix for proper z_const usage |
| 43 |
* - Check for input buffer malloc failure |
| 44 |
*/ |
| 45 |
|
| 46 |
/* |
| 47 |
gzappend takes a gzip file and appends to it, compressing files from the |
| 48 |
command line or data from stdin. The gzip file is written to directly, to |
| 49 |
avoid copying that file, in case it's large. Note that this results in the |
| 50 |
unfriendly behavior that if gzappend fails, the gzip file is corrupted. |
| 51 |
|
| 52 |
This program was written to illustrate the use of the new Z_BLOCK option of |
| 53 |
zlib 1.2.x's inflate() function. This option returns from inflate() at each |
| 54 |
block boundary to facilitate locating and modifying the last block bit at |
| 55 |
the start of the final deflate block. Also whether using Z_BLOCK or not, |
| 56 |
another required feature of zlib 1.2.x is that inflate() now provides the |
| 57 |
number of unusued bits in the last input byte used. gzappend will not work |
| 58 |
with versions of zlib earlier than 1.2.1. |
| 59 |
|
| 60 |
gzappend first decompresses the gzip file internally, discarding all but |
| 61 |
the last 32K of uncompressed data, and noting the location of the last block |
| 62 |
bit and the number of unused bits in the last byte of the compressed data. |
| 63 |
The gzip trailer containing the CRC-32 and length of the uncompressed data |
| 64 |
is verified. This trailer will be later overwritten. |
| 65 |
|
| 66 |
Then the last block bit is cleared by seeking back in the file and rewriting |
| 67 |
the byte that contains it. Seeking forward, the last byte of the compressed |
| 68 |
data is saved along with the number of unused bits to initialize deflate. |
| 69 |
|
| 70 |
A deflate process is initialized, using the last 32K of the uncompressed |
| 71 |
data from the gzip file to initialize the dictionary. If the total |
| 72 |
uncompressed data was less than 32K, then all of it is used to initialize |
| 73 |
the dictionary. The deflate output bit buffer is also initialized with the |
| 74 |
last bits from the original deflate stream. From here on, the data to |
| 75 |
append is simply compressed using deflate, and written to the gzip file. |
| 76 |
When that is complete, the new CRC-32 and uncompressed length are written |
| 77 |
as the trailer of the gzip file. |
| 78 |
*/ |
| 79 |
|
| 80 |
#include <stdio.h> |
| 81 |
#include <stdlib.h> |
| 82 |
#include <string.h> |
| 83 |
#include <fcntl.h> |
| 84 |
#include <unistd.h> |
| 85 |
#include "zlib.h" |
| 86 |
|
| 87 |
#define local static |
| 88 |
#define LGCHUNK 14 |
| 89 |
#define CHUNK (1U << LGCHUNK) |
| 90 |
#define DSIZE 32768U |
| 91 |
|
| 92 |
/* print an error message and terminate with extreme prejudice */ |
| 93 |
local void bye(char *msg1, char *msg2) |
| 94 |
{ |
| 95 |
fprintf(stderr, "gzappend error: %s%s\n", msg1, msg2); |
| 96 |
exit(1); |
| 97 |
} |
| 98 |
|
| 99 |
/* return the greatest common divisor of a and b using Euclid's algorithm, |
| 100 |
modified to be fast when one argument much greater than the other, and |
| 101 |
coded to avoid unnecessary swapping */ |
| 102 |
local unsigned gcd(unsigned a, unsigned b) |
| 103 |
{ |
| 104 |
unsigned c; |
| 105 |
|
| 106 |
while (a && b) |
| 107 |
if (a > b) { |
| 108 |
c = b; |
| 109 |
while (a - c >= c) |
| 110 |
c <<= 1; |
| 111 |
a -= c; |
| 112 |
} |
| 113 |
else { |
| 114 |
c = a; |
| 115 |
while (b - c >= c) |
| 116 |
c <<= 1; |
| 117 |
b -= c; |
| 118 |
} |
| 119 |
return a + b; |
| 120 |
} |
| 121 |
|
| 122 |
/* rotate list[0..len-1] left by rot positions, in place */ |
| 123 |
local void rotate(unsigned char *list, unsigned len, unsigned rot) |
| 124 |
{ |
| 125 |
unsigned char tmp; |
| 126 |
unsigned cycles; |
| 127 |
unsigned char *start, *last, *to, *from; |
| 128 |
|
| 129 |
/* normalize rot and handle degenerate cases */ |
| 130 |
if (len < 2) return; |
| 131 |
if (rot >= len) rot %= len; |
| 132 |
if (rot == 0) return; |
| 133 |
|
| 134 |
/* pointer to last entry in list */ |
| 135 |
last = list + (len - 1); |
| 136 |
|
| 137 |
/* do simple left shift by one */ |
| 138 |
if (rot == 1) { |
| 139 |
tmp = *list; |
| 140 |
memcpy(list, list + 1, len - 1); |
| 141 |
*last = tmp; |
| 142 |
return; |
| 143 |
} |
| 144 |
|
| 145 |
/* do simple right shift by one */ |
| 146 |
if (rot == len - 1) { |
| 147 |
tmp = *last; |
| 148 |
memmove(list + 1, list, len - 1); |
| 149 |
*list = tmp; |
| 150 |
return; |
| 151 |
} |
| 152 |
|
| 153 |
/* otherwise do rotate as a set of cycles in place */ |
| 154 |
cycles = gcd(len, rot); /* number of cycles */ |
| 155 |
do { |
| 156 |
start = from = list + cycles; /* start index is arbitrary */ |
| 157 |
tmp = *from; /* save entry to be overwritten */ |
| 158 |
for (;;) { |
| 159 |
to = from; /* next step in cycle */ |
| 160 |
from += rot; /* go right rot positions */ |
| 161 |
if (from > last) from -= len; /* (pointer better not wrap) */ |
| 162 |
if (from == start) break; /* all but one shifted */ |
| 163 |
*to = *from; /* shift left */ |
| 164 |
} |
| 165 |
*to = tmp; /* complete the circle */ |
| 166 |
} while (--cycles); |
| 167 |
} |
| 168 |
|
| 169 |
/* structure for gzip file read operations */ |
| 170 |
typedef struct { |
| 171 |
int fd; /* file descriptor */ |
| 172 |
int size; /* 1 << size is bytes in buf */ |
| 173 |
unsigned left; /* bytes available at next */ |
| 174 |
unsigned char *buf; /* buffer */ |
| 175 |
z_const unsigned char *next; /* next byte in buffer */ |
| 176 |
char *name; /* file name for error messages */ |
| 177 |
} file; |
| 178 |
|
| 179 |
/* reload buffer */ |
| 180 |
local int readin(file *in) |
| 181 |
{ |
| 182 |
int len; |
| 183 |
|
| 184 |
len = read(in->fd, in->buf, 1 << in->size); |
| 185 |
if (len == -1) bye("error reading ", in->name); |
| 186 |
in->left = (unsigned)len; |
| 187 |
in->next = in->buf; |
| 188 |
return len; |
| 189 |
} |
| 190 |
|
| 191 |
/* read from file in, exit if end-of-file */ |
| 192 |
local int readmore(file *in) |
| 193 |
{ |
| 194 |
if (readin(in) == 0) bye("unexpected end of ", in->name); |
| 195 |
return 0; |
| 196 |
} |
| 197 |
|
| 198 |
#define read1(in) (in->left == 0 ? readmore(in) : 0, \ |
| 199 |
in->left--, *(in->next)++) |
| 200 |
|
| 201 |
/* skip over n bytes of in */ |
| 202 |
local void skip(file *in, unsigned n) |
| 203 |
{ |
| 204 |
unsigned bypass; |
| 205 |
|
| 206 |
if (n > in->left) { |
| 207 |
n -= in->left; |
| 208 |
bypass = n & ~((1U << in->size) - 1); |
| 209 |
if (bypass) { |
| 210 |
if (lseek(in->fd, (off_t)bypass, SEEK_CUR) == -1) |
| 211 |
bye("seeking ", in->name); |
| 212 |
n -= bypass; |
| 213 |
} |
| 214 |
readmore(in); |
| 215 |
if (n > in->left) |
| 216 |
bye("unexpected end of ", in->name); |
| 217 |
} |
| 218 |
in->left -= n; |
| 219 |
in->next += n; |
| 220 |
} |
| 221 |
|
| 222 |
/* read a four-byte unsigned integer, little-endian, from in */ |
| 223 |
unsigned long read4(file *in) |
| 224 |
{ |
| 225 |
unsigned long val; |
| 226 |
|
| 227 |
val = read1(in); |
| 228 |
val += (unsigned)read1(in) << 8; |
| 229 |
val += (unsigned long)read1(in) << 16; |
| 230 |
val += (unsigned long)read1(in) << 24; |
| 231 |
return val; |
| 232 |
} |
| 233 |
|
| 234 |
/* skip over gzip header */ |
| 235 |
local void gzheader(file *in) |
| 236 |
{ |
| 237 |
int flags; |
| 238 |
unsigned n; |
| 239 |
|
| 240 |
if (read1(in) != 31 || read1(in) != 139) bye(in->name, " not a gzip file"); |
| 241 |
if (read1(in) != 8) bye("unknown compression method in", in->name); |
| 242 |
flags = read1(in); |
| 243 |
if (flags & 0xe0) bye("unknown header flags set in", in->name); |
| 244 |
skip(in, 6); |
| 245 |
if (flags & 4) { |
| 246 |
n = read1(in); |
| 247 |
n += (unsigned)(read1(in)) << 8; |
| 248 |
skip(in, n); |
| 249 |
} |
| 250 |
if (flags & 8) while (read1(in) != 0) ; |
| 251 |
if (flags & 16) while (read1(in) != 0) ; |
| 252 |
if (flags & 2) skip(in, 2); |
| 253 |
} |
| 254 |
|
| 255 |
/* decompress gzip file "name", return strm with a deflate stream ready to |
| 256 |
continue compression of the data in the gzip file, and return a file |
| 257 |
descriptor pointing to where to write the compressed data -- the deflate |
| 258 |
stream is initialized to compress using level "level" */ |
| 259 |
local int gzscan(char *name, z_stream *strm, int level) |
| 260 |
{ |
| 261 |
int ret, lastbit, left, full; |
| 262 |
unsigned have; |
| 263 |
unsigned long crc, tot; |
| 264 |
unsigned char *window; |
| 265 |
off_t lastoff, end; |
| 266 |
file gz; |
| 267 |
|
| 268 |
/* open gzip file */ |
| 269 |
gz.name = name; |
| 270 |
gz.fd = open(name, O_RDWR, 0); |
| 271 |
if (gz.fd == -1) bye("cannot open ", name); |
| 272 |
gz.buf = malloc(CHUNK); |
| 273 |
if (gz.buf == NULL) bye("out of memory", ""); |
| 274 |
gz.size = LGCHUNK; |
| 275 |
gz.left = 0; |
| 276 |
|
| 277 |
/* skip gzip header */ |
| 278 |
gzheader(&gz); |
| 279 |
|
| 280 |
/* prepare to decompress */ |
| 281 |
window = malloc(DSIZE); |
| 282 |
if (window == NULL) bye("out of memory", ""); |
| 283 |
strm->zalloc = Z_NULL; |
| 284 |
strm->zfree = Z_NULL; |
| 285 |
strm->opaque = Z_NULL; |
| 286 |
ret = inflateInit2(strm, -15); |
| 287 |
if (ret != Z_OK) bye("out of memory", " or library mismatch"); |
| 288 |
|
| 289 |
/* decompress the deflate stream, saving append information */ |
| 290 |
lastbit = 0; |
| 291 |
lastoff = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; |
| 292 |
left = 0; |
| 293 |
strm->avail_in = gz.left; |
| 294 |
strm->next_in = gz.next; |
| 295 |
crc = crc32(0L, Z_NULL, 0); |
| 296 |
have = full = 0; |
| 297 |
do { |
| 298 |
/* if needed, get more input */ |
| 299 |
if (strm->avail_in == 0) { |
| 300 |
readmore(&gz); |
| 301 |
strm->avail_in = gz.left; |
| 302 |
strm->next_in = gz.next; |
| 303 |
} |
| 304 |
|
| 305 |
/* set up output to next available section of sliding window */ |
| 306 |
strm->avail_out = DSIZE - have; |
| 307 |
strm->next_out = window + have; |
| 308 |
|
| 309 |
/* inflate and check for errors */ |
| 310 |
ret = inflate(strm, Z_BLOCK); |
| 311 |
if (ret == Z_STREAM_ERROR) bye("internal stream error!", ""); |
| 312 |
if (ret == Z_MEM_ERROR) bye("out of memory", ""); |
| 313 |
if (ret == Z_DATA_ERROR) |
| 314 |
bye("invalid compressed data--format violated in", name); |
| 315 |
|
| 316 |
/* update crc and sliding window pointer */ |
| 317 |
crc = crc32(crc, window + have, DSIZE - have - strm->avail_out); |
| 318 |
if (strm->avail_out) |
| 319 |
have = DSIZE - strm->avail_out; |
| 320 |
else { |
| 321 |
have = 0; |
| 322 |
full = 1; |
| 323 |
} |
| 324 |
|
| 325 |
/* process end of block */ |
| 326 |
if (strm->data_type & 128) { |
| 327 |
if (strm->data_type & 64) |
| 328 |
left = strm->data_type & 0x1f; |
| 329 |
else { |
| 330 |
lastbit = strm->data_type & 0x1f; |
| 331 |
lastoff = lseek(gz.fd, 0L, SEEK_CUR) - strm->avail_in; |
| 332 |
} |
| 333 |
} |
| 334 |
} while (ret != Z_STREAM_END); |
| 335 |
inflateEnd(strm); |
| 336 |
gz.left = strm->avail_in; |
| 337 |
gz.next = strm->next_in; |
| 338 |
|
| 339 |
/* save the location of the end of the compressed data */ |
| 340 |
end = lseek(gz.fd, 0L, SEEK_CUR) - gz.left; |
| 341 |
|
| 342 |
/* check gzip trailer and save total for deflate */ |
| 343 |
if (crc != read4(&gz)) |
| 344 |
bye("invalid compressed data--crc mismatch in ", name); |
| 345 |
tot = strm->total_out; |
| 346 |
if ((tot & 0xffffffffUL) != read4(&gz)) |
| 347 |
bye("invalid compressed data--length mismatch in", name); |
| 348 |
|
| 349 |
/* if not at end of file, warn */ |
| 350 |
if (gz.left || readin(&gz)) |
| 351 |
fprintf(stderr, |
| 352 |
"gzappend warning: junk at end of gzip file overwritten\n"); |
| 353 |
|
| 354 |
/* clear last block bit */ |
| 355 |
lseek(gz.fd, lastoff - (lastbit != 0), SEEK_SET); |
| 356 |
if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); |
| 357 |
*gz.buf = (unsigned char)(*gz.buf ^ (1 << ((8 - lastbit) & 7))); |
| 358 |
lseek(gz.fd, -1L, SEEK_CUR); |
| 359 |
if (write(gz.fd, gz.buf, 1) != 1) bye("writing after seek to ", name); |
| 360 |
|
| 361 |
/* if window wrapped, build dictionary from window by rotating */ |
| 362 |
if (full) { |
| 363 |
rotate(window, DSIZE, have); |
| 364 |
have = DSIZE; |
| 365 |
} |
| 366 |
|
| 367 |
/* set up deflate stream with window, crc, total_in, and leftover bits */ |
| 368 |
ret = deflateInit2(strm, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); |
| 369 |
if (ret != Z_OK) bye("out of memory", ""); |
| 370 |
deflateSetDictionary(strm, window, have); |
| 371 |
strm->adler = crc; |
| 372 |
strm->total_in = tot; |
| 373 |
if (left) { |
| 374 |
lseek(gz.fd, --end, SEEK_SET); |
| 375 |
if (read(gz.fd, gz.buf, 1) != 1) bye("reading after seek on ", name); |
| 376 |
deflatePrime(strm, 8 - left, *gz.buf); |
| 377 |
} |
| 378 |
lseek(gz.fd, end, SEEK_SET); |
| 379 |
|
| 380 |
/* clean up and return */ |
| 381 |
free(window); |
| 382 |
free(gz.buf); |
| 383 |
return gz.fd; |
| 384 |
} |
| 385 |
|
| 386 |
/* append file "name" to gzip file gd using deflate stream strm -- if last |
| 387 |
is true, then finish off the deflate stream at the end */ |
| 388 |
local void gztack(char *name, int gd, z_stream *strm, int last) |
| 389 |
{ |
| 390 |
int fd, len, ret; |
| 391 |
unsigned left; |
| 392 |
unsigned char *in, *out; |
| 393 |
|
| 394 |
/* open file to compress and append */ |
| 395 |
fd = 0; |
| 396 |
if (name != NULL) { |
| 397 |
fd = open(name, O_RDONLY, 0); |
| 398 |
if (fd == -1) |
| 399 |
fprintf(stderr, "gzappend warning: %s not found, skipping ...\n", |
| 400 |
name); |
| 401 |
} |
| 402 |
|
| 403 |
/* allocate buffers */ |
| 404 |
in = malloc(CHUNK); |
| 405 |
out = malloc(CHUNK); |
| 406 |
if (in == NULL || out == NULL) bye("out of memory", ""); |
| 407 |
|
| 408 |
/* compress input file and append to gzip file */ |
| 409 |
do { |
| 410 |
/* get more input */ |
| 411 |
len = read(fd, in, CHUNK); |
| 412 |
if (len == -1) { |
| 413 |
fprintf(stderr, |
| 414 |
"gzappend warning: error reading %s, skipping rest ...\n", |
| 415 |
name); |
| 416 |
len = 0; |
| 417 |
} |
| 418 |
strm->avail_in = (unsigned)len; |
| 419 |
strm->next_in = in; |
| 420 |
if (len) strm->adler = crc32(strm->adler, in, (unsigned)len); |
| 421 |
|
| 422 |
/* compress and write all available output */ |
| 423 |
do { |
| 424 |
strm->avail_out = CHUNK; |
| 425 |
strm->next_out = out; |
| 426 |
ret = deflate(strm, last && len == 0 ? Z_FINISH : Z_NO_FLUSH); |
| 427 |
left = CHUNK - strm->avail_out; |
| 428 |
while (left) { |
| 429 |
len = write(gd, out + CHUNK - strm->avail_out - left, left); |
| 430 |
if (len == -1) bye("writing gzip file", ""); |
| 431 |
left -= (unsigned)len; |
| 432 |
} |
| 433 |
} while (strm->avail_out == 0 && ret != Z_STREAM_END); |
| 434 |
} while (len != 0); |
| 435 |
|
| 436 |
/* write trailer after last entry */ |
| 437 |
if (last) { |
| 438 |
deflateEnd(strm); |
| 439 |
out[0] = (unsigned char)(strm->adler); |
| 440 |
out[1] = (unsigned char)(strm->adler >> 8); |
| 441 |
out[2] = (unsigned char)(strm->adler >> 16); |
| 442 |
out[3] = (unsigned char)(strm->adler >> 24); |
| 443 |
out[4] = (unsigned char)(strm->total_in); |
| 444 |
out[5] = (unsigned char)(strm->total_in >> 8); |
| 445 |
out[6] = (unsigned char)(strm->total_in >> 16); |
| 446 |
out[7] = (unsigned char)(strm->total_in >> 24); |
| 447 |
len = 8; |
| 448 |
do { |
| 449 |
ret = write(gd, out + 8 - len, len); |
| 450 |
if (ret == -1) bye("writing gzip file", ""); |
| 451 |
len -= ret; |
| 452 |
} while (len); |
| 453 |
close(gd); |
| 454 |
} |
| 455 |
|
| 456 |
/* clean up and return */ |
| 457 |
free(out); |
| 458 |
free(in); |
| 459 |
if (fd > 0) close(fd); |
| 460 |
} |
| 461 |
|
| 462 |
/* process the compression level option if present, scan the gzip file, and |
| 463 |
append the specified files, or append the data from stdin if no other file |
| 464 |
names are provided on the command line -- the gzip file must be writable |
| 465 |
and seekable */ |
| 466 |
int main(int argc, char **argv) |
| 467 |
{ |
| 468 |
int gd, level; |
| 469 |
z_stream strm; |
| 470 |
|
| 471 |
/* ignore command name */ |
| 472 |
argc--; argv++; |
| 473 |
|
| 474 |
/* provide usage if no arguments */ |
| 475 |
if (*argv == NULL) { |
| 476 |
printf( |
| 477 |
"gzappend 1.2 (11 Oct 2012) Copyright (C) 2003, 2012 Mark Adler\n" |
| 478 |
); |
| 479 |
printf( |
| 480 |
"usage: gzappend [-level] file.gz [ addthis [ andthis ... ]]\n"); |
| 481 |
return 0; |
| 482 |
} |
| 483 |
|
| 484 |
/* set compression level */ |
| 485 |
level = Z_DEFAULT_COMPRESSION; |
| 486 |
if (argv[0][0] == '-') { |
| 487 |
if (argv[0][1] < '0' || argv[0][1] > '9' || argv[0][2] != 0) |
| 488 |
bye("invalid compression level", ""); |
| 489 |
level = argv[0][1] - '0'; |
| 490 |
if (*++argv == NULL) bye("no gzip file name after options", ""); |
| 491 |
} |
| 492 |
|
| 493 |
/* prepare to append to gzip file */ |
| 494 |
gd = gzscan(*argv++, &strm, level); |
| 495 |
|
| 496 |
/* append files on command line, or from stdin if none */ |
| 497 |
if (*argv == NULL) |
| 498 |
gztack(NULL, gd, &strm, 1); |
| 499 |
else |
| 500 |
do { |
| 501 |
gztack(*argv, gd, &strm, argv[1] == NULL); |
| 502 |
} while (*++argv != NULL); |
| 503 |
return 0; |
| 504 |
} |
