| 1 |
/* gzjoin -- command to join gzip files into one gzip file |
| 2 |
|
| 3 |
Copyright (C) 2004, 2005, 2012 Mark Adler, all rights reserved |
| 4 |
version 1.2, 14 Aug 2012 |
| 5 |
|
| 6 |
This software is provided 'as-is', without any express or implied |
| 7 |
warranty. In no event will the author be held liable for any damages |
| 8 |
arising from the use of this software. |
| 9 |
|
| 10 |
Permission is granted to anyone to use this software for any purpose, |
| 11 |
including commercial applications, and to alter it and redistribute it |
| 12 |
freely, subject to the following restrictions: |
| 13 |
|
| 14 |
1. The origin of this software must not be misrepresented; you must not |
| 15 |
claim that you wrote the original software. If you use this software |
| 16 |
in a product, an acknowledgment in the product documentation would be |
| 17 |
appreciated but is not required. |
| 18 |
2. Altered source versions must be plainly marked as such, and must not be |
| 19 |
misrepresented as being the original software. |
| 20 |
3. This notice may not be removed or altered from any source distribution. |
| 21 |
|
| 22 |
Mark Adler madler@alumni.caltech.edu |
| 23 |
*/ |
| 24 |
|
| 25 |
/* |
| 26 |
* Change history: |
| 27 |
* |
| 28 |
* 1.0 11 Dec 2004 - First version |
| 29 |
* 1.1 12 Jun 2005 - Changed ssize_t to long for portability |
| 30 |
* 1.2 14 Aug 2012 - Clean up for z_const usage |
| 31 |
*/ |
| 32 |
|
| 33 |
/* |
| 34 |
gzjoin takes one or more gzip files on the command line and writes out a |
| 35 |
single gzip file that will uncompress to the concatenation of the |
| 36 |
uncompressed data from the individual gzip files. gzjoin does this without |
| 37 |
having to recompress any of the data and without having to calculate a new |
| 38 |
crc32 for the concatenated uncompressed data. gzjoin does however have to |
| 39 |
decompress all of the input data in order to find the bits in the compressed |
| 40 |
data that need to be modified to concatenate the streams. |
| 41 |
|
| 42 |
gzjoin does not do an integrity check on the input gzip files other than |
| 43 |
checking the gzip header and decompressing the compressed data. They are |
| 44 |
otherwise assumed to be complete and correct. |
| 45 |
|
| 46 |
Each joint between gzip files removes at least 18 bytes of previous trailer |
| 47 |
and subsequent header, and inserts an average of about three bytes to the |
| 48 |
compressed data in order to connect the streams. The output gzip file |
| 49 |
has a minimal ten-byte gzip header with no file name or modification time. |
| 50 |
|
| 51 |
This program was written to illustrate the use of the Z_BLOCK option of |
| 52 |
inflate() and the crc32_combine() function. gzjoin will not compile with |
| 53 |
versions of zlib earlier than 1.2.3. |
| 54 |
*/ |
| 55 |
|
| 56 |
#include <stdio.h> /* fputs(), fprintf(), fwrite(), putc() */ |
| 57 |
#include <stdlib.h> /* exit(), malloc(), free() */ |
| 58 |
#include <fcntl.h> /* open() */ |
| 59 |
#include <unistd.h> /* close(), read(), lseek() */ |
| 60 |
#include "zlib.h" |
| 61 |
/* crc32(), crc32_combine(), inflateInit2(), inflate(), inflateEnd() */ |
| 62 |
|
| 63 |
#define local static |
| 64 |
|
| 65 |
/* exit with an error (return a value to allow use in an expression) */ |
| 66 |
local int bail(char *why1, char *why2) |
| 67 |
{ |
| 68 |
fprintf(stderr, "gzjoin error: %s%s, output incomplete\n", why1, why2); |
| 69 |
exit(1); |
| 70 |
return 0; |
| 71 |
} |
| 72 |
|
| 73 |
/* -- simple buffered file input with access to the buffer -- */ |
| 74 |
|
| 75 |
#define CHUNK 32768 /* must be a power of two and fit in unsigned */ |
| 76 |
|
| 77 |
/* bin buffered input file type */ |
| 78 |
typedef struct { |
| 79 |
char *name; /* name of file for error messages */ |
| 80 |
int fd; /* file descriptor */ |
| 81 |
unsigned left; /* bytes remaining at next */ |
| 82 |
unsigned char *next; /* next byte to read */ |
| 83 |
unsigned char *buf; /* allocated buffer of length CHUNK */ |
| 84 |
} bin; |
| 85 |
|
| 86 |
/* close a buffered file and free allocated memory */ |
| 87 |
local void bclose(bin *in) |
| 88 |
{ |
| 89 |
if (in != NULL) { |
| 90 |
if (in->fd != -1) |
| 91 |
close(in->fd); |
| 92 |
if (in->buf != NULL) |
| 93 |
free(in->buf); |
| 94 |
free(in); |
| 95 |
} |
| 96 |
} |
| 97 |
|
| 98 |
/* open a buffered file for input, return a pointer to type bin, or NULL on |
| 99 |
failure */ |
| 100 |
local bin *bopen(char *name) |
| 101 |
{ |
| 102 |
bin *in; |
| 103 |
|
| 104 |
in = malloc(sizeof(bin)); |
| 105 |
if (in == NULL) |
| 106 |
return NULL; |
| 107 |
in->buf = malloc(CHUNK); |
| 108 |
in->fd = open(name, O_RDONLY, 0); |
| 109 |
if (in->buf == NULL || in->fd == -1) { |
| 110 |
bclose(in); |
| 111 |
return NULL; |
| 112 |
} |
| 113 |
in->left = 0; |
| 114 |
in->next = in->buf; |
| 115 |
in->name = name; |
| 116 |
return in; |
| 117 |
} |
| 118 |
|
| 119 |
/* load buffer from file, return -1 on read error, 0 or 1 on success, with |
| 120 |
1 indicating that end-of-file was reached */ |
| 121 |
local int bload(bin *in) |
| 122 |
{ |
| 123 |
long len; |
| 124 |
|
| 125 |
if (in == NULL) |
| 126 |
return -1; |
| 127 |
if (in->left != 0) |
| 128 |
return 0; |
| 129 |
in->next = in->buf; |
| 130 |
do { |
| 131 |
len = (long)read(in->fd, in->buf + in->left, CHUNK - in->left); |
| 132 |
if (len < 0) |
| 133 |
return -1; |
| 134 |
in->left += (unsigned)len; |
| 135 |
} while (len != 0 && in->left < CHUNK); |
| 136 |
return len == 0 ? 1 : 0; |
| 137 |
} |
| 138 |
|
| 139 |
/* get a byte from the file, bail if end of file */ |
| 140 |
#define bget(in) (in->left ? 0 : bload(in), \ |
| 141 |
in->left ? (in->left--, *(in->next)++) : \ |
| 142 |
bail("unexpected end of file on ", in->name)) |
| 143 |
|
| 144 |
/* get a four-byte little-endian unsigned integer from file */ |
| 145 |
local unsigned long bget4(bin *in) |
| 146 |
{ |
| 147 |
unsigned long val; |
| 148 |
|
| 149 |
val = bget(in); |
| 150 |
val += (unsigned long)(bget(in)) << 8; |
| 151 |
val += (unsigned long)(bget(in)) << 16; |
| 152 |
val += (unsigned long)(bget(in)) << 24; |
| 153 |
return val; |
| 154 |
} |
| 155 |
|
| 156 |
/* skip bytes in file */ |
| 157 |
local void bskip(bin *in, unsigned skip) |
| 158 |
{ |
| 159 |
/* check pointer */ |
| 160 |
if (in == NULL) |
| 161 |
return; |
| 162 |
|
| 163 |
/* easy case -- skip bytes in buffer */ |
| 164 |
if (skip <= in->left) { |
| 165 |
in->left -= skip; |
| 166 |
in->next += skip; |
| 167 |
return; |
| 168 |
} |
| 169 |
|
| 170 |
/* skip what's in buffer, discard buffer contents */ |
| 171 |
skip -= in->left; |
| 172 |
in->left = 0; |
| 173 |
|
| 174 |
/* seek past multiples of CHUNK bytes */ |
| 175 |
if (skip > CHUNK) { |
| 176 |
unsigned left; |
| 177 |
|
| 178 |
left = skip & (CHUNK - 1); |
| 179 |
if (left == 0) { |
| 180 |
/* exact number of chunks: seek all the way minus one byte to check |
| 181 |
for end-of-file with a read */ |
| 182 |
lseek(in->fd, skip - 1, SEEK_CUR); |
| 183 |
if (read(in->fd, in->buf, 1) != 1) |
| 184 |
bail("unexpected end of file on ", in->name); |
| 185 |
return; |
| 186 |
} |
| 187 |
|
| 188 |
/* skip the integral chunks, update skip with remainder */ |
| 189 |
lseek(in->fd, skip - left, SEEK_CUR); |
| 190 |
skip = left; |
| 191 |
} |
| 192 |
|
| 193 |
/* read more input and skip remainder */ |
| 194 |
bload(in); |
| 195 |
if (skip > in->left) |
| 196 |
bail("unexpected end of file on ", in->name); |
| 197 |
in->left -= skip; |
| 198 |
in->next += skip; |
| 199 |
} |
| 200 |
|
| 201 |
/* -- end of buffered input functions -- */ |
| 202 |
|
| 203 |
/* skip the gzip header from file in */ |
| 204 |
local void gzhead(bin *in) |
| 205 |
{ |
| 206 |
int flags; |
| 207 |
|
| 208 |
/* verify gzip magic header and compression method */ |
| 209 |
if (bget(in) != 0x1f || bget(in) != 0x8b || bget(in) != 8) |
| 210 |
bail(in->name, " is not a valid gzip file"); |
| 211 |
|
| 212 |
/* get and verify flags */ |
| 213 |
flags = bget(in); |
| 214 |
if ((flags & 0xe0) != 0) |
| 215 |
bail("unknown reserved bits set in ", in->name); |
| 216 |
|
| 217 |
/* skip modification time, extra flags, and os */ |
| 218 |
bskip(in, 6); |
| 219 |
|
| 220 |
/* skip extra field if present */ |
| 221 |
if (flags & 4) { |
| 222 |
unsigned len; |
| 223 |
|
| 224 |
len = bget(in); |
| 225 |
len += (unsigned)(bget(in)) << 8; |
| 226 |
bskip(in, len); |
| 227 |
} |
| 228 |
|
| 229 |
/* skip file name if present */ |
| 230 |
if (flags & 8) |
| 231 |
while (bget(in) != 0) |
| 232 |
; |
| 233 |
|
| 234 |
/* skip comment if present */ |
| 235 |
if (flags & 16) |
| 236 |
while (bget(in) != 0) |
| 237 |
; |
| 238 |
|
| 239 |
/* skip header crc if present */ |
| 240 |
if (flags & 2) |
| 241 |
bskip(in, 2); |
| 242 |
} |
| 243 |
|
| 244 |
/* write a four-byte little-endian unsigned integer to out */ |
| 245 |
local void put4(unsigned long val, FILE *out) |
| 246 |
{ |
| 247 |
putc(val & 0xff, out); |
| 248 |
putc((val >> 8) & 0xff, out); |
| 249 |
putc((val >> 16) & 0xff, out); |
| 250 |
putc((val >> 24) & 0xff, out); |
| 251 |
} |
| 252 |
|
| 253 |
/* Load up zlib stream from buffered input, bail if end of file */ |
| 254 |
local void zpull(z_streamp strm, bin *in) |
| 255 |
{ |
| 256 |
if (in->left == 0) |
| 257 |
bload(in); |
| 258 |
if (in->left == 0) |
| 259 |
bail("unexpected end of file on ", in->name); |
| 260 |
strm->avail_in = in->left; |
| 261 |
strm->next_in = in->next; |
| 262 |
} |
| 263 |
|
| 264 |
/* Write header for gzip file to out and initialize trailer. */ |
| 265 |
local void gzinit(unsigned long *crc, unsigned long *tot, FILE *out) |
| 266 |
{ |
| 267 |
fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out); |
| 268 |
*crc = crc32(0L, Z_NULL, 0); |
| 269 |
*tot = 0; |
| 270 |
} |
| 271 |
|
| 272 |
/* Copy the compressed data from name, zeroing the last block bit of the last |
| 273 |
block if clr is true, and adding empty blocks as needed to get to a byte |
| 274 |
boundary. If clr is false, then the last block becomes the last block of |
| 275 |
the output, and the gzip trailer is written. crc and tot maintains the |
| 276 |
crc and length (modulo 2^32) of the output for the trailer. The resulting |
| 277 |
gzip file is written to out. gzinit() must be called before the first call |
| 278 |
of gzcopy() to write the gzip header and to initialize crc and tot. */ |
| 279 |
local void gzcopy(char *name, int clr, unsigned long *crc, unsigned long *tot, |
| 280 |
FILE *out) |
| 281 |
{ |
| 282 |
int ret; /* return value from zlib functions */ |
| 283 |
int pos; /* where the "last block" bit is in byte */ |
| 284 |
int last; /* true if processing the last block */ |
| 285 |
bin *in; /* buffered input file */ |
| 286 |
unsigned char *start; /* start of compressed data in buffer */ |
| 287 |
unsigned char *junk; /* buffer for uncompressed data -- discarded */ |
| 288 |
z_off_t len; /* length of uncompressed data (support > 4 GB) */ |
| 289 |
z_stream strm; /* zlib inflate stream */ |
| 290 |
|
| 291 |
/* open gzip file and skip header */ |
| 292 |
in = bopen(name); |
| 293 |
if (in == NULL) |
| 294 |
bail("could not open ", name); |
| 295 |
gzhead(in); |
| 296 |
|
| 297 |
/* allocate buffer for uncompressed data and initialize raw inflate |
| 298 |
stream */ |
| 299 |
junk = malloc(CHUNK); |
| 300 |
strm.zalloc = Z_NULL; |
| 301 |
strm.zfree = Z_NULL; |
| 302 |
strm.opaque = Z_NULL; |
| 303 |
strm.avail_in = 0; |
| 304 |
strm.next_in = Z_NULL; |
| 305 |
ret = inflateInit2(&strm, -15); |
| 306 |
if (junk == NULL || ret != Z_OK) |
| 307 |
bail("out of memory", ""); |
| 308 |
|
| 309 |
/* inflate and copy compressed data, clear last-block bit if requested */ |
| 310 |
len = 0; |
| 311 |
zpull(&strm, in); |
| 312 |
start = in->next; |
| 313 |
last = start[0] & 1; |
| 314 |
if (last && clr) |
| 315 |
start[0] &= ~1; |
| 316 |
strm.avail_out = 0; |
| 317 |
for (;;) { |
| 318 |
/* if input used and output done, write used input and get more */ |
| 319 |
if (strm.avail_in == 0 && strm.avail_out != 0) { |
| 320 |
fwrite(start, 1, strm.next_in - start, out); |
| 321 |
start = in->buf; |
| 322 |
in->left = 0; |
| 323 |
zpull(&strm, in); |
| 324 |
} |
| 325 |
|
| 326 |
/* decompress -- return early when end-of-block reached */ |
| 327 |
strm.avail_out = CHUNK; |
| 328 |
strm.next_out = junk; |
| 329 |
ret = inflate(&strm, Z_BLOCK); |
| 330 |
switch (ret) { |
| 331 |
case Z_MEM_ERROR: |
| 332 |
bail("out of memory", ""); |
| 333 |
case Z_DATA_ERROR: |
| 334 |
bail("invalid compressed data in ", in->name); |
| 335 |
} |
| 336 |
|
| 337 |
/* update length of uncompressed data */ |
| 338 |
len += CHUNK - strm.avail_out; |
| 339 |
|
| 340 |
/* check for block boundary (only get this when block copied out) */ |
| 341 |
if (strm.data_type & 128) { |
| 342 |
/* if that was the last block, then done */ |
| 343 |
if (last) |
| 344 |
break; |
| 345 |
|
| 346 |
/* number of unused bits in last byte */ |
| 347 |
pos = strm.data_type & 7; |
| 348 |
|
| 349 |
/* find the next last-block bit */ |
| 350 |
if (pos != 0) { |
| 351 |
/* next last-block bit is in last used byte */ |
| 352 |
pos = 0x100 >> pos; |
| 353 |
last = strm.next_in[-1] & pos; |
| 354 |
if (last && clr) |
| 355 |
in->buf[strm.next_in - in->buf - 1] &= ~pos; |
| 356 |
} |
| 357 |
else { |
| 358 |
/* next last-block bit is in next unused byte */ |
| 359 |
if (strm.avail_in == 0) { |
| 360 |
/* don't have that byte yet -- get it */ |
| 361 |
fwrite(start, 1, strm.next_in - start, out); |
| 362 |
start = in->buf; |
| 363 |
in->left = 0; |
| 364 |
zpull(&strm, in); |
| 365 |
} |
| 366 |
last = strm.next_in[0] & 1; |
| 367 |
if (last && clr) |
| 368 |
in->buf[strm.next_in - in->buf] &= ~1; |
| 369 |
} |
| 370 |
} |
| 371 |
} |
| 372 |
|
| 373 |
/* update buffer with unused input */ |
| 374 |
in->left = strm.avail_in; |
| 375 |
in->next = in->buf + (strm.next_in - in->buf); |
| 376 |
|
| 377 |
/* copy used input, write empty blocks to get to byte boundary */ |
| 378 |
pos = strm.data_type & 7; |
| 379 |
fwrite(start, 1, in->next - start - 1, out); |
| 380 |
last = in->next[-1]; |
| 381 |
if (pos == 0 || !clr) |
| 382 |
/* already at byte boundary, or last file: write last byte */ |
| 383 |
putc(last, out); |
| 384 |
else { |
| 385 |
/* append empty blocks to last byte */ |
| 386 |
last &= ((0x100 >> pos) - 1); /* assure unused bits are zero */ |
| 387 |
if (pos & 1) { |
| 388 |
/* odd -- append an empty stored block */ |
| 389 |
putc(last, out); |
| 390 |
if (pos == 1) |
| 391 |
putc(0, out); /* two more bits in block header */ |
| 392 |
fwrite("\0\0\xff\xff", 1, 4, out); |
| 393 |
} |
| 394 |
else { |
| 395 |
/* even -- append 1, 2, or 3 empty fixed blocks */ |
| 396 |
switch (pos) { |
| 397 |
case 6: |
| 398 |
putc(last | 8, out); |
| 399 |
last = 0; |
| 400 |
case 4: |
| 401 |
putc(last | 0x20, out); |
| 402 |
last = 0; |
| 403 |
case 2: |
| 404 |
putc(last | 0x80, out); |
| 405 |
putc(0, out); |
| 406 |
} |
| 407 |
} |
| 408 |
} |
| 409 |
|
| 410 |
/* update crc and tot */ |
| 411 |
*crc = crc32_combine(*crc, bget4(in), len); |
| 412 |
*tot += (unsigned long)len; |
| 413 |
|
| 414 |
/* clean up */ |
| 415 |
inflateEnd(&strm); |
| 416 |
free(junk); |
| 417 |
bclose(in); |
| 418 |
|
| 419 |
/* write trailer if this is the last gzip file */ |
| 420 |
if (!clr) { |
| 421 |
put4(*crc, out); |
| 422 |
put4(*tot, out); |
| 423 |
} |
| 424 |
} |
| 425 |
|
| 426 |
/* join the gzip files on the command line, write result to stdout */ |
| 427 |
int main(int argc, char **argv) |
| 428 |
{ |
| 429 |
unsigned long crc, tot; /* running crc and total uncompressed length */ |
| 430 |
|
| 431 |
/* skip command name */ |
| 432 |
argc--; |
| 433 |
argv++; |
| 434 |
|
| 435 |
/* show usage if no arguments */ |
| 436 |
if (argc == 0) { |
| 437 |
fputs("gzjoin usage: gzjoin f1.gz [f2.gz [f3.gz ...]] > fjoin.gz\n", |
| 438 |
stderr); |
| 439 |
return 0; |
| 440 |
} |
| 441 |
|
| 442 |
/* join gzip files on command line and write to stdout */ |
| 443 |
gzinit(&crc, &tot, stdout); |
| 444 |
while (argc--) |
| 445 |
gzcopy(*argv++, argc, &crc, &tot, stdout); |
| 446 |
|
| 447 |
/* done */ |
| 448 |
return 0; |
| 449 |
} |
