Attachment #647336 for bug #645438

View | Details | Raw Unified | Return to bug 645438
Collapse All | Expand All




/* pigz.c -- parallel implementation of gzip
 * Copyright (C) 2007-2017 Mark Adler
 * Version 2.4.1x  xx Dec 2017  Mark Adler
 */

/*

                       Fix sign error in compression reduction percentage
 */

#define VERSION "pigz 2.4.1x"

/* To-do:
    - make source portable for Windows, VMS, etc. (see gzip source code)

#if __STDC_VERSION__-0 >= 199901L || __GNUC__-0 >= 3
#  include <inttypes.h> // intmax_t, uintmax_t
   typedef uintmax_t length_t;
   typedef uint32_t crc_t;
#else
   typedef unsigned long length_t;
   typedef unsigned long crc_t;
#endif

#ifdef PIGZ_DEBUG

#define INBUFS(p) (((p)<<1)+3)
#define OUTPOOL(s) ((s)+((s)>>4)+DICT)

// Input buffer size, and augmentation for re-inserting a central header.
#define BUF 32768
#define CEN 42
#define EXT (BUF + CEN)     // provide enough room to unget a header

// Globals (modified by main thread only when it's the only thread).
local struct {

    int force;              // true to overwrite, compress links, cat
    int sync;               // true to flush output file
    int form;               // gzip = 0, zlib = 1, zip = 2 or 3
    int magic1;             // first byte of possible header when decoding
    int recurse;            // true to dive down into directory structure
    char *sufx;             // suffix to use (".gz" or user supplied)
    char *name;             // name for gzip or zip header
    char *alias;            // name for zip header when input is stdin
    char *comment;          // comment for gzip or zip header.
    time_t mtime;           // time stamp from input file for gzip header
    int list;               // true to list files instead of compress
    int first;              // true if we need to print listing header

    int procs;              // maximum number of compression threads (>= 1)
    int setdict;            // true to initialize dictionary in each thread
    size_t block;           // uncompressed input size per thread (>= 32K)
    crc_t shift;            // pre-calculated CRC-32 shift for length block

    // saved gzip/zip header data for decompression, testing, and listing
    time_t stamp;           // time stamp from gzip header
    char *hname;            // name from header (allocated)
    char *hcomm;            // comment from header (allocated)
    unsigned long zip_crc;  // local header crc
    length_t zip_clen;      // local header compressed length
    length_t zip_ulen;      // local header uncompressed length
    int zip64;              // true if has zip64 extended information

    // globals for decompression and listing buffered reading
    unsigned char in_buf[EXT];  // input buffer
    unsigned char *in_next; // next unused byte in buffer
    size_t in_left;         // number of unused bytes in buffer
    int in_eof;             // true if reached end of file on input


#ifndef NOTHREAD
    // globals for decompression parallel reading
    unsigned char in_buf2[EXT]; // second buffer for parallel reads
    size_t in_len;          // data waiting in next buffer
    int in_which;           // -1: start, 0: in_buf2, 1: in_buf
    lock *load_state;       // value = 0 to wait, 1 to read a buffer


// Memory tracking.

#define MAXMEM 131072   // maximum number of tracked pointers

local struct mem_track_s {
    size_t num;         // current number of allocations
    size_t size;        // total size of current allocations
    size_t tot;         // maximum number of allocations
    size_t max;         // maximum size of allocations
#ifndef NOTHREAD
    lock *lock;         // lock for access across threads
#endif
    size_t have;        // number in array (possibly != num)
    void *mem[MAXMEM];  // sorted array of allocated pointers
} mem_track;

#ifndef NOTHREAD

#  define mem_track_drop(m)
#endif

// Return the leftmost insert location of ptr in the sorted list mem->mem[],
// which currently has mem->have elements. If ptr is already in the list, the
// returned value will point to its first occurrence. The return location will
// be one after the last element if ptr is greater than all of the elements.
local size_t search_track(struct mem_track_s *mem, void *ptr) {
    ptrdiff_t left = 0;
    ptrdiff_t right = mem->have - 1;
    while (left <= right) {
        ptrdiff_t mid = (left + right) >> 1;
        if (mem->mem[mid] < ptr)
            left = mid + 1;
        else
            right = mid - 1;
    }
    return left;
}

// Insert ptr in the sorted list mem->mem[] and update the memory allocation
// statistics.
local void insert_track(struct mem_track_s *mem, void *ptr) {
    mem_track_grab(mem);
    assert(mem->have < MAXMEM && "increase MAXMEM in source and try again");
    size_t i = search_track(mem, ptr);
    if (i < mem->have && mem->mem[i] == ptr)
        complain("mem_track: duplicate pointer %p\n", ptr);
    memmove(&mem->mem[i + 1], &mem->mem[i],
            (mem->have - i) * sizeof(void *));
    mem->mem[i] = ptr;
    mem->have++;
    mem->num++;
    mem->size += MALLOC_SIZE(ptr);
    if (mem->num > mem->tot)
        mem->tot = mem->num;
    if (mem->size > mem->max)
        mem->max = mem->size;
    mem_track_drop(mem);
}

// Find and delete ptr from the sorted list mem->mem[] and update the memory
// allocation statistics.
local void delete_track(struct mem_track_s *mem, void *ptr) {
    mem_track_grab(mem);
    size_t i = search_track(mem, ptr);
    if (i < mem->num && mem->mem[i] == ptr) {
        memmove(&mem->mem[i], &mem->mem[i + 1],
                (mem->have - (i + 1)) * sizeof(void *));
        mem->have--;
    }
    else
        complain("mem_track: missing pointer %p\n", ptr);
    mem->num--;
    mem->size -= MALLOC_SIZE(ptr);
    mem_track_drop(mem);
}

local void *malloc_track(struct mem_track_s *mem, size_t size) {
    void *ptr = malloc(size);
    if (ptr != NULL)
        insert_track(mem, ptr);
    return ptr;
}

local void *realloc_track(struct mem_track_s *mem, void *ptr, size_t size) {
    size_t was;

    if (ptr == NULL)
        return malloc_track(mem, size);
    delete_track(mem, ptr);
    void *got = realloc(ptr, size);
    insert_track(mem, got == NULL ? ptr : got);
    return got;
        mem_track_grab(mem);
        mem->size -= was;
        mem->size += size;
        if (mem->size > mem->max)
            mem->max = mem->size;
        mem_track_drop(mem);
    }
    return ptr;
}

local void free_track(struct mem_track_s *mem, void *ptr) {
    size_t size;

    if (ptr != NULL) {
        delete_track(mem, ptr);
        mem_track_grab(mem);
        mem->num--;
        mem->size -= size;
        mem_track_drop(mem);
        free(ptr);
    }
}

    if (log_tail == NULL) {
        mem_track.num = 0;
        mem_track.size = 0;
        mem_track.num = 0;
        mem_track.max = 0;
        mem_track.have = 0;
#ifndef NOTHREAD
        mem_track.lock = new_lock(0);
        yarn_mem(yarn_malloc, yarn_free);

        complain("memory leak: %lu allocs of %lu bytes total",
                 mem_track.num, mem_track.size);
    if (mem_track.max)
        fprintf(stderr, "%lu bytes of memory used in %lu allocs\n",
                mem_track.max, mem_track.tot);
}

// Debugging macro.


    // write wrap[] to out and return the number of bytes written
    writen(out, wrap, count);
    FREE(wrap);
    return count;
}


            4, (val_t)0,            // crc (not here)
            4, (val_t)LOW32,        // compressed length (not here)
            4, (val_t)LOW32,        // uncompressed length (not here)
            2, (val_t)(strlen(g.name == NULL ? g.alias : g.name)),  // name len
            2, (val_t)29,           // length of extra field (see below)
            0);

        // write file name (use g.alias for stdin)
        len += writen(g.outd, g.name == NULL ? g.alias : g.name,
                      strlen(g.name == NULL ? g.alias : g.name));

        // write Zip64 and extended timestamp extra field blocks (29 bytes)
        len += put(g.outd,

            0);
    }
    else if (g.form) {              // zlib
        if (g.comment != NULL)
            complain("can't store comment in zlib format -- ignoring");
        unsigned head;
        head = (0x78 << 8) +        // deflate, 32K window
               (g.level >= 9 ? 3 << 6 :

            1, (val_t)31,
            1, (val_t)139,
            1, (val_t)8,            // deflate
            1, (val_t)((g.name != NULL ? 8 : 0) +
                       (g.comment != NULL ? 16 : 0)),
            4, (val_t)g.mtime,
            1, (val_t)(g.level >= 9 ? 2 : g.level == 1 ? 4 : 0),
            1, (val_t)3,            // unix
            0);
        if (g.name != NULL)
            len += writen(g.outd, g.name, strlen(g.name) + 1);
        if (g.comment != NULL)
            len += writen(g.outd, g.comment, strlen(g.comment) + 1);
    }
    return len;
}

            4, (val_t)check,        // crc
            4, (val_t)(zip64 ? LOW32 : clen),   // compressed length
            4, (val_t)(zip64 ? LOW32 : ulen),   // uncompressed length
            2, (val_t)(strlen(g.name == NULL ? g.alias : g.name)),  // name len
            2, (val_t)(zip64 ? 29 : 9), // extra field size (see below)
            2, (val_t)(g.comment == NULL ? 0 : strlen(g.comment)),  // comment
            2, (val_t)0,            // disk number 0
            2, (val_t)0,            // internal file attributes
            4, (val_t)0,            // external file attributes (ignored)
            4, (val_t)0,            // offset of local header
            0);

        // write file name (use g.alias for stdin)
        cent += writen(g.outd, g.name == NULL ? g.alias : g.name,
                       strlen(g.name == NULL ? g.alias : g.name));

        // write Zip64 extra field block (20 bytes)
        if (zip64)

            4, (val_t)g.mtime,      // mod time
            0);

        // write comment, if requested
        if (g.comment != NULL)
            cent += writen(g.outd, g.comment, strlen(g.comment));

        // here zip64 is true if the offset of the central directory does not
        // fit in 32 bits, in which case insert the Zip64 end records to
        // provide a 64-bit offset

// We copy the combination routines from zlib here, in order to avoid linkage
// issues with the zlib 1.2.3 builds on Sun, Ubuntu, and others.

// CRC-32 polynomial, reflected.
#define POLY 0xedb88320

// Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC
// polynomial, reflected. For speed, this requires that a not be zero.
local crc_t multmodp(crc_t a, crc_t b) {
    crc_t m = (crc_t)1 << 31;
    crc_t p = 0;
    for (;;) {
        if (a & m) {
            p ^= b;
            if ((a & (m - 1)) == 0)
                break;
        }
        m >>= 1;
        b = b & 1 ? (b >> 1) ^ POLY : b >> 1;
    }
    return p;
}

// Table of x^2^n modulo p(x).
local const crc_t x2n_table[] = {
    0x40000000, 0x20000000, 0x08000000, 0x00800000, 0x00008000,
    0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467,
    0xd7bbfe6a, 0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0,
    0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3, 0x31fec169,
    0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37,
    0x2e4e5eef, 0x4eaba214, 0xa8a472c0, 0x429a969e, 0x148d302a,
    0xc40ba6d0, 0xc4e22c3c};

// Return x^(n*2^k) modulo p(x).
local crc_t x2nmodp(size_t n, unsigned k) {
    crc_t p = (crc_t)1 << 31;       // x^0 == 1
    while (n) {
        if (n & 1)
            p = multmodp(x2n_table[k & 31], p);
        n >>= 1;
        k++;
    }
    return p;
}

// This uses the pre-computed g.shift value most of the time. Only the last
// combination requires a new x2nmodp() calculation.
local unsigned long crc32_comb(unsigned long crc1, unsigned long crc2,
                               size_t len2) {
    return multmodp(len2 == g.block ? g.shift : x2nmodp(len2, 3), crc1) ^ crc2;
    unsigned long row;
    unsigned long even[32];     // even-power-of-two zeros operator
    unsigned long odd[32];      // odd-power-of-two zeros operator

    // degenerate case
    if (len2 == 0)
        return crc1;

    // put operator for one zero bit in odd
    odd[0] = 0xedb88320UL;          // CRC-32 polynomial
    row = 1;
    for (n = 1; n < 32; n++) {
        odd[n] = row;
        row <<= 1;
    }

    // put operator for two zero bits in even
    gf2_matrix_square(even, odd);

    // put operator for four zero bits in odd
    gf2_matrix_square(odd, even);

    // apply len2 zeros to crc1 (first square will put the operator for one
    // zero byte, eight zero bits, in even)
    do {
        // apply zeros operator for this bit of len2
        gf2_matrix_square(even, odd);
        if (len2 & 1)
            crc1 = gf2_matrix_times(even, crc1);
        len2 >>= 1;

        // if no more bits set, then done
        if (len2 == 0)
            break;

        // another iteration of the loop with odd and even swapped
        gf2_matrix_square(odd, even);
        if (len2 & 1)
            crc1 = gf2_matrix_times(odd, crc1);
        len2 >>= 1;

        // if no more bits set, then done
    } while (len2 != 0);

    // return combined crc
    crc1 ^= crc2;
    return crc1;
}

#define BASE 65521U     // largest prime smaller than 65536

    // if a space is available, pull it from the list and return it
    if (pool->head != NULL) {
        space = pool->head;
        possess(space->use);
        pool->head = space->next;
        twist(pool->have, BY, -1);      // one less in pool
        possess(space->use);
        twist(space->use, TO, 1);       // initially one user
        space->len = 0;
        return space;

// Increment the use count to require one more drop before returning this space
// to the pool.
local void use_space(struct space *space) {
    long use;

    possess(space->use);
    use = peek_lock(space->use);
    assert(use != 0);
    twist(space->use, BY, +1);
}


    possess(space->use);
    use = peek_lock(space->use);
    assert(use != 0);
    twist(space->use, BY, -1);
    if (use == 1) {
        pool = space->pool;
        possess(pool->have);

        pool->head = space;
        twist(pool->have, BY, +1);
    }
    twist(space->use, BY, -1);
}

// Free the memory and lock resources of a pool. Return number of spaces for

            wait_for(job->calc, TO_BE, 1);
            release(job->calc);
            check = COMB(check, job->check, len);
            Trace(("-- combined #%ld%s", seq, more ? "" : " (last)"));

            // free the job
            free_lock(job->calc);

                    }
                }
                else
#else
                {
                    DEFLATE_WRITE(Z_SYNC_FLUSH);
#else
                DEFLATE_WRITE(Z_SYNC_FLUSH);
#endif
                if (!g.setdict)             // two markers when independent
                    DEFLATE_WRITE(Z_FULL_FLUSH);

}

// Terminate the load() operation. Empty buffer, mark end, close file (if not
// stdin), and free the name and comment obtained from the header, if present.
local void load_end(void) {
#ifndef NOTHREAD
    // if the read thread is running, then end it

    if (g.ind != 0)
        close(g.ind);
    RELEASE(g.hname);
    RELEASE(g.hcomm);
}

// Initialize for reading new input.

// Buffered reading macros for decompression and listing.
#define GET() (g.in_left == 0 && (g.in_eof || load() == 0) ? 0 : \
               (g.in_left--, *g.in_next++))
#define UNGET(n) (g.in_left += (n), g.in_next -= (n))
#define GET2() (tmp2 = GET(), tmp2 + ((unsigned)(GET()) << 8))
#define GET4() (tmp4 = GET2(), tmp4 + ((unsigned long)(GET2()) << 16))
#define SKIP(dist) \

        g.in_next += togo; \
    } while (0)

// Get a zero-terminated string into allocated memory, with crc update.
#define GETZC(str) \
    do { \
        unsigned char *end; \
        size_t copy, have, size = 0; \
        have = 0; \
        do { \
            if (g.in_left == 0 && load() == 0) \
                return -3; \
            end = memchr(g.in_next, 0, g.in_left); \
            copy = end == NULL ? g.in_left : (size_t)(end - g.in_next) + 1; \
            have = vmemcpy(&str, &size, have, g.in_next, copy); \
            g.in_left -= copy; \
            g.in_next += copy; \
        } while (end == NULL); \
        crc = crc32z(crc, (unsigned char *)str, have); \
    } while (0)

// Pull LSB order or MSB order integers from an unsigned char buffer.
#define PULL2L(p) ((p)[0] + ((unsigned)((p)[1]) << 8))
#define PULL4L(p) (PULL2L(p) + ((unsigned long)(PULL2L((p) + 2)) << 16))

        len -= size;
        if (id == 0x0001) {
            // Zip64 Extended Information Extra Field
            g.zip64 = 1;
            if (g.zip_ulen == LOW32 && size >= 8) {
                g.zip_ulen = GET4();
                SKIP(4);

    if (save) {
        g.stamp = 0;
        RELEASE(g.hname);
        RELEASE(g.hcomm);
    }

    // see if it's a gzip, zlib, or lzw file
    g.form = -1;
    g.magic1 = GET();
    if (g.in_eof) {
        g.magic1 = -1;
        return -1;
    }
    magic = (unsigned)g.magic1 << 8;
    magic += GET();
    if (g.in_eof)
        UNGET(1);
        return -2;
    }
    if (magic % 31 == 0 && (magic & 0x8f20) == 0x0800) {
        // it's zlib
        g.form = 1;
        return 8;
    }
    if (magic == 0x1f9d) {          // it's lzw
        g.form = -1;
        return 257;
    }
    if (magic == 0x504b) {          // it's zip
        magic = GET2();             // the rest of the signature
        if (g.in_eof)

            return -5;              // central header or archive extra
        if (magic != 0x0403)
            return -4;              // not a local header
        g.zip64 = 0;
        SKIP(2);
        flags = GET2();
        if (flags & 0xf7f0)
            return -4;
        method = GET();             // return low byte of method or 256
        if (GET() != 0 || flags & 1)

        return g.in_eof ? -3 : (int)method;
    }
    if (magic != 0x1f8b) {          // not gzip
        g.in_left++;                // return the second byte
        g.in_next--;
        return -2;
    }


        SKIPC(GET2C());

    // read file name, if present, into allocated memory
    if (flags & 8) {
        if (save)
            GETZC(g.hname);
        else
            while (GETC() != 0)
                ;
                return -3;
            end = memchr(g.in_next, 0, g.in_left);
            copy = end == NULL ? g.in_left : (size_t)(end - g.in_next) + 1;
            have = vmemcpy(&g.hname, &size, have, g.in_next, copy);
            g.in_left -= copy;
            g.in_next += copy;
        } while (end == NULL);
        crc = crc32z(crc, (unsigned char *)g.hname, have);
    }
    else if (flags & 8)
        while (GETC() != 0)
            ;

    // read comment, if present, into allocated memory
    if (flags & 16) {
        if (save)
            GETZC(g.hcomm);
        else
            while (GETC() != 0)
                ;
    }

    // check header crc
    if ((flags & 2) && GET2() != (crc & 0xffff))

    return g.in_eof ? -3 : (int)method;
}

// Process the remainder of a zip file after the first entry. Return true if
// the next signature is another local file header. If listing verbosely, then
// search the remainder of the zip file for the central file header
// corresponding to the first zip entry, and save the file comment, if any.
local int more_zip_entries(void) {
    unsigned long sig;
    int ret, n;
    unsigned char *first;
    unsigned tmp2;              // for macro
    unsigned long tmp4;         // for macro
    unsigned char const central[] = {0x50, 0x4b, 1, 2};

    sig = GET4();
    ret = !g.in_eof && sig == 0x04034b50;   // true if another entry follows
    if (!g.list || g.verbosity < 2)
        return ret;

    // if it was a central file header signature, then already four bytes
    // into a central directory header -- otherwise search for the next one
    n = sig == 0x02014b50 ? 4 : 0;  // number of bytes into central header
    for (;;) {
        // assure that more input is available
        if (g.in_left == 0 && load() == 0)      // never found it!
            return ret;
        if (n == 0) {
            // look for first byte in central signature
            first = memchr(g.in_next, central[0], g.in_left);
            if (first == NULL) {
                // not found -- go get the next buffer and keep looking
                g.in_left = 0;
            }
            else {
                // found -- continue search at next byte
                n++;
                g.in_left -= first - g.in_next + 1;
                g.in_next = first + 1;
            }
        }
        else if (n < 4) {
            // look for the remaining bytes in the central signature
            if (g.in_next[0] == central[n]) {
                n++;
                g.in_next++;
                g.in_left--;
            }
            else
                n = 0;      // mismatch -- restart search with this byte
        }
        else {
            // Now in a suspected central file header, just past the signature.
            // Read the rest of the fixed-length portion of the header.
            unsigned char head[CEN];
            size_t need = CEN, part = 0, len, i;

            if (need > g.in_left) {     // will only need to do this once
                part = g.in_left;
                memcpy(head + CEN - need, g.in_next, part);
                need -= part;
                g.in_left = 0;
                if (load() == 0)                // never found it!
                    return ret;
            }
            memcpy(head + CEN - need, g.in_next, need);

            // Determine to sufficient probability that this is the droid we're
            // looking for, by checking the CRC and the local header offset.
            if (PULL4L(head + 12) == g.out_check && PULL4L(head + 38) == 0) {
                // Update the number of bytes consumed from the current buffer.
                g.in_next += need;
                g.in_left -= need;

                // Get the comment length.
                len = PULL2L(head + 28);
                if (len == 0)                   // no comment
                    return ret;

                // Skip the file name and extra field.
                SKIP(PULL2L(head + 24) + (unsigned long)PULL2L(head + 26));

                // Save the comment field.
                need = len;
                g.hcomm = alloc(NULL, len + 1);
                while (need > g.in_left) {
                    memcpy(g.hcomm + len - need, g.in_next, g.in_left);
                    need -= g.in_left;
                    g.in_left = 0;
                    if (load() == 0) {          // premature EOF
                        RELEASE(g.hcomm);
                        return ret;
                    }
                }
                memcpy(g.hcomm + len - need, g.in_next, need);
                g.in_next += need;
                g.in_left -= need;
                for (i = 0; i < len; i++)
                    if (g.hcomm[i] == 0)
                        g.hcomm[i] = ' ';
                g.hcomm[len] = 0;
                return ret;
            }
            else {
                // Nope, false alarm. Restart the search at the first byte
                // after what we thought was the central file header signature.
                if (part) {
                    // Move buffer data up and insert the part of the header
                    // data read from the previous buffer.
                    memmove(g.in_next + part, g.in_next, g.in_left);
                    memcpy(g.in_next, head, part);
                    g.in_left += part;
                }
                n = 0;
            }
        }
    }
}

// --- list contents of compressed input (gzip, zlib, or lzw) ---

// Find standard compressed file suffix, return length of suffix.

#define NAMEMAX1 48     // name display limit at verbosity 1
#define NAMEMAX2 16     // name display limit at verbosity 2

// Print gzip, lzw, zlib, or zip file information.
local void show_info(int method, unsigned long check, length_t len, int cont) {
    size_t max;             // maximum name length for current verbosity
    size_t n;               // name length without suffix

                   g.in_tot, len, red, tag);
#endif
    }
    if (g.verbosity > 1 && g.hcomm != NULL)
        puts(g.hcomm);
}

// List content information about the gzip file at ind (only works if the gzip


    // list zip file
    if (g.form > 1) {
        more_zip_entries();         // get first entry comment, if any
        g.in_tot = g.zip_clen;
        show_info(method, g.zip_crc, g.zip_ulen, 0);
        return;

// --- copy input to output (when acting like cat) ---

local void cat(void) {
    // copy the first header byte read, if any
    if (g.magic1 != -1) {
        unsigned char buf[1] = {g.magic1};
        g.out_tot += writen(g.outd, buf, 1);
    }

    // copy the remainder of the input to the output
    while (g.in_left) {
        g.out_tot += writen(g.outd, g.in_next, g.in_left);

// write and check threads and return for more decompression while that's going
// on (or just write and check if no threads or if proc == 1).
local int outb(void *desc, unsigned char *buf, unsigned len) {
    (void)desc;

#ifndef NOTHREAD
    static thread *wr, *ch;


        // if requested with len == 0, clean up -- terminate and join write and
        // check threads, free lock
        if (len == 0 && outb_write_more != NULL) {
            join(ch);
            join(wr);
                destruct(wr);
            }
            else {
                join(ch);
                join(wr);
            }
            free_lock(outb_check_more);
            free_lock(outb_write_more);
            outb_write_more = NULL;

    }
#endif

    (void)desc;

    // if just one process or no threads, then do it without threads
    if (len) {
        if (g.decode == 1)

    return 0;
}

// Zip file data descriptor signature. This signature may or may not precede
// the CRC and lengths, with either resulting in a valid zip file! There is
// some odd code below that tries to detect and accommodate both cases.
#define SIG 0x08074b50

// Inflate for decompression or testing. Decompress from ind to outd unless
// decode != 1, in which case just test ind, and then also list if list != 0;
// look for and decode multiple, concatenated gzip and/or zlib streams; read
// and check the gzip, zlib, or zip trailer.
local void infchk(void) {
    int ret, cont, more;
    unsigned long check, len;
    z_stream strm;
    unsigned tmp2;
    unsigned long tmp4;
    length_t clen;

    cont = more = 0;
    do {
        // header already read -- set up for decompression
        g.in_tot = g.in_left;       // track compressed data length

        // read and check trailer
        if (g.form > 1) {           // zip local trailer (if any)
            if (g.form == 3) {      // data descriptor follows
                // get data descriptor values, assuming no signature
                g.zip_crc = GET4();
                g.zip_clen = GET4();
                g.zip_ulen = GET4();        // ZIP64 -> high clen, not ulen

                // deduce whether or not a signature precedes the values
                if (g.zip_crc == SIG &&         // might be the signature
                    // if the expected CRC is not SIG, then it's a signature
                    (g.out_check != SIG ||      // assume signature
                     // now we're in a very rare case where CRC == SIG -- the
                     // first four bytes could be the signature or the CRC
                     (g.zip_clen == SIG &&      // if not, then no signature
                      // now we have the first two words are SIG and the
                      // expected CRC is SIG, so it could be a signature and
                      // the CRC, or it could be the CRC and a compressed
                      // length that is *also* SIG (!) -- so check the low 32
                      // bits of the expected compressed length for SIG
                      ((clen & LOW32) != SIG || // assume signature and CRC
                       // now the expected CRC *and* the expected low 32 bits
                       // of the compressed length are SIG -- this is so
                       // incredibly unlikely, clearly someone is messing with
                       // us, but we continue ... if the next four bytes are
                       // not SIG, then there is not a signature -- check those
                       // bytes, currently in g.zip_ulen:
                       (g.zip_ulen == SIG &&    // if not, then no signature
                        // we have three SIGs in a row in the descriptor, and
                        // both the expected CRC and the expected clen are SIG
                        // -- the first one is a signature if we don't expect
                        // the third word to be SIG, which is either the low 32
                        // bits of ulen, or if ZIP64, the high 32 bits of clen:
                        (g.zip64 ? clen >> 32 : g.out_tot) != SIG
                        // if that last compare was equal, then the expected
                        // values for the CRC, the low 32 bits of clen, *and*
                        // the low 32 bits of ulen are all SIG (!!), or in the
                        // case of ZIP64, even crazier, the CRC and *both*
                        // 32-bit halves of clen are all SIG (clen > 500
                        // petabytes!!!) ... we can no longer discriminate the
                        // hypotheses, so we will assume no signature
                        ))))) {
                    // first four bytes were actually the descriptor -- shift
                    // the values down and get another four bytes
                    g.zip_crc = g.zip_clen;
                    g.zip_clen = g.zip_ulen;
                    g.zip_ulen = GET4();
                }

                // if ZIP64, then ulen is really the high word of clen -- get
                // the actual ulen and skip its high word as well (we only
                // compare the low 32 bits of the lengths to verify)
                if (g.zip64) {
                    g.zip_ulen = GET4();
                    (void)GET4();
                }

                    throw(EDOM, "%s: corrupted entry -- missing trailer",
                          g.inf);
            }
            check = g.zip_crc;
            if (check != g.out_check)
                throw(EDOM, "%s: corrupted entry -- crc32 mismatch", g.inf);
            if (g.zip_clen != (clen & LOW32) ||
                g.zip_ulen != (g.out_tot & LOW32))
                throw(EDOM, "%s: corrupted entry -- length mismatch",
                      g.inf);
            more = more_zip_entries();  // see if more entries, get comment
        }
        else if (g.form == 1) {     // zlib (big-endian) trailer
            check = (unsigned long)(GET()) << 24;


        // if a gzip entry follows a gzip entry, decompress it (don't replace
        // saved header information from first entry)
    } while (g.form == 0 && (ret = get_header(0)) == 8);
    } while (was == 0 && (ret = get_header(0)) == 8 && g.form == 0);

    // gzip -cdf copies junk after gzip stream directly to output
    if (g.form == 0 && ret == -2 && g.force && g.pipeout && g.decode != 2 &&
        !g.list)
        cat();

    // check for more entries in zip file
    else if (more)
        complain("warning: %s: entries after the first were ignored", g.inf);

    // check for non-gzip after gzip stream, or anything after zlib stream
    else if ((g.verbosity > 1 && g.form == 0 && ret != -1) ||
             (g.form == 1 && (GET(), !g.in_eof)))
        complain("warning: %s: trailing junk was ignored", g.inf);
}


                    punt(err);
                complain("skipping: %s", err.why);
                drop(err);
                outb(NULL, NULL, 0);
            }
            load_end();
            return;

                punt(err);
            complain("skipping: %s", err.why);
            drop(err);
            outb(NULL, NULL, 0);
            if (g.outd != -1 && g.outd != 1) {
                close(g.outd);
                g.outd = -1;

"  -0 to -9, -11        Compression level (level 11, zopfli, is much slower)",
#endif
"  --fast, --best       Compression levels 1 and 9 respectively",
"  -A, --alias xxx      Use xxx as the name for any --zip entry from stdin",
"  -b, --blocksize mmm  Set compression block size to mmmK (default 128K)",
"  -c, --stdout         Write all processed output to stdout (won't delete)",
"  -C, --comment ccc    Put comment ccc in the gzip or zip header",
"  -d, --decompress     Decompress the compressed input",
"  -f, --force          Force overwrite, compress .gz, links, and to terminal",
#ifndef NOZOPFLI

    g.procs = nprocs(8);
#endif
    g.block = 131072UL;             // 128K
    g.shift = x2nmodp(g.block, 3);
    g.rsync = 0;                    // don't do rsync blocking
    g.setdict = 1;                  // initialize dictionary each thread
    g.verbosity = 1;                // normal message level

                                    // where 01 is name and 10 is time
    g.pipeout = 0;                  // don't force output to stdout
    g.sufx = ".gz";                 // compressed file suffix
    g.comment = NULL;               // no comment
    g.decode = 0;                   // compress
    g.list = 0;                     // compress
    g.keep = 0;                     // delete input file once compressed


// Long options conversion to short options.
local char *longopts[][2] = {
    {"LZW", "Z"}, {"lzw", "Z"}, {"alias", "A"}, {"ascii", "a"}, {"best", "9"},
    {"bits", "Z"}, {"blocksize", "b"}, {"decompress", "d"}, {"fast", "1"},
    {"force", "f"}, {"comment", "C"},
#ifndef NOZOPFLI
    {"first", "F"}, {"iterations", "I"}, {"maxsplits", "J"}, {"oneblock", "O"},
#endif


    // if no argument or dash option, check status of get
    if (get && (arg == NULL || *arg == '-')) {
        bad[1] = "bpSIJAC"[get - 1];
        throw(EINVAL, "missing parameter after %s", bad);
    }
    if (arg == NULL)

            // options until we have the parameter
            if (get) {
                if (get == 3)
                    throw(EINVAL,
                          "invalid usage: -S must be followed by space");
                if (get == 7)
                    throw(EINVAL,
                          "invalid usage: -C must be followed by space");
                break;      // allow -*nnn to fall to parameter code
            }

            // process next single character option or compression level

                if (g.level == 10 || g.level > 11)
                    throw(EINVAL, "only levels 0..9 and 11 are allowed");
                break;
            case 'A':  get = 6;  break;
            case 'C':  get = 7;  break;
#ifndef NOZOPFLI
            case 'F':  g.zopts.blocksplittinglast = 1;  break;
            case 'I':  get = 4;  break;

#endif
            case 'K':  g.form = 2;  g.sufx = ".zip";  break;
            case 'L':
                puts(VERSION);
                puts("Copyright (C) 2007-2017 Mark Adler");
                puts("Subject to the terms of the zlib license.");
                puts("No warranty is provided or implied.");
                fputs("No warranty is provided or implied.\n", stderr);
                exit(0);
            case 'M':  g.headis |= 0xa;  break;
            case 'N':  g.headis = 0xf;  break;

            case 'S':  get = 3;  break;
                // -T defined below as an alternative for -m
            case 'V':
                puts(VERSION);
                if (g.verbosity > 1)
                    printf("zlib %s\n", zlibVersion());
                exit(0);
            case 'Y':  g.sync = 1;  break;
            case 'Z':

            return 1;
    }

    // process option parameter for -b, -p, -A, -S, -I, or -J
    if (get) {
        size_t n;

        if (get == 1) {
            n = num(arg);
            g.block = n << 10;                  // chunk size
            g.shift = x2nmodp(g.block, 3);
            if (g.block < DICT)
                throw(EINVAL, "block size too small (must be >= 32K)");
            if (n != g.block >> 10 ||

            g.zopts.numiterations = (int)num(arg);  // optimize iterations
        else if (get == 5)
            g.zopts.blocksplittingmax = (int)num(arg);  // max block splits
        else if (get == 6)
            g.alias = arg;                      // zip name for stdin
#endif
        else if (get == 7)
            g.comment = arg;                    // header comment
        get = 0;
        return 1;
    }

#ifndef NOTHREAD
// handle error received from yarn function
local void cut_yarn(int err) {
    throw(err, "internal threads error");
}
#endif


#ifndef NOTHREAD
        g.in_which = -1;
#endif
        g.alias = "-";
        g.outf = NULL;
        g.first = 1;
        g.hname = NULL;
        g.hcomm = NULL;

        // save pointer to program name for error messages
        p = strrchr(argv[0], '/');

        if (zlib_vernum() < 0x1230)
           throw(EINVAL, "zlib version less than 1.2.3");

        // create CRC table, in case zlib compiled with dynamic tables
        get_crc_table();

        // process user environment variable defaults in GZIP
        opts = getenv("GZIP");
        if (opts != NULL) {

Return to bug 645438