pyi_archive.c

/*
 * ****************************************************************************
 * Copyright (c) 2013-2021, PyInstaller Development Team.
 *
 * Distributed under the terms of the GNU General Public License (version 2
 * or later) with exception for distributing the bootloader.
 *
 * The full license is in the file COPYING.txt, distributed with this software.
 *
 * SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception)
 * ****************************************************************************
 */

/*
 * Fuctions related to PyInstaller archive embedded in executable.
 */

#ifdef _WIN32
    #if BYTE_ORDER == LITTLE_ENDIAN
        #if defined(_MSC_VER)
            #include <stdlib.h>
            #define pyi_be32toh(x) _byteswap_ulong(x)
        #elif defined(__GNUC__) || defined(__clang__)
            #define pyi_be32toh(x) __builtin_bswap32(x)
        #else
            #error Unsupported compiler
        #endif
    #elif BYTE_ORDER == BIG_ENDIAN
        #define pyi_be32toh(x) (x)
    #else
        #error Unsupported byte order
    #endif
#else
    #ifdef __FreeBSD__
/* freebsd issue #188316 */
        #include <arpa/inet.h>  /* ntohl */
    #else
        #include <netinet/in.h>  /* ntohl */
    #endif
    #define pyi_be32toh(x) ntohl(x)
    #include <stdlib.h>   /* malloc */
    #include <string.h>   /* strncmp, strcpy, strcat */
    #include <sys/stat.h> /* fchmod */
#endif /* ifdef _WIN32 */
#include <stddef.h>  /* ptrdiff_t */
#include <stdio.h>

/* PyInstaller headers. */
#include "zlib.h"
#include "pyi_global.h"
#include "pyi_path.h"
#include "pyi_archive.h"
#include "pyi_utils.h"
#include "pyi_python.h"

int pyvers = 0;

/*
 * Return pointer to next toc entry.
 */
TOC *
pyi_arch_increment_toc_ptr(const ARCHIVE_STATUS *status, const TOC* ptoc)
{
    TOC *result = (TOC*)((char *)ptoc + ptoc->structlen);

    if (result < status->tocbuff) {
        FATALERROR("Cannot read Table of Contents.\n");
        return status->tocend;
    }
    return result;
}

/*
 * Open archive file if needed
 */
static int
pyi_arch_open_fp(ARCHIVE_STATUS *status)
{
    if (status->fp == NULL) {
        status->fp = pyi_path_fopen(status->archivename, "rb");

        if (status->fp == NULL) {
            return -1;
        }
    }
    return 0;
}

/*
 * Close archive file
 * File should close after unused to avoid locking
 */
static void
pyi_arch_close_fp(ARCHIVE_STATUS *status)
{
    if (status->fp != NULL) {
        pyi_path_fclose(status->fp);
        status->fp = NULL;
    }
}

/*
 * Decompress data in buff, described by ptoc.
 * Return in malloc'ed buffer (needs to be freed)
 */
static unsigned char *
decompress(unsigned char * buff, TOC *ptoc)
{
    unsigned char *out;
    z_stream zstream;
    int rc;

    out = (unsigned char *)malloc(ptoc->ulen);

    if (out == NULL) {
        OTHERERROR("Error allocating decompression buffer\n");
        return NULL;
    }

    zstream.zalloc = NULL;
    zstream.zfree = NULL;
    zstream.opaque = NULL;
    zstream.next_in = buff;
    zstream.avail_in = ptoc->len;
    zstream.next_out = out;
    zstream.avail_out = ptoc->ulen;
    rc = inflateInit(&zstream);

    if (rc >= 0) {
        rc = (inflate)(&zstream, Z_FINISH);

        if (rc >= 0) {
            rc = (inflateEnd)(&zstream);
        }
        else {
            OTHERERROR("Error %d from inflate: %s\n", rc, zstream.msg);
            return NULL;
        }
    }
    else {
        OTHERERROR("Error %d from inflateInit: %s\n", rc, zstream.msg);
        return NULL;
    }

    return out;
}

/*
 * Extract an archive entry.
 * Returns pointer to the data (must be freed).
 */
unsigned char *
pyi_arch_extract(ARCHIVE_STATUS *status, TOC *ptoc)
{
    unsigned char *data;
    unsigned char *tmp;

    if (pyi_arch_open_fp(status) != 0) {
        OTHERERROR("Cannot open archive file\n");
        return NULL;
    }

    fseek(status->fp, (long)(status->pkgstart + ptoc->pos), SEEK_SET);
    data = (unsigned char *)malloc(ptoc->len);

    if (data == NULL) {
        OTHERERROR("Could not allocate read buffer\n");
        return NULL;
    }

    if (fread(data, ptoc->len, 1, status->fp) < 1) {
        OTHERERROR("Could not read from file\n");
        free(data);
        return NULL;
    }

    if (ptoc->cflag == '\1') {
        tmp = decompress(data, ptoc);
        free(data);
        data = tmp;

        if (data == NULL) {
            OTHERERROR("Error decompressing %s\n", ptoc->name);
            return NULL;
        }
    }

    pyi_arch_close_fp(status);
    return data;
}

/*
 * Extract from the archive and copy to the filesystem.
 * The path is relative to the directory the archive is in.
 */
int
pyi_arch_extract2fs(ARCHIVE_STATUS *status, TOC *ptoc)
{
    FILE *out;
    size_t result, len;
    unsigned char *data = pyi_arch_extract(status, ptoc);

    /* Create tmp dir _MEIPASSxxx. */
    if (pyi_create_temp_path(status) == -1) {
        return -1;
    }

    out = pyi_open_target(status->temppath, ptoc->name);
    len = ptoc->ulen;

    if (out == NULL) {
        FATAL_PERROR("fopen", "%s could not be extracted!\n", ptoc->name);
        return -1;
    }
    else {
        result = fwrite(data, len, 1, out);

        if ((1 != result) && (len > 0)) {
            FATAL_PERROR("fwrite", "Failed to write all bytes for %s\n", ptoc->name);
            return -1;
        }
#ifndef WIN32
        fchmod(fileno(out), S_IRUSR | S_IWUSR | S_IXUSR);
#endif
        fclose(out);
    }
    free(data);

    return 0;
}

/*
 * Perform full back-to-front scan of the file to search for the
 * MAGIC pattern of the embedded archive's COOKIE header.
 *
 * Returns offset within the file if MAGIC pattern is found, 0 otherwise.
 */
static size_t
_pyi_find_cookie_offset(FILE *fp)
{
    static const unsigned char MAGIC[] = { 'M', 'E', 'I', 014, 013, 012, 013, 016 };
    static const int SEARCH_CHUNK_SIZE = 8192;
    unsigned char *buffer = NULL;
    size_t start_pos, end_pos;
    size_t offset = 0;  /* return value */

    /* Allocate the read buffer */
    buffer = malloc(SEARCH_CHUNK_SIZE);
    if (!buffer) {
        VS("LOADER: failed to allocate read buffer (%d bytes)!\n", SEARCH_CHUNK_SIZE);
        goto cleanup;
    }

    /* Determine file size */
    if (fseek(fp, 0, SEEK_END) < 0) {
        VS("LOADER: failed to seek to the end of the file!\n");
        goto cleanup;
    }
    end_pos = ftell(fp);

    /* Sanity check */
    if (end_pos < sizeof(MAGIC)) {
        VS("LOADER: file is too short!\n");
        goto cleanup;
    }

    /* Search the file back to front, in overlapping SEARCH_CHUNK_SIZE
     * chunks. */
    do {
        size_t chunk_size;
        start_pos = (end_pos >= SEARCH_CHUNK_SIZE) ? (end_pos - SEARCH_CHUNK_SIZE) : 0;
        chunk_size = end_pos - start_pos;

        /* Is the remaining chunk large enough to hold the pattern? */
        if (chunk_size < sizeof(MAGIC)) {
            break;
        }

        /* Read the chunk */
        if (fseek(fp, start_pos, SEEK_SET) < 0) {
            VS("LOADER: failed to seek to the offset 0x%zX!\n", start_pos);
            goto cleanup;
        }
        if (fread(buffer, 1, chunk_size, fp) != chunk_size) {
            VS("LOADER: failed to read chunk (%zd bytes)!\n", chunk_size);
            goto cleanup;
        }

        /* Scan the chunk */
        for (size_t i = chunk_size - sizeof(MAGIC) + 1; i > 0; i--) {
            if (memcmp(buffer + i - 1, MAGIC, sizeof(MAGIC)) == 0) {
                offset = start_pos + i - 1;
                goto cleanup;
            }
        }

        /* Adjust search location for next chunk; ensure proper overlap */
        end_pos = start_pos + sizeof(MAGIC) - 1;
    } while (start_pos > 0);

cleanup:
    free(buffer);

    return offset;
}

/*
 * Fix the endianess of fields in the TOC entries.
 */
static void
_pyi_arch_fix_toc_endianess(ARCHIVE_STATUS *status)
{
    TOC *ptoc = status->tocbuff;
    while (ptoc < status->tocend) {
        /* Fixup the current entry */
        ptoc->structlen = pyi_be32toh(ptoc->structlen);
        ptoc->pos = pyi_be32toh(ptoc->pos);
        ptoc->len = pyi_be32toh(ptoc->len);
        ptoc->ulen = pyi_be32toh(ptoc->ulen);
        /* Jump to next entry; with the current entry fixed up, we can
         * use pyi_arch_increment_toc_ptr() */
        ptoc = pyi_arch_increment_toc_ptr(status, ptoc);
    }
}

/*
 * Open the archive.
 * Sets f_archiveFile, f_pkgstart, f_tocbuff and f_cookie.
 */
int
pyi_arch_open(ARCHIVE_STATUS *status)
{
    size_t cookie_pos = 0;
    VS("LOADER: archivename is %s\n", status->archivename);

    /* Physically open the file */
    if (pyi_arch_open_fp(status) != 0) {
        VS("LOADER: Cannot open archive: %s\n", status->archivename);
        return -1;
    }

    /* Search for the embedded archive's cookie */
    cookie_pos = _pyi_find_cookie_offset(status->fp);
    if (cookie_pos == 0) {
        VS("LOADER: Cannot find cookie!\n");
        return -1;
    }
    VS("LOADER: Cookie found at offset 0x%zX\n", cookie_pos);

    /* Read the cookie */
    if (fseek(status->fp, cookie_pos, SEEK_SET) < 0) {
        FATAL_PERROR("fseek", "failed to seek to cookie position.");
        return -1;
    }
    if (fread(&status->cookie, sizeof(COOKIE), 1, status->fp) < 1) {
        FATAL_PERROR("fread", "failed to read cookie.");
        return -1;
    }
    /* Fix endianess of COOKIE fields */
    status->cookie.len = pyi_be32toh(status->cookie.len);
    status->cookie.TOC = pyi_be32toh(status->cookie.TOC);
    status->cookie.TOClen = pyi_be32toh(status->cookie.TOClen);
    status->cookie.pyvers = pyi_be32toh(status->cookie.pyvers);

    /* From the cookie position and declared archive size, calculate
     * the archive start position */
    status->pkgstart = cookie_pos + sizeof(COOKIE) - status->cookie.len;

    /* Set the flag that Python library was not loaded yet. */
    status->is_pylib_loaded = false;

    /* Set the the Python version used. */
    pyvers = pyi_arch_get_pyversion(status);

    /* Read in in the table of contents */
    fseek(status->fp, (long)(status->pkgstart + status->cookie.TOC), SEEK_SET);
    status->tocbuff = (TOC *) malloc(status->cookie.TOClen);

    if (status->tocbuff == NULL) {
        FATAL_PERROR("malloc", "Could not allocate buffer for TOC.");
        return -1;
    }

    if (fread(status->tocbuff, status->cookie.TOClen, 1, status->fp) < 1) {
        FATAL_PERROR("fread", "Could not read from file.");
        return -1;
    }
    status->tocend = (TOC *) (((char *)status->tocbuff) + status->cookie.TOClen);

    /* Check input file is still ok (should be). */
    if (ferror(status->fp)) {
        FATALERROR("Error on file\n.");
        return -1;
    }

    /* Fix the endianess of the fields in the TOC entries */
    _pyi_arch_fix_toc_endianess(status);

    /* Close file handler
     * if file not close here it will be close in pyi_arch_status_free */
    pyi_arch_close_fp(status);
    return 0;
}

/* Setup the archive with python modules and the paths required by rest of
 * this module (this always needs to be done).
 * Sets f_archivename, f_homepath, f_mainpath
 */
bool
pyi_arch_setup(ARCHIVE_STATUS *status, char const * archivePath)
{
    /* Get the archive Path */
    if (strlen(archivePath) >= PATH_MAX) {
        // Should never come here, since `archivePath` was already processed
        // by pyi_path_executable or pyi_path_archivefile.
        return false;
    }

    strcpy(status->archivename, archivePath);
    /* Set homepath to where the archive is */
    pyi_path_dirname(status->homepath, archivePath);
    /*
     * Initial value of mainpath is homepath. It might be overriden
     * by temppath if it is available.
     */
    status->has_temp_directory = false;
    strcpy(status->mainpath, status->homepath);

    /* Open the archive */
    if (pyi_arch_open(status)) {
        /* If this is not an archive, we MUST close the file, */
        /* otherwise the open file-handle will be reused when */
        /* testing the next file. */
        pyi_arch_close_fp(status);
        return false;
    }
    return true;
}

/*
 * external API for iterating TOCs
 */
TOC *
getFirstTocEntry(ARCHIVE_STATUS *status)
{
    return status->tocbuff;
}
TOC *
getNextTocEntry(ARCHIVE_STATUS *status, TOC *entry)
{
    TOC *rslt = (TOC*)((char *)entry + entry->structlen);

    if (rslt >= status->tocend) {
        return NULL;
    }
    return rslt;
}

/*
 * Helpers for embedders.
 */
int
pyi_arch_get_pyversion(ARCHIVE_STATUS *status)
{
    return status->cookie.pyvers;
}

/*
 * Allocate memory for archive status.
 */
ARCHIVE_STATUS *
pyi_arch_status_new() {
    ARCHIVE_STATUS *archive_status;
    archive_status = (ARCHIVE_STATUS *) calloc(1, sizeof(ARCHIVE_STATUS));
    if (archive_status == NULL) {
        FATAL_PERROR("calloc", "Cannot allocate memory for ARCHIVE_STATUS\n");
    }
    return archive_status;
}

/*
 * Free memory allocated for archive status.
 */
void
pyi_arch_status_free(ARCHIVE_STATUS *archive_status)
{
    if (archive_status != NULL) {
        VS("LOADER: Freeing archive status for %s\n", archive_status->archivename);

        /* Free the TOC memory from the archive status first. */
        if (archive_status->tocbuff != NULL) {
            free(archive_status->tocbuff);
        }
        /* Close file handler */
        pyi_arch_close_fp(archive_status);
        free(archive_status);
    }
}

/*
 * Returns the value of the pyi bootloader option given by optname. Returns
 * NULL if the option is not present. Returns an empty string if the option is present,
 * but has no associated value.
 *
 * The string returned is owned by the ARCHIVE_STATUS; the caller is NOT responsible
 * for freeing it.
 */
char *
pyi_arch_get_option(const ARCHIVE_STATUS * status, char * optname)
{
    /* TODO: option-cache? */
    size_t optlen;
    TOC *ptoc = status->tocbuff;

    optlen = strlen(optname);

    for (; ptoc < status->tocend; ptoc = pyi_arch_increment_toc_ptr(status, ptoc)) {
        if (ptoc->typcd == ARCHIVE_ITEM_RUNTIME_OPTION) {
            if (0 == strncmp(ptoc->name, optname, optlen)) {
                if (0 != ptoc->name[optlen]) {
                    /* Space separates option name from option value, so add 1. */
                    return ptoc->name + optlen + 1;
                }
                else {
                    /* No option value, just return the empty string. */
                    return ptoc->name + optlen;
                }

            }
        }
    }
    return NULL;
}