Skip to content

Commit

Permalink
bootloader: clean up endianess conversion (pyinstaller#5457)
Browse files Browse the repository at this point in the history 
* bootloader: perform endianess fix-up of archive fields in only one place

The conversion of multi-byte integer fields in COOKIE and TOC
structures is a data (de)serialization detail, and as such should
be performed only once, after the data is read; as opposed to
on-the-fly endianess conversions on every field access, which are
scattered throughout the codebase.

* bootloader: on Windows, avoid using ntohl() from ws2_32 library

On Windows, use built-in byte-swap macros instead of ntohl(), which
is available from winsock2 library. Using the latter requires the
bootloader to be linked against the winsock2 library, which may have
other unintended consequences.

On all other platforms, continue using ntohl() for now, but hide it
behind the pyi_be32toh() macro.
  • Loading branch information
@rokm
rokm committed Feb 13, 2021
1 parent ce61bfc commit c24117a
Show file tree
Hide file tree
Showing 7 changed files with 67 additions and 42 deletions.
75 changes: 58 additions & 17 deletions bootloader/src/pyi_archive.c
View file
Expand Up @@ -16,15 +16,28 @@
*/

#ifdef _WIN32
/* TODO verify windows includes */
#include <winsock.h> /* ntohl */
#if BYTE_ORDER == LITTLE_ENDIAN
#if defined(_MSC_VER)
#include <stdlib.h>
#define pyi_be32toh(x) _byteswap_ulong(x)
#elif defined(__GNUC__) || defined(__clang__)
#define pyi_be32toh(x) __builtin_bswap32(x)
#else
#error Unsupported compiler
#endif
#elif BYTE_ORDER == BIG_ENDIAN
#define pyi_be32toh(x) (x)
#else
#error Unsupported byte order
#endif
#else
#ifdef __FreeBSD__
/* freebsd issue #188316 */
#include <arpa/inet.h> /* ntohl */
#else
#include <netinet/in.h> /* ntohl */
#endif
#define pyi_be32toh(x) ntohl(x)
#include <stdlib.h> /* malloc */
#include <string.h> /* strncmp, strcpy, strcat */
#include <sys/stat.h> /* fchmod */
Expand All @@ -51,7 +64,7 @@ int pyvers = 0;
TOC *
pyi_arch_increment_toc_ptr(const ARCHIVE_STATUS *status, const TOC* ptoc)
{
TOC *result = (TOC*)((char *)ptoc + ntohl(ptoc->structlen));
TOC *result = (TOC*)((char *)ptoc + ptoc->structlen);

if (result < status->tocbuff) {
FATALERROR("Cannot read Table of Contents.\n");
Expand Down Expand Up @@ -100,7 +113,7 @@ decompress(unsigned char * buff, TOC *ptoc)
z_stream zstream;
int rc;

out = (unsigned char *)malloc(ntohl(ptoc->ulen));
out = (unsigned char *)malloc(ptoc->ulen);

if (out == NULL) {
OTHERERROR("Error allocating decompression buffer\n");
Expand All @@ -111,9 +124,9 @@ decompress(unsigned char * buff, TOC *ptoc)
zstream.zfree = NULL;
zstream.opaque = NULL;
zstream.next_in = buff;
zstream.avail_in = ntohl(ptoc->len);
zstream.avail_in = ptoc->len;
zstream.next_out = out;
zstream.avail_out = ntohl(ptoc->ulen);
zstream.avail_out = ptoc->ulen;
rc = inflateInit(&zstream);

if (rc >= 0) {
Expand Down Expand Up @@ -150,15 +163,15 @@ pyi_arch_extract(ARCHIVE_STATUS *status, TOC *ptoc)
return NULL;
}

fseek(status->fp, (long)(status->pkgstart + ntohl(ptoc->pos)), SEEK_SET);
data = (unsigned char *)malloc(ntohl(ptoc->len));
fseek(status->fp, (long)(status->pkgstart + ptoc->pos), SEEK_SET);
data = (unsigned char *)malloc(ptoc->len);

if (data == NULL) {
OTHERERROR("Could not allocate read buffer\n");
return NULL;
}

if (fread(data, ntohl(ptoc->len), 1, status->fp) < 1) {
if (fread(data, ptoc->len, 1, status->fp) < 1) {
OTHERERROR("Could not read from file\n");
free(data);
return NULL;
Expand Down Expand Up @@ -196,7 +209,7 @@ pyi_arch_extract2fs(ARCHIVE_STATUS *status, TOC *ptoc)
}

out = pyi_open_target(status->temppath, ptoc->name);
len = ntohl(ptoc->ulen);
len = ptoc->ulen;

if (out == NULL) {
FATAL_PERROR("fopen", "%s could not be extracted!\n", ptoc->name);
Expand Down Expand Up @@ -265,8 +278,14 @@ pyi_arch_find_cookie(ARCHIVE_STATUS *status, int search_end)
/* MAGIC found - Copy COOKIE to status->cookie */
memcpy(&status->cookie, search_ptr, sizeof(COOKIE));

/* Fix endianess of COOKIE fields */
status->cookie.len = pyi_be32toh(status->cookie.len);
status->cookie.TOC = pyi_be32toh(status->cookie.TOC);
status->cookie.TOClen = pyi_be32toh(status->cookie.TOClen);
status->cookie.pyvers = pyi_be32toh(status->cookie.pyvers);

/* From the cookie, calculate the archive start */
status->pkgstart = search_start + sizeof(COOKIE) + (search_ptr - buf) - ntohl(status->cookie.len);
status->pkgstart = search_start + sizeof(COOKIE) + (search_ptr - buf) - status->cookie.len;

return 0;
}
Expand Down Expand Up @@ -374,6 +393,25 @@ findDigitalSignature(ARCHIVE_STATUS * const status)
#endif /* ifdef _WIN32 */
}

/*
* Fix the endianess of fields in the TOC entries.
*/
static void
_pyi_arch_fix_toc_endianess(ARCHIVE_STATUS *status)
{
TOC *ptoc = status->tocbuff;
while (ptoc < status->tocend) {
/* Fixup the current entry */
ptoc->structlen = pyi_be32toh(ptoc->structlen);
ptoc->pos = pyi_be32toh(ptoc->pos);
ptoc->len = pyi_be32toh(ptoc->len);
ptoc->ulen = pyi_be32toh(ptoc->ulen);
/* Jump to next entry; with the current entry fixed up, we can
* use pyi_arch_increment_toc_ptr() */
ptoc = pyi_arch_increment_toc_ptr(status, ptoc);
}
}

/*
* Open the archive.
* Sets f_archiveFile, f_pkgstart, f_tocbuff and f_cookie.
Expand Down Expand Up @@ -418,26 +456,29 @@ pyi_arch_open(ARCHIVE_STATUS *status)
pyvers = pyi_arch_get_pyversion(status);

/* Read in in the table of contents */
fseek(status->fp, (long)(status->pkgstart + ntohl(status->cookie.TOC)), SEEK_SET);
status->tocbuff = (TOC *) malloc(ntohl(status->cookie.TOClen));
fseek(status->fp, (long)(status->pkgstart + status->cookie.TOC), SEEK_SET);
status->tocbuff = (TOC *) malloc(status->cookie.TOClen);

if (status->tocbuff == NULL) {
FATAL_PERROR("malloc", "Could not allocate buffer for TOC.");
return -1;
}

if (fread(status->tocbuff, ntohl(status->cookie.TOClen), 1, status->fp) < 1) {
if (fread(status->tocbuff, status->cookie.TOClen, 1, status->fp) < 1) {
FATAL_PERROR("fread", "Could not read from file.");
return -1;
}
status->tocend = (TOC *) (((char *)status->tocbuff) + ntohl(status->cookie.TOClen));
status->tocend = (TOC *) (((char *)status->tocbuff) + status->cookie.TOClen);

/* Check input file is still ok (should be). */
if (ferror(status->fp)) {
FATALERROR("Error on file\n.");
return -1;
}

/* Fix the endianess of the fields in the TOC entries */
_pyi_arch_fix_toc_endianess(status);

/* Close file handler
* if file not close here it will be close in pyi_arch_status_free */
pyi_arch_close_fp(status);
Expand Down Expand Up @@ -490,7 +531,7 @@ getFirstTocEntry(ARCHIVE_STATUS *status)
TOC *
getNextTocEntry(ARCHIVE_STATUS *status, TOC *entry)
{
TOC *rslt = (TOC*)((char *)entry + ntohl(entry->structlen));
TOC *rslt = (TOC*)((char *)entry + entry->structlen);

if (rslt >= status->tocend) {
return NULL;
Expand All @@ -504,7 +545,7 @@ getNextTocEntry(ARCHIVE_STATUS *status, TOC *entry)
int
pyi_arch_get_pyversion(ARCHIVE_STATUS *status)
{
return ntohl(status->cookie.pyvers);
return status->cookie.pyvers;
}

/*
Expand Down
1 change: 1 addition & 0 deletions bootloader/src/pyi_global.h
View file
Expand Up @@ -51,6 +51,7 @@ typedef int bool;

/* Type for dynamic library. */
#ifdef _WIN32
#include <windows.h> /* HINSTANCE */
#define dylib_t HINSTANCE
#else
#define dylib_t void *
Expand Down
9 changes: 1 addition & 8 deletions bootloader/src/pyi_launch.c
View file
Expand Up @@ -21,14 +21,7 @@

#ifdef _WIN32
#include <windows.h>
#include <winsock.h> /* ntohl */
#else
#ifdef __FreeBSD__
/* freebsd issue #188316 */
#include <arpa/inet.h> /* ntohl */
#else
#include <netinet/in.h> /* ntohl */
#endif
#include <langinfo.h> /* CODESET, nl_langinfo */
#include <stdlib.h> /* malloc */
#endif
Expand Down Expand Up @@ -460,7 +453,7 @@ pyi_launch_run_scripts(ARCHIVE_STATUS *status)
Py_DECREF(__file__);

/* Unmarshall code object */
code = PI_PyMarshal_ReadObjectFromString((const char *) data, ntohl(ptoc->ulen));
code = PI_PyMarshal_ReadObjectFromString((const char *) data, ptoc->ulen);

if (!code) {
FATALERROR("Failed to unmarshal code object for %s\n", ptoc->name);
Expand Down
1 change: 0 additions & 1 deletion bootloader/src/pyi_python.c
View file
Expand Up @@ -17,7 +17,6 @@

#ifdef _WIN32
#include <windows.h> /* HMODULE */
#include <winsock.h> /* ntohl */
#else
#include <dlfcn.h> /* dlsym */
#endif
Expand Down
15 changes: 4 additions & 11 deletions bootloader/src/pyi_pythonlib.c
View file
Expand Up @@ -21,15 +21,8 @@
#include <windows.h> /* HMODULE */
#include <fcntl.h> /* O_BINARY */
#include <io.h> /* _setmode */
#include <winsock.h> /* ntohl */
#else
#include <dlfcn.h> /* dlerror */
#ifdef __FreeBSD__
/* freebsd issue #188316 */
#include <arpa/inet.h> /* ntohl */
#else
#include <netinet/in.h> /* ntohl */
#endif
#include <stdlib.h> /* mbstowcs */
#endif /* ifdef _WIN32 */
#include <stddef.h> /* ptrdiff_t */
Expand Down Expand Up @@ -565,14 +558,14 @@ pyi_pylib_import_modules(ARCHIVE_STATUS *status)
if (pyvers >= 37) {
/* Python >= 3.7 the header: size was changed to 16 bytes. */
co = PI_PyObject_CallFunction(loadfunc, "y#", modbuf + 16,
ntohl(ptoc->ulen) - 16);
ptoc->ulen - 16);
}
else {
/* It looks like from python 3.3 the header */
/* size was changed to 12 bytes. */
co =
PI_PyObject_CallFunction(loadfunc, "y#", modbuf + 12, ntohl(
ptoc->ulen) - 12);
PI_PyObject_CallFunction(loadfunc, "y#", modbuf + 12,
ptoc->ulen - 12);
};

if (co != NULL) {
Expand Down Expand Up @@ -619,7 +612,7 @@ int
pyi_pylib_install_zlib(ARCHIVE_STATUS *status, TOC *ptoc)
{
int rc = 0;
size_t zlibpos = status->pkgstart + ntohl(ptoc->pos);
size_t zlibpos = status->pkgstart + ptoc->pos;
PyObject * sys_path, *zlib_entry, *archivename_obj;

/* Note that sys.path contains PyUnicode on py3. Ensure
Expand Down
3 changes: 1 addition & 2 deletions bootloader/tests/wscript
View file
Expand Up @@ -17,10 +17,9 @@ def build(ctx):

def test_program(name):
if ctx.env.DEST_OS == 'win32':
# WS2_32: ntohl()
# Z: inflate*()
# ADVAPI32: ConvertStringSecurityDescriptorToSecurityDescriptorW()
extra_libs=['ADVAPI32', 'WS2_32', 'Z']
extra_libs=['ADVAPI32', 'Z']
else:
extra_libs=[]
ctx.program(
Expand Down
5 changes: 2 additions & 3 deletions bootloader/wscript
View file
Expand Up @@ -556,14 +556,13 @@ def configure(ctx):

if ctx.env.DEST_OS == 'win32':
if ctx.env.CC_NAME == 'msvc':
ctx.check_libs_msvc('user32 comctl32 kernel32 advapi32 ws2_32',
ctx.check_libs_msvc('user32 comctl32 kernel32 advapi32',
mandatory=True)
else:
ctx.check_cc(lib='user32', mandatory=True)
ctx.check_cc(lib='comctl32', mandatory=True)
ctx.check_cc(lib='kernel32', mandatory=True)
ctx.check_cc(lib='advapi32', mandatory=True)
ctx.check_cc(lib='ws2_32', mandatory=True)
else:
# Mac OS X and FreeBSD do not need libdl.
# https://stackoverflow.com/questions/20169660/where-is-libdl-so-on-mac-os-x
Expand Down Expand Up @@ -750,7 +749,7 @@ def build(ctx):
source=['src/main.c', icon_rc],
target=exe_name,
install_path=install_path,
use='OBJECTS USER32 COMCTL32 KERNEL32 ADVAPI32 WS2_32 Z',
use='OBJECTS USER32 COMCTL32 KERNEL32 ADVAPI32 Z',
includes='src windows zlib',
features=features,
)
Expand Down

0 comments on commit c24117a

Please sign in to comment.