macosx_libfile.py

"""
This module contains function to analyse dynamic library
headers to extract system information

Currently only for MacOSX

Library file on macosx system starts with Mach-O or Fat field.
This can be distinguish by first 32 bites and it is called magic number.
Proper value of magic number is with suffix _MAGIC. Suffix _CIGAM means
reversed bytes order.
Both fields can occur in two types: 32 and 64 bytes.

FAT field inform that this library contains few version of library
(typically for different types version). It contains
information where Mach-O headers starts.

Each section started with Mach-O header contains one library
(So if file starts with this field it contains only one version).

After filed Mach-O there are section fields.
Each of them starts with two fields:
cmd - magic number for this command
cmdsize - total size occupied by this section information.

In this case only sections LC_VERSION_MIN_MACOSX (for macosx 10.13 and earlier)
and LC_BUILD_VERSION (for macosx 10.14 and newer) are interesting,
because them contains information about minimal system version.

Important remarks:
- For fat files this implementation looks for maximum number version.
  It not check if it is 32 or 64 and do not compare it with currently builded package.
  So it is possible to false report higher version that needed.
- All structures signatures are taken form macosx header files.
- I think that binary format will be more stable than `otool` output.
  and if apple introduce some changes both implementation will need to be updated.
"""

import ctypes
import os
import sys

"""here the needed const and struct from mach-o header files"""

FAT_MAGIC = 0xcafebabe
FAT_CIGAM = 0xbebafeca
FAT_MAGIC_64 = 0xcafebabf
FAT_CIGAM_64 = 0xbfbafeca
MH_MAGIC = 0xfeedface
MH_CIGAM = 0xcefaedfe
MH_MAGIC_64 = 0xfeedfacf
MH_CIGAM_64 = 0xcffaedfe

LC_VERSION_MIN_MACOSX = 0x24
LC_BUILD_VERSION = 0x32


mach_header_fields = [
        ("magic", ctypes.c_uint32), ("cputype", ctypes.c_int),
        ("cpusubtype", ctypes.c_int), ("filetype", ctypes.c_uint32),
        ("ncmds", ctypes.c_uint32), ("sizeofcmds", ctypes.c_uint32),
        ("flags", ctypes.c_uint32)
    ]
"""
struct mach_header {
    uint32_t	magic;		/* mach magic number identifier */
    cpu_type_t	cputype;	/* cpu specifier */
    cpu_subtype_t	cpusubtype;	/* machine specifier */
    uint32_t	filetype;	/* type of file */
    uint32_t	ncmds;		/* number of load commands */
    uint32_t	sizeofcmds;	/* the size of all the load commands */
    uint32_t	flags;		/* flags */
};
typedef integer_t cpu_type_t;
typedef integer_t cpu_subtype_t;
"""

mach_header_fields_64 = mach_header_fields + [("reserved", ctypes.c_uint32)]
"""
struct mach_header_64 {
    uint32_t	magic;		/* mach magic number identifier */
    cpu_type_t	cputype;	/* cpu specifier */
    cpu_subtype_t	cpusubtype;	/* machine specifier */
    uint32_t	filetype;	/* type of file */
    uint32_t	ncmds;		/* number of load commands */
    uint32_t	sizeofcmds;	/* the size of all the load commands */
    uint32_t	flags;		/* flags */
    uint32_t	reserved;	/* reserved */
};
"""

fat_header_fields = [("magic", ctypes.c_uint32), ("nfat_arch", ctypes.c_uint32)]
"""
struct fat_header {
    uint32_t	magic;		/* FAT_MAGIC or FAT_MAGIC_64 */
    uint32_t	nfat_arch;	/* number of structs that follow */
};
"""

fat_arch_fields = [
    ("cputype", ctypes.c_int), ("cpusubtype", ctypes.c_int),
    ("offset", ctypes.c_uint32), ("size", ctypes.c_uint32),
    ("align", ctypes.c_uint32)
]
"""
struct fat_arch {
    cpu_type_t	cputype;	/* cpu specifier (int) */
    cpu_subtype_t	cpusubtype;	/* machine specifier (int) */
    uint32_t	offset;		/* file offset to this object file */
    uint32_t	size;		/* size of this object file */
    uint32_t	align;		/* alignment as a power of 2 */
};
"""

fat_arch_64_fields = [
    ("cputype", ctypes.c_int), ("cpusubtype", ctypes.c_int),
    ("offset", ctypes.c_uint64), ("size", ctypes.c_uint64),
    ("align", ctypes.c_uint32), ("reserved", ctypes.c_uint32)
]
"""
struct fat_arch_64 {
    cpu_type_t	cputype;	/* cpu specifier (int) */
    cpu_subtype_t	cpusubtype;	/* machine specifier (int) */
    uint64_t	offset;		/* file offset to this object file */
    uint64_t	size;		/* size of this object file */
    uint32_t	align;		/* alignment as a power of 2 */
    uint32_t	reserved;	/* reserved */
};
"""

segment_base_fields = [("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32)]
"""base for reading segment info"""

segment_command_fields = [
    ("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32),
    ("segname", ctypes.c_char * 16), ("vmaddr", ctypes.c_uint32),
    ("vmsize", ctypes.c_uint32), ("fileoff", ctypes.c_uint32),
    ("filesize", ctypes.c_uint32), ("maxprot", ctypes.c_int),
    ("initprot", ctypes.c_int), ("nsects", ctypes.c_uint32),
    ("flags", ctypes.c_uint32),
    ]
"""
struct segment_command { /* for 32-bit architectures */
    uint32_t	cmd;		/* LC_SEGMENT */
    uint32_t	cmdsize;	/* includes sizeof section structs */
    char		segname[16];	/* segment name */
    uint32_t	vmaddr;		/* memory address of this segment */
    uint32_t	vmsize;		/* memory size of this segment */
    uint32_t	fileoff;	/* file offset of this segment */
    uint32_t	filesize;	/* amount to map from the file */
    vm_prot_t	maxprot;	/* maximum VM protection */
    vm_prot_t	initprot;	/* initial VM protection */
    uint32_t	nsects;		/* number of sections in segment */
    uint32_t	flags;		/* flags */
};
typedef int vm_prot_t;
"""

segment_command_fields_64 = [
    ("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32),
    ("segname", ctypes.c_char * 16), ("vmaddr", ctypes.c_uint64),
    ("vmsize", ctypes.c_uint64), ("fileoff", ctypes.c_uint64),
    ("filesize", ctypes.c_uint64), ("maxprot", ctypes.c_int),
    ("initprot", ctypes.c_int), ("nsects", ctypes.c_uint32),
    ("flags", ctypes.c_uint32),
    ]
"""
struct segment_command_64 { /* for 64-bit architectures */
    uint32_t	cmd;		/* LC_SEGMENT_64 */
    uint32_t	cmdsize;	/* includes sizeof section_64 structs */
    char		segname[16];	/* segment name */
    uint64_t	vmaddr;		/* memory address of this segment */
    uint64_t	vmsize;		/* memory size of this segment */
    uint64_t	fileoff;	/* file offset of this segment */
    uint64_t	filesize;	/* amount to map from the file */
    vm_prot_t	maxprot;	/* maximum VM protection */
    vm_prot_t	initprot;	/* initial VM protection */
    uint32_t	nsects;		/* number of sections in segment */
    uint32_t	flags;		/* flags */
};
"""

version_min_command_fields = segment_base_fields + \
    [("version", ctypes.c_uint32), ("sdk", ctypes.c_uint32)]
"""
struct version_min_command {
    uint32_t	cmd;		/* LC_VERSION_MIN_MACOSX or
                               LC_VERSION_MIN_IPHONEOS or
                               LC_VERSION_MIN_WATCHOS or
                               LC_VERSION_MIN_TVOS */
    uint32_t	cmdsize;	/* sizeof(struct min_version_command) */
    uint32_t	version;	/* X.Y.Z is encoded in nibbles xxxx.yy.zz */
    uint32_t	sdk;		/* X.Y.Z is encoded in nibbles xxxx.yy.zz */
};
"""

build_version_command_fields = segment_base_fields + \
    [("platform", ctypes.c_uint32), ("minos", ctypes.c_uint32),
     ("sdk", ctypes.c_uint32), ("ntools", ctypes.c_uint32)]
"""
struct build_version_command {
    uint32_t	cmd;		/* LC_BUILD_VERSION */
    uint32_t	cmdsize;	/* sizeof(struct build_version_command) plus */
                                /* ntools * sizeof(struct build_tool_version) */
    uint32_t	platform;	/* platform */
    uint32_t	minos;		/* X.Y.Z is encoded in nibbles xxxx.yy.zz */
    uint32_t	sdk;		/* X.Y.Z is encoded in nibbles xxxx.yy.zz */
    uint32_t	ntools;		/* number of tool entries following this */
};
"""


def swap32(x):
    return (((x << 24) & 0xFF000000) |
            ((x << 8) & 0x00FF0000) |
            ((x >> 8) & 0x0000FF00) |
            ((x >> 24) & 0x000000FF))


def get_base_class_and_magic_number(lib_file, seek=None):
    if seek is None:
        seek = lib_file.tell()
    else:
        lib_file.seek(seek)
    magic_number = ctypes.c_uint32.from_buffer_copy(
        lib_file.read(ctypes.sizeof(ctypes.c_uint32))).value

    # Handle wrong byte order
    if magic_number in [FAT_CIGAM, FAT_CIGAM_64, MH_CIGAM, MH_CIGAM_64]:
        if sys.byteorder == "little":
            BaseClass = ctypes.BigEndianStructure
        else:
            BaseClass = ctypes.LittleEndianStructure

        magic_number = swap32(magic_number)
    else:
        BaseClass = ctypes.Structure

    lib_file.seek(seek)
    return BaseClass, magic_number


def read_data(struct_class, lib_file):
    return struct_class.from_buffer_copy(lib_file.read(
                        ctypes.sizeof(struct_class)))


def extract_macosx_min_system_version(path_to_lib):
    with open(path_to_lib, "rb") as lib_file:
        BaseClass, magic_number = get_base_class_and_magic_number(lib_file, 0)
        if magic_number not in [FAT_MAGIC, FAT_MAGIC_64, MH_MAGIC, MH_MAGIC_64]:
            return

        if magic_number in [FAT_MAGIC, FAT_CIGAM_64]:
            class FatHeader(BaseClass):
                _fields_ = fat_header_fields

            fat_header = read_data(FatHeader, lib_file)
            if magic_number == FAT_MAGIC:

                class FatArch(BaseClass):
                    _fields_ = fat_arch_fields
            else:

                class FatArch(BaseClass):
                    _fields_ = fat_arch_64_fields

            fat_arch_list = [read_data(FatArch, lib_file) for _ in range(fat_header.nfat_arch)]

            versions_list = []
            for el in fat_arch_list:
                try:
                    version = read_mach_header(lib_file, el.offset)
                    if version is not None:
                        versions_list.append(version)
                except ValueError:
                    pass

            if len(versions_list) > 0:
                return max(versions_list)
            else:
                return None

        else:
            try:
                return read_mach_header(lib_file, 0)
            except ValueError:
                """when some error during read library files"""
                return None


def read_mach_header(lib_file, seek=None):
    """
    This funcition parse mach-O header and extract
    information about minimal system version

    :param lib_file: reference to opened library file with pointer
    """
    if seek is not None:
        lib_file.seek(seek)
    base_class, magic_number = get_base_class_and_magic_number(lib_file)
    arch = "32" if magic_number == MH_MAGIC else "64"

    class SegmentBase(base_class):
        _fields_ = segment_base_fields

    if arch == "32":

        class MachHeader(base_class):
            _fields_ = mach_header_fields

    else:

        class MachHeader(base_class):
            _fields_ = mach_header_fields_64

    mach_header = read_data(MachHeader, lib_file)
    for _i in range(mach_header.ncmds):
        pos = lib_file.tell()
        segment_base = read_data(SegmentBase, lib_file)
        lib_file.seek(pos)
        if segment_base.cmd == LC_VERSION_MIN_MACOSX:
            class VersionMinCommand(base_class):
                _fields_ = version_min_command_fields

            version_info = read_data(VersionMinCommand, lib_file)
            return parse_version(version_info.version)
        elif segment_base.cmd == LC_BUILD_VERSION:
            class VersionBuild(base_class):
                _fields_ = build_version_command_fields

            version_info = read_data(VersionBuild, lib_file)
            return parse_version(version_info.minos)
        else:
            lib_file.seek(pos + segment_base.cmdsize)
            continue


def parse_version(version):
    x = (version & 0xffff0000) >> 16
    y = (version & 0x0000ff00) >> 8
    z = (version & 0x000000ff)
    return x, y, z


def calculate_macosx_platform_tag(archive_root, platform_tag):
    """
    Calculate proper macosx platform tag basing on files which are included to wheel

    Example platform tag `macosx-10.14-x86_64`
    """
    prefix, base_version, suffix = platform_tag.split('-')
    base_version = tuple([int(x) for x in base_version.split(".")])
    base_version = base_version[:2]
    if base_version[0] > 10:
        base_version = (base_version[0], 0)
    assert len(base_version) == 2
    if "MACOSX_DEPLOYMENT_TARGET" in os.environ:
        deploy_target = tuple([int(x) for x in os.environ[
            "MACOSX_DEPLOYMENT_TARGET"].split(".")])
        deploy_target = deploy_target[:2]
        if deploy_target[0] > 10:
            deploy_target = (deploy_target[0], 0)
        if deploy_target < base_version:
            sys.stderr.write(
                 "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value ({}) than the "
                 "version on which the Python interpreter was compiled ({}), and will be "
                 "ignored.\n".format('.'.join(str(x) for x in deploy_target),
                                     '.'.join(str(x) for x in base_version))
                )
        else:
            base_version = deploy_target

    assert len(base_version) == 2
    start_version = base_version
    versions_dict = {}
    for (dirpath, dirnames, filenames) in os.walk(archive_root):
        for filename in filenames:
            if filename.endswith('.dylib') or filename.endswith('.so'):
                lib_path = os.path.join(dirpath, filename)
                min_ver = extract_macosx_min_system_version(lib_path)
                if min_ver is not None:
                    min_ver = min_ver[0:2]
                    if min_ver[0] > 10:
                        min_ver = (min_ver[0], 0)
                    versions_dict[lib_path] = min_ver

    if len(versions_dict) > 0:
        base_version = max(base_version, max(versions_dict.values()))

    # macosx platform tag do not support minor bugfix release
    fin_base_version = "_".join([str(x) for x in base_version])
    if start_version < base_version:
        problematic_files = [k for k, v in versions_dict.items() if v > start_version]
        problematic_files = "\n".join(problematic_files)
        if len(problematic_files) == 1:
            files_form = "this file"
        else:
            files_form = "these files"
        error_message = \
            "[WARNING] This wheel needs a higher macOS version than {}  " \
            "To silence this warning, set MACOSX_DEPLOYMENT_TARGET to at least " +\
            fin_base_version + " or recreate " + files_form + " with lower " \
            "MACOSX_DEPLOYMENT_TARGET:  \n" + problematic_files

        if "MACOSX_DEPLOYMENT_TARGET" in os.environ:
            error_message = error_message.format("is set in MACOSX_DEPLOYMENT_TARGET variable.")
        else:
            error_message = error_message.format(
                "the version your Python interpreter is compiled against.")

        sys.stderr.write(error_message)

    platform_tag = prefix + "_" + fin_base_version + "_" + suffix
    return platform_tag