[WIP] Python 3.12 support

lief-project · Oct 22, 2023 · bfa3d55 · bfa3d55
1 parent e5ef1da
commit bfa3d55
Show file tree

Hide file tree

Showing 24 changed files with 82 additions and 138 deletions.
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
@@ -36,7 +36,7 @@ jobs:
            -e CCACHE_DIR=/ccache      \
            -v $GITHUB_WORKSPACE:/work \
            -v $HOME/.ccache:/ccache   \
-          --rm liefproject/android-${ARCH} bash /work/scripts/docker/android-${ARCH}.sh
+          --rm liefproject/android-${ARCH} bash /work/scripts/docker/android-sdk-${ARCH}.sh
       - name: 'Upload Artifact'
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/linux-aarch64.yml b/.github/workflows/linux-aarch64.yml
@@ -31,7 +31,7 @@ jobs:
             -v $GITHUB_WORKSPACE:/work            \
             -v $RUNNER_TEMP/lief-install:/install \
             -v $HOME/.ccache:/ccache              \
-            liefproject/manylinux2014-aarch64 bash /work/scripts/docker/linux-aarch64.sh
+            liefproject/manylinux2014-aarch64 bash /work/scripts/docker/linux-sdk-aarch64
       - name: 'Upload SDK Artifact'
         uses: actions/upload-artifact@v3
         with:
@@ -49,7 +49,7 @@ jobs:
     needs: linux-sdk
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', '3.11']
+        python-version: ['3.12']
     steps:
       - uses: actions/checkout@v3
         with:
@@ -87,7 +87,7 @@ jobs:
            -v $GITHUB_WORKSPACE:/work                           \
            -v $HOME/.ccache:/ccache                             \
            -v $RUNNER_TEMP/lief-install:/install                \
-           liefproject/manylinux2014-aarch64 bash /work/scripts/docker/manylinux2014-aarch64.sh
+           liefproject/manylinux2014-aarch64 bash /work/scripts/docker/linux-py-aarch64
       - name: 'Artifact'
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/linux-x86-64.yml b/.github/workflows/linux-x86-64.yml
@@ -24,14 +24,15 @@ jobs:
         shell: bash
         run: |
           mkdir -p $RUNNER_TEMP/lief-install
-          docker run --name lief_sdk --rm                        \
-            -e CCACHE_COMPRESS=1                                 \
-            -e CCACHE_MAXSIZE=10.0G                              \
-            -e CCACHE_DIR=/ccache                                \
-            -v $GITHUB_WORKSPACE:/src                            \
-            -v $RUNNER_TEMP/lief-install:/install                       \
-            -v $HOME/.ccache:/ccache                             \
-            liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/run_linux_sdk.sh
+          docker run --name lief_sdk --rm          \
+            -e CCACHE_COMPRESS=1                   \
+            -e CCACHE_MAXSIZE=10.0G                \
+            -e CCACHE_DIR=/ccache                  \
+            -v $GITHUB_WORKSPACE:/src              \
+            -v $RUNNER_TEMP/lief-install:/install  \
+            -v $HOME/.ccache:/ccache               \
+            liefproject/manylinux_2_28_x86_64      \
+            bash /src/scripts/docker/linux-sdk-x64
       - name: 'Upload SDK Artifact'
         uses: actions/upload-artifact@v3
         with:
@@ -77,13 +78,14 @@ jobs:
            -e PYTHON_BINARY=/opt/python/cp38-cp38/bin/python3.8 \
            -v $GITHUB_WORKSPACE:/src                            \
            -v $HOME/.ccache:/ccache                             \
-           liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/run_linux_test.sh
+           liefproject/manylinux_2_28_x86_64 \
+           bash /src/scripts/docker/test-linux-x64
   pywheels:
     runs-on: ubuntu-latest
     needs: linux-sdk
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.12']
     steps:
       - uses: actions/checkout@v3
         with:
@@ -121,7 +123,8 @@ jobs:
            -v $GITHUB_WORKSPACE:/src                            \
            -v $HOME/.ccache:/ccache                             \
            -v $RUNNER_TEMP/lief-install:/install                       \
-           liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/run_linux.sh
+           liefproject/manylinux_2_28_x86_64 \
+           bash /src/scripts/docker/linux-py-x64
       - name: 'Artifact'
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
@@ -119,7 +119,7 @@ jobs:
     needs: osx-sdk
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', '3.11']
+        python-version: [3.8, 3.9, '3.10', '3.11', '3.12']
         arch: ['x64', 'arm64']
       fail-fast: false
     steps:

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -50,7 +50,7 @@ jobs:
            -e PYTHON_VERSION=$PYTHON_VERSION \
            -e PYTHON_BINARY=/opt/python/cp39-cp39/bin/python3.9 \
            -v $GITHUB_WORKSPACE:/src \
-           liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/run_linux_test.sh
+           liefproject/manylinux_2_28_x86_64 bash /src/scripts/docker/test-linux-x64
   windows:
     runs-on: windows-latest
     steps:

diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml
@@ -15,7 +15,7 @@ jobs:
            -e BUILD_DIR=/tmp/build_gcc \
            -e SRC_DIR=/src \
            -v $GITHUB_WORKSPACE:/src \
-           liefproject/sanity:latest bash /src/scripts/docker/gcc.sh
+           liefproject/sanity:latest bash /src/scripts/docker/test-gcc
   clang:
     runs-on: ubuntu-latest
     steps:
@@ -29,7 +29,7 @@ jobs:
            -e BUILD_DIR=/tmp/build_clang \
            -e SRC_DIR=/src \
            -v $GITHUB_WORKSPACE:/src \
-           liefproject/sanity:latest bash /src/scripts/docker/clang.sh
+           liefproject/sanity:latest bash /src/scripts/docker/test-clang
   asan:
     runs-on: ubuntu-latest
     steps:
@@ -41,4 +41,4 @@ jobs:
         run: |
           docker run --rm \
            -v $GITHUB_WORKSPACE:/src \
-           liefproject/sanity:latest bash /src/scripts/docker/asan_check.sh
+           liefproject/sanity:latest bash /src/scripts/docker/test-asan
diff --git a/.github/workflows/windows-all.yml b/.github/workflows/windows-all.yml
@@ -150,7 +150,7 @@ jobs:
     needs: build-sdk
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', '3.11', '3.12-dev']
+        python-version: [3.8, 3.9, '3.10', '3.11', '3.12']
         arch: ['x86', 'x64']
       fail-fast: false
     env:

diff --git a/.github/workflows/windows-quick.yml b/.github/workflows/windows-quick.yml
@@ -9,7 +9,7 @@ on:
       - "**"
 
 env:
-  python-version: '3.9'
+  python-version: '3.12'
 
 jobs:
   quick-test:

diff --git a/api/python/setup.py b/api/python/setup.py
@@ -377,7 +377,9 @@ def __init__(self, *args, **kwargs):
     def run(self):
         for ext in self.extensions:
             self.build_extension(ext)
-        self.copy_extensions_to_source()
+
+        if self.inplace:
+            self.copy_extensions_to_source()
 
     def _fix_platform(self):
         if sys.platform == "darwin":
@@ -498,9 +500,8 @@ class egg_info(_egg_info):
     long_description=long_description.read_text(),
     long_description_content_type="text/x-rst; charset=UTF-8",
     distclass=LiefDistribution,
-    scripts=['examples/elf_reader.py', 'examples/pe_reader.py', 'examples/macho_reader.py'],
     packages=["lief"],
-    package_data={"lief": ["py.typed", "*.pyi", "*.so", "*.pyd"]},
+    package_data={"lief": ["py.typed", "*.pyi", "*.pyd"]},
     ext_modules=[Module("lief._lief")],
     cmdclass=cmdclass,
     version=version

diff --git a/scripts/docker/android-arm.sh → scripts/docker/android-sdk-arm b/scripts/docker/android-arm.sh → scripts/docker/android-sdk-arm
@@ -11,27 +11,27 @@ git config --global --add safe.directory /work
 mkdir -p build/$ARCH_DIR/static-release && mkdir -p build/$ARCH_DIR/shared-release
 pushd build/$ARCH_DIR/shared-release
 
-cmake ../../.. -GNinja \
-  -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
-  -DCMAKE_C_FLAGS="$CFLAGS" \
+cmake ../../.. -GNinja                            \
+  -DCMAKE_CXX_FLAGS="$CXXFLAGS"                   \
+  -DCMAKE_C_FLAGS="$CFLAGS"                       \
   -DCMAKE_SHARED_LINKER_FLAGS="-static-libstdc++" \
-  -DCMAKE_LINK_WHAT_YOU_USE=on \
-  -DBUILD_SHARED_LIBS=on \
-  -DLIEF_PYTHON_API=off \
-  -DLIEF_INSTALL_COMPILED_EXAMPLES=off \
+  -DCMAKE_LINK_WHAT_YOU_USE=on                    \
+  -DBUILD_SHARED_LIBS=on                          \
+  -DLIEF_PYTHON_API=off                           \
+  -DLIEF_INSTALL_COMPILED_EXAMPLES=off            \
   -DCMAKE_BUILD_TYPE=Release
 
 ninja
 
 popd
 pushd build/$ARCH_DIR/static-release
 
-cmake ../../.. -GNinja \
-  -DCMAKE_LINK_WHAT_YOU_USE=on \
-  -DBUILD_SHARED_LIBS=off \
+cmake ../../.. -GNinja                         \
+  -DCMAKE_LINK_WHAT_YOU_USE=on                 \
+  -DBUILD_SHARED_LIBS=off                      \
   -DCMAKE_EXE_LINKER_FLAGS="-static-libstdc++" \
-  -DLIEF_PYTHON_API=off \
-  -DLIEF_INSTALL_COMPILED_EXAMPLES=on \
+  -DLIEF_PYTHON_API=off                        \
+  -DLIEF_INSTALL_COMPILED_EXAMPLES=on          \
   -DCMAKE_BUILD_TYPE=Release
 
 ninja

diff --git a/scripts/docker/android-arm64.sh → scripts/docker/android-sdk-arm64 b/scripts/docker/android-arm64.sh → scripts/docker/android-sdk-arm64
@@ -11,27 +11,27 @@ git config --global --add safe.directory /work
 mkdir -p build/$ARCH_DIR/static-release && mkdir -p build/$ARCH_DIR/shared-release
 pushd build/$ARCH_DIR/shared-release
 
-cmake ../../.. -GNinja \
-  -DCMAKE_CXX_FLAGS="$CXXFLAGS" \
-  -DCMAKE_C_FLAGS="$CFLAGS" \
+cmake ../../.. -GNinja                            \
+  -DCMAKE_CXX_FLAGS="$CXXFLAGS"                   \
+  -DCMAKE_C_FLAGS="$CFLAGS"                       \
   -DCMAKE_SHARED_LINKER_FLAGS="-static-libstdc++" \
-  -DCMAKE_LINK_WHAT_YOU_USE=on \
-  -DBUILD_SHARED_LIBS=on \
-  -DLIEF_PYTHON_API=off \
-  -DLIEF_INSTALL_COMPILED_EXAMPLES=off \
+  -DCMAKE_LINK_WHAT_YOU_USE=on                    \
+  -DBUILD_SHARED_LIBS=on                          \
+  -DLIEF_PYTHON_API=off                           \
+  -DLIEF_INSTALL_COMPILED_EXAMPLES=off            \
   -DCMAKE_BUILD_TYPE=Release
 
 ninja
 
 popd
 pushd build/$ARCH_DIR/static-release
 
-cmake ../../.. -GNinja \
-  -DCMAKE_LINK_WHAT_YOU_USE=on \
-  -DBUILD_SHARED_LIBS=off \
+cmake ../../.. -GNinja                         \
+  -DCMAKE_LINK_WHAT_YOU_USE=on                 \
+  -DBUILD_SHARED_LIBS=off                      \
   -DCMAKE_EXE_LINKER_FLAGS="-static-libstdc++" \
-  -DLIEF_PYTHON_API=off \
-  -DLIEF_INSTALL_COMPILED_EXAMPLES=on \
+  -DLIEF_PYTHON_API=off                        \
+  -DLIEF_INSTALL_COMPILED_EXAMPLES=on          \
   -DCMAKE_BUILD_TYPE=Release
 
 ninja

diff --git a/scripts/docker/pylinux-aarch64.toml → scripts/docker/config/pylinux-aarch64.toml b/scripts/docker/pylinux-aarch64.toml → scripts/docker/config/pylinux-aarch64.toml
diff --git a/scripts/docker/pylinux-test-x64.toml → scripts/docker/config/pylinux-test-x64.toml b/scripts/docker/pylinux-test-x64.toml → scripts/docker/config/pylinux-test-x64.toml
diff --git a/scripts/docker/pylinux-x64.toml → scripts/docker/config/pylinux-x64.toml b/scripts/docker/pylinux-x64.toml → scripts/docker/config/pylinux-x64.toml
diff --git a/scripts/docker/linux-py-aarch64 b/scripts/docker/linux-py-aarch64
@@ -0,0 +1,18 @@
+#!/usr/bin/bash
+set -ex
+
+export CXXFLAGS='-ffunction-sections -fdata-sections -fvisibility-inlines-hidden -static-libgcc'
+export CFLAGS='-ffunction-sections -fdata-sections -static-libgcc'
+export LDFLAGS='-Wl,--gc-sections'
+export LIEF_BUILD_DIR=/tmp/lief-build
+export _PYTHON_HOST_PLATFORM="manylinux2014-aarch64"
+export SETUPTOOLS_EXT_SUFFIX=$($PYTHON_BINARY -c "import sysconfig;print(sysconfig.get_config_var('EXT_SUFFIX').replace('x86_64', 'aarch64'))")
+
+git config --global --add safe.directory /work
+
+$PYTHON_BINARY -m pip install tomli pip wheel
+
+pushd /work/api/python
+export PYLIEF_CONF=/work/scripts/docker/config/pylinux-aarch64.toml
+$PYTHON_BINARY -m pip -vvv wheel --no-build-isolation --wheel-dir=/work/dist .
+popd
diff --git a/scripts/docker/run_linux.sh → scripts/docker/linux-py-x64 b/scripts/docker/run_linux.sh → scripts/docker/linux-py-x64
@@ -4,18 +4,16 @@ set -ex
 export CXXFLAGS='-ffunction-sections -fdata-sections -fvisibility-inlines-hidden -static-libgcc'
 export CFLAGS='-ffunction-sections -fdata-sections -static-libgcc'
 export LDFLAGS='-Wl,--gc-sections -Wl,--strip-all'
-
-export LIEF_BUILD_DIR="/tmp/lief-build"
+export PYLIEF_CONF=/src/scripts/docker/config/pylinux-x64.toml
+export LIEF_BUILD_DIR=/tmp/lief-build
 
 $PYTHON_BINARY -m pip install tomli pip wheel
 
 pushd /src/api/python
-export PYLIEF_CONF=/src/scripts/docker/pylinux-x64.toml \
 
 $PYTHON_BINARY -m pip -vvv wheel --no-build-isolation --wheel-dir=/src/wheel_stage .
 $PYTHON_BINARY -m pip -vvv install --user .
 popd
 
-find /src/wheel_stage -iname "*-cp${PYTHON_VERSION}-*" -exec auditwheel repair -w /src/dist {} \;
-
-chown -R 1000:1000 /src/dist /src/wheel_stage
+find /src/wheel_stage -iname "*-cp${PYTHON_VERSION}-*" \
+     -exec auditwheel repair -w /src/dist {} \;
diff --git a/scripts/docker/linux-aarch64.sh → scripts/docker/linux-sdk-aarch64 b/scripts/docker/linux-aarch64.sh → scripts/docker/linux-sdk-aarch64
@@ -1,24 +1,15 @@
 #!/usr/bin/bash
 set -ex
 
-# Script to be run with liefproject/manylinux2014-aarch64:
-# ==============================================================================================
-# docker run \
-#  -e CCACHE_DIR=/ccache \
-#  -v $LIEF_SRC:/work \
-#  -v $HOME/.ccache:/ccache \
-#  --rm liefproject/manylinux2014-aarch64 bash /work/scripts/docker/linux-aarch64.sh
-# ==============================================================================================
-#
-
 CXXFLAGS='-ffunction-sections -fdata-sections -fvisibility-inlines-hidden -static-libgcc -static-libstdc++'
 CFLAGS='-ffunction-sections -fdata-sections -static-libgcc'
 export LDFLAGS='-Wl,--gc-sections -Wl,--exclude-libs,ALL'
 
 ARCH_DIR="linux-aarch64"
 git config --global --add safe.directory /work
 
-mkdir -p build/$ARCH_DIR/static-release && mkdir -p build/$ARCH_DIR/shared-release
+mkdir -p build/$ARCH_DIR/static-release
+mkdir -p build/$ARCH_DIR/shared-release
 
 cmake -S /work -B /work/build/$ARCH_DIR/shared-release -GNinja \
   -DCMAKE_CXX_FLAGS="$CXXFLAGS"                                \
@@ -46,5 +37,3 @@ popd
 
 /bin/mv build/$ARCH_DIR/*.tar.gz build/
 ls -alh build
-
-chown -R 1000:1000 build/
diff --git a/scripts/docker/run_linux_sdk.sh → scripts/docker/linux-sdk-x64 b/scripts/docker/run_linux_sdk.sh → scripts/docker/linux-sdk-x64
diff --git a/scripts/docker/manylinux2014-aarch64.sh b/scripts/docker/manylinux2014-aarch64.sh
diff --git a/scripts/docker/run_archlinux.sh b/scripts/docker/run_archlinux.sh
diff --git a/scripts/docker/asan_check.sh → scripts/docker/test-asan b/scripts/docker/asan_check.sh → scripts/docker/test-asan
diff --git a/scripts/docker/clang.sh → scripts/docker/test-clang b/scripts/docker/clang.sh → scripts/docker/test-clang
diff --git a/scripts/docker/gcc.sh → scripts/docker/test-gcc b/scripts/docker/gcc.sh → scripts/docker/test-gcc
@@ -2,5 +2,6 @@
 set -ex
 cmake -B ${BUILD_DIR} -S ${SRC_DIR} -GNinja \
       -DLIEF_USE_CCACHE=off \
-      -DCMAKE_CXX_COMPILER=/usr/bin/g++ -DCMAKE_C_COMPILER=/usr/bin/gcc
+      -DCMAKE_CXX_COMPILER=/usr/bin/g++ \
+      -DCMAKE_C_COMPILER=/usr/bin/gcc
 ninja -C ${BUILD_DIR} LIB_LIEF