pygame · illume · Oct 16, 2021 · Oct 10, 2021
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -27,7 +27,37 @@ on:
     - '.github/workflows/sdl1-sdist.yml'
 
 jobs:
+  deps:
+    runs-on: macos-10.15
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Test for Mac Deps cache hit
+        id: macdep-cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ github.workspace }}/pygame_mac_deps
+          # The hash of all files in buildconfig manylinux-build and macdependencies is 
+          # the key to the cache. If anything changes here, the deps are built again
+          key: macdep-${{ hashFiles('buildconfig/manylinux-build/**') }}-${{ hashFiles('buildconfig/macdependencies/*.sh') }}
+
+      # build mac deps on cache miss
+      - name: Build Mac Deps
+        if: steps.macdep-cache.outputs.cache-hit != 'true'
+        run: |
+          brew install coreutils
+          cd buildconfig/macdependencies
+          bash ./build_mac_deps.sh
+
+      # Uncomment when you want to manually verify the deps by downloading them
+      # - name: Upload Mac deps
+      #   uses: actions/upload-artifact@v2
+      #   with:
+      #     name: pygame-mac-deps
+      #     path: ${{ github.workspace }}/pygame_mac_deps
+
   build:
+    needs: deps
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false  # if a particular matrix build fails, don't skip the rest
@@ -52,45 +82,43 @@ jobs:
         #   - os: windows-latest
         #     pyver: 'pp27'
 
+    env:
+      SDL_VIDEODRIVER: "dummy"
+      SDL_AUDIODRIVER: "disk"
+
+      CIBW_BEFORE_ALL_MACOS: cd buildconfig/macdependencies && bash ./install_mac_deps.sh
+
+      CIBW_BEFORE_BUILD: pip install requests numpy
+      CIBW_TEST_COMMAND: python -m pygame.tests -v --exclude opengl,timing --time_out 300
+
+      # Increase pip debugging output
+      CIBW_BUILD_VERBOSITY: 2
+
     steps:
       - uses: actions/checkout@v2
 
+      - name: Fetch Mac deps
+        id: macdep-cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ github.workspace }}/pygame_mac_deps
+          key: macdep-${{ hashFiles('buildconfig/manylinux-build/**') }}-${{ hashFiles('buildconfig/macdependencies/*.sh') }}
+
       # use cibuildwheel v1 to build on older python versions
       - name: Build wheels with CIBuildWheel v1
         # if: matrix.pyver == 'cp27' || matrix.pyver == 'cp35' || matrix.pyver == 'pp27' || matrix.pyver == 'pp36'
         if: matrix.cibuildver == 'CIBuildWheel v1'
         uses: pypa/cibuildwheel@v1.12.0
         env:
-          SDL_VIDEODRIVER: "dummy"
-          SDL_AUDIODRIVER: "disk"
-
           # CIBW_BUILD: ${{ matrix.pyver }}-*
           CIBW_BUILD: cp27-* cp35-* pp27-* pp36-*
 
-          # Install some dependencies
-          CIBW_BEFORE_BUILD_MACOS: |
-            brew install coreutils
-            cd buildconfig/macdependencies
-            ./install_mac_deps.sh
-            cd ../..
-            pip install requests numpy
-
-          CIBW_BEFORE_BUILD_WINDOWS: pip install requests numpy
-
-          CIBW_TEST_COMMAND: python -m pygame.tests -v --exclude opengl,timing --time_out 300
-
-          # Increase pip debugging output
-          CIBW_BUILD_VERBOSITY: 2
-
       # use latest cibuildwheel for newer python versions
       - name: Build wheels with CIBuildWheel v2
         # if: matrix.pyver != 'cp27' && matrix.pyver != 'cp35' && matrix.pyver != 'pp27' && matrix.pyver != 'pp36'
         if: matrix.cibuildver == 'CIBuildWheel v2'
-        uses: pypa/cibuildwheel@v2.1.2
+        uses: pypa/cibuildwheel@v2.1.3
         env:
-          SDL_VIDEODRIVER: "dummy"
-          SDL_AUDIODRIVER: "disk"
-
           # CIBW_BUILD: ${{ matrix.pyver }}-*
           CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* cp310-* pp37-*
 
@@ -100,21 +128,6 @@ jobs:
           # uncomment when arm builds are working
           # CIBW_ARCHS_MACOS: "x86_64 universal2"
 
-          # Install some dependencies
-          CIBW_BEFORE_BUILD_MACOS: |
-            brew install coreutils
-            cd buildconfig/macdependencies
-            ./install_mac_deps.sh
-            cd ../..
-            pip install requests numpy
-
-          CIBW_BEFORE_BUILD_WINDOWS: pip install requests numpy
-
-          CIBW_TEST_COMMAND: python -m pygame.tests -v --exclude opengl,timing --time_out 300
-
-          # Increase pip debugging output
-          CIBW_BUILD_VERBOSITY: 2
-
       - uses: actions/upload-artifact@v2
         with:
           name: pygame-wheels

diff --git a/buildconfig/macdependencies/build_mac_deps.sh b/buildconfig/macdependencies/build_mac_deps.sh
@@ -0,0 +1,58 @@
+# This uses manylinux build scripts to build dependencies
+#  on mac.
+#
+# Warning: this should probably not be run on your own mac.
+#   Since it will install all these deps all over the place,
+#   and they may conflict with existing installs you have.
+
+set -e -x
+
+export MACDEP_CACHE_PREFIX_PATH=${GITHUB_WORKSPACE}/pygame_mac_deps
+
+bash ./clean_usr_local.sh
+mkdir $MACDEP_CACHE_PREFIX_PATH
+
+# to use the gnu readlink, needs `brew install coreutils`
+export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+
+# for great speed.
+export MAKEFLAGS="-j 4"
+export MACOSX_DEPLOYMENT_TARGET=10.9
+
+cd ../manylinux-build/docker_base
+
+# Now start installing dependencies
+# ---------------------------------
+
+# sdl_image deps
+bash zlib/build-zlib.sh
+bash libpng/build-png.sh # depends on zlib
+bash libjpeg/build-jpeg.sh
+bash libtiff/build-tiff.sh
+bash libwebp/build-webp.sh
+
+# sdl_ttf deps
+# export EXTRA_CONFIG_FREETYPE=--without-harfbuzz
+# bash freetype/build-freetype.sh
+# bash harfbuzz/build-harfbuzz.sh
+# export EXTRA_CONFIG_FREETYPE=
+bash freetype/build-freetype.sh
+
+# sdl_mixer deps
+bash libmodplug/build-libmodplug.sh
+bash ogg/build-ogg.sh
+bash flac/build-flac.sh
+bash mpg123/build-mpg123.sh
+
+# fluidsynth (for sdl_mixer)
+bash gettext/build-gettext.sh
+bash glib/build-glib.sh # depends on gettext
+bash sndfile/build-sndfile.sh
+sudo mkdir -p /usr/local/lib64 # the install tries to put something in here
+sudo mkdir -p ${MACDEP_CACHE_PREFIX_PATH}/usr/local/lib64
+sudo bash fluidsynth/build-fluidsynth.sh # sudo otherwise install doesn't work.
+
+bash sdl_libs/build-sdl2-libs.sh
+
+# for pygame.midi
+bash portmidi/build-portmidi.sh
diff --git a/buildconfig/macdependencies/clean_usr_local.sh b/buildconfig/macdependencies/clean_usr_local.sh
@@ -0,0 +1,43 @@
+# Cleans /usr/local for the install of mac deps, deleting things that are not
+# required, or things that will be replaced with something else
+
+
+# First clean up some homebrew stuff we don't want linked in
+# ----------------------------------------------------------
+
+rm -rf /usr/local/bin/curl
+rm -rf /usr/local/opt/curl
+rm -rf /usr/local/bin/git
+rm -rf /usr/local/opt/git
+# Use the apple provided curl, and git.
+#     The homebrew ones depend on libs we don't want to include.
+# ln -s /usr/bin/curl /usr/local/bin/curl
+ln -s /usr/bin/git /usr/local/bin/git
+
+rm -rf /usr/local/lib/libtiff*
+rm -rf /usr/local/lib/libsndfile*
+rm -rf /usr/local/lib/glib*
+rm -rf /usr/local/lib/libglib*
+rm -rf /usr/local/lib/libgthread*
+rm -rf /usr/local/lib/libintl*
+rm -rf /usr/local/lib/libbrotlidec*
+rm -rf /usr/local/lib/libopus*
+rm -rf /usr/local/opt/freetype
+
+rm -rf /usr/local/Cellar/libtiff
+rm -rf /usr/local/Cellar/libsndfile
+rm -rf /usr/local/Cellar/glib
+rm -rf /usr/local/Cellar/brotli
+rm -rf /usr/local/Cellar/pcre
+rm -rf /usr/local/Cellar/opus
+rm -rf /usr/local/Cellar/freetype
+
+rm -rf /usr/local/opt/gettext
+
+rm -rf /usr/local/share/doc/tiff-*
+rm -rf /usr/local/share/doc/libsndfile
+rm -rf /usr/local/share/glib-2.0
+rm -rf /usr/local/share/gdb/auto-load
+
+rm -rf /usr/local/include/glib-2.0
+rm -rf /usr/local/include/gio-unix-2.0
diff --git a/buildconfig/macdependencies/install_mac_deps.py b/buildconfig/macdependencies/install_mac_deps.py
@@ -0,0 +1,61 @@
+"""
+A python helper script to install built (cached) mac deps into /usr/local
+"""
+
+import shutil
+import sys
+from pathlib import Path
+
+
+def rmpath(path: Path, verbose: bool = False):
+    """
+    Tries to remove a path of any kind
+    """
+    if path.is_symlink():
+        if verbose:
+            print(f"- Removing existing symlink at '{path}'")
+
+        path.unlink()
+
+    elif path.is_file():
+        if verbose:
+            print(f"- Removing existing file at '{path}'")
+
+        path.unlink()
+
+    elif path.is_dir():
+        if verbose:
+            print(f"- Removing existing directory at '{path}'")
+
+        shutil.rmtree(path)
+
+
+def symtree(srcdir: Path, destdir: Path, verbose: bool = False):
+    """
+    This function creates symlinks pointing to srcdir, from destdir, such that
+    existing folders and files in the tree of destdir are retained
+    """
+    if not destdir.is_dir():
+        # dest dir does not exist at all, create dir symlink
+        rmpath(destdir, verbose)
+        if verbose:
+            print(
+                f"- Creating directory symlink from '{destdir}' pointing to '{srcdir}'"
+            )
+
+        destdir.symlink_to(srcdir)
+        return
+
+    for path in srcdir.glob("*"):
+        destpath = destdir / path.name
+        if path.is_dir():
+            symtree(path, destpath, verbose)
+        else:
+            rmpath(destpath, verbose)
+            if verbose:
+                print(f"- Creating file symlink from '{destpath}' pointing to '{path}'")
+
+            destpath.symlink_to(path)
+
+
+symtree(Path(sys.argv[1]), Path("/"), verbose=True)
diff --git a/buildconfig/macdependencies/install_mac_deps.sh b/buildconfig/macdependencies/install_mac_deps.sh
@@ -1,95 +1,9 @@
-# This uses manylinux build scripts to build dependencies
-#  on mac.
-#
-# Warning: this should probably not be run on your own mac.
-#   Since it will install all these deps all over the place,
-#   and they may conflict with existing installs you have.
-
+# A script to install mac deps in /usr/local
 set -e -x
-cd ../manylinux-build/docker_base
-
-# to use the gnu readlink, needs `brew install coreutils`
-export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
-
-# for great speed.
-export MAKEFLAGS="-j 4"
-export MACOSX_DEPLOYMENT_TARGET=10.9
-
-
-# First clean up some homebrew stuff we don't want linked in
-# ----------------------------------------------------------
-
-rm -rf /usr/local/bin/curl
-rm -rf /usr/local/opt/curl
-rm -rf /usr/local/bin/git
-rm -rf /usr/local/opt/git
-# Use the apple provided curl, and git.
-#     The homebrew ones depend on libs we don't want to include.
-# ln -s /usr/bin/curl /usr/local/bin/curl
-ln -s /usr/bin/git /usr/local/bin/git
-
-rm -rf /usr/local/lib/libtiff*
-rm -rf /usr/local/lib/libsndfile*
-rm -rf /usr/local/lib/glib*
-rm -rf /usr/local/lib/libglib*
-rm -rf /usr/local/lib/libgthread*
-rm -rf /usr/local/lib/libintl*
-rm -rf /usr/local/lib/libbrotlidec*
-rm -rf /usr/local/lib/libopus*
-rm -rf /usr/local/opt/freetype
-
-rm -rf /usr/local/Cellar/libtiff
-rm -rf /usr/local/Cellar/libsndfile
-rm -rf /usr/local/Cellar/glib
-rm -rf /usr/local/Cellar/brotli
-rm -rf /usr/local/Cellar/pcre
-rm -rf /usr/local/Cellar/opus
-rm -rf /usr/local/Cellar/freetype
-
-rm -rf /usr/local/opt/gettext
-
-rm -rf /usr/local/share/doc/tiff-*
-rm -rf /usr/local/share/doc/libsndfile
-rm -rf /usr/local/share/glib-2.0
-rm -rf /usr/local/share/gdb/auto-load
-
-rm -rf /usr/local/include/glib-2.0
-rm -rf /usr/local/include/gio-unix-2.0
-
-
-# Now start installing dependencies
-# ---------------------------------
-
-# sdl_image deps
-bash zlib/build-zlib.sh
-bash libpng/build-png.sh # depends on zlib
-bash libjpeg/build-jpeg.sh
-bash libtiff/build-tiff.sh
-bash libwebp/build-webp.sh
-
-# sdl_ttf deps
-# export EXTRA_CONFIG_FREETYPE=--without-harfbuzz
-# bash freetype/build-freetype.sh
-# bash harfbuzz/build-harfbuzz.sh
-# export EXTRA_CONFIG_FREETYPE=
-bash freetype/build-freetype.sh
-
-# sdl_mixer deps
-bash libmodplug/build-libmodplug.sh
-bash ogg/build-ogg.sh
-bash flac/build-flac.sh
-bash mpg123/build-mpg123.sh
-
-# fluidsynth (for sdl_mixer)
-bash gettext/build-gettext.sh
-bash glib/build-glib.sh # depends on gettext
-bash sndfile/build-sndfile.sh
-sudo mkdir -p /usr/local/lib64 # the install tries to put something in here
-sudo bash fluidsynth/build-fluidsynth.sh # sudo otherwise install doesn't work.
 
-bash sdl_libs/build-sdl2-libs.sh
+bash ./clean_usr_local.sh
+sudo python3 install_mac_deps.py ${GITHUB_WORKSPACE}/pygame_mac_deps
 
-# for pygame.midi
-bash portmidi/build-portmidi.sh
-# strangely somehow the built pygame links against the libportmidi.dylib here:
-cp /usr/local/lib/libportmidi.dylib /Users/runner/work/pygame/pygame/libportmidi.dylib
+# strangely somehow the built pygame links against the libportmidi.dylib here, so
+# copy the dylib
+cp /usr/local/lib/libportmidi.dylib ${GITHUB_WORKSPACE}/libportmidi.dylib