Skip to content

Commit

Permalink
Merge pull request #2762 from pygame/ankith26-gh-actions-cache
Browse files Browse the repository at this point in the history
GH Actions make /usr/local cache for mac deps
  • Loading branch information
illume committed Oct 16, 2021
2 parents 867ce99 + 122b60f commit 68de48c
Show file tree
Hide file tree
Showing 23 changed files with 381 additions and 133 deletions.
87 changes: 50 additions & 37 deletions .github/workflows/build.yml
Expand Up @@ -27,7 +27,37 @@ on:
- '.github/workflows/sdl1-sdist.yml'

jobs:
deps:
runs-on: macos-10.15
steps:
- uses: actions/checkout@v2

- name: Test for Mac Deps cache hit
id: macdep-cache
uses: actions/cache@v2
with:
path: ${{ github.workspace }}/pygame_mac_deps
# The hash of all files in buildconfig manylinux-build and macdependencies is
# the key to the cache. If anything changes here, the deps are built again
key: macdep-${{ hashFiles('buildconfig/manylinux-build/**') }}-${{ hashFiles('buildconfig/macdependencies/*.sh') }}

# build mac deps on cache miss
- name: Build Mac Deps
if: steps.macdep-cache.outputs.cache-hit != 'true'
run: |
brew install coreutils
cd buildconfig/macdependencies
bash ./build_mac_deps.sh
# Uncomment when you want to manually verify the deps by downloading them
# - name: Upload Mac deps
# uses: actions/upload-artifact@v2
# with:
# name: pygame-mac-deps
# path: ${{ github.workspace }}/pygame_mac_deps

build:
needs: deps
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false # if a particular matrix build fails, don't skip the rest
Expand All @@ -52,45 +82,43 @@ jobs:
# - os: windows-latest
# pyver: 'pp27'

env:
SDL_VIDEODRIVER: "dummy"
SDL_AUDIODRIVER: "disk"

CIBW_BEFORE_ALL_MACOS: cd buildconfig/macdependencies && bash ./install_mac_deps.sh

CIBW_BEFORE_BUILD: pip install requests numpy
CIBW_TEST_COMMAND: python -m pygame.tests -v --exclude opengl,timing --time_out 300

# Increase pip debugging output
CIBW_BUILD_VERBOSITY: 2

steps:
- uses: actions/checkout@v2

- name: Fetch Mac deps
id: macdep-cache
uses: actions/cache@v2
with:
path: ${{ github.workspace }}/pygame_mac_deps
key: macdep-${{ hashFiles('buildconfig/manylinux-build/**') }}-${{ hashFiles('buildconfig/macdependencies/*.sh') }}

# use cibuildwheel v1 to build on older python versions
- name: Build wheels with CIBuildWheel v1
# if: matrix.pyver == 'cp27' || matrix.pyver == 'cp35' || matrix.pyver == 'pp27' || matrix.pyver == 'pp36'
if: matrix.cibuildver == 'CIBuildWheel v1'
uses: pypa/cibuildwheel@v1.12.0
env:
SDL_VIDEODRIVER: "dummy"
SDL_AUDIODRIVER: "disk"

# CIBW_BUILD: ${{ matrix.pyver }}-*
CIBW_BUILD: cp27-* cp35-* pp27-* pp36-*

# Install some dependencies
CIBW_BEFORE_BUILD_MACOS: |
brew install coreutils
cd buildconfig/macdependencies
./install_mac_deps.sh
cd ../..
pip install requests numpy
CIBW_BEFORE_BUILD_WINDOWS: pip install requests numpy

CIBW_TEST_COMMAND: python -m pygame.tests -v --exclude opengl,timing --time_out 300

# Increase pip debugging output
CIBW_BUILD_VERBOSITY: 2

# use latest cibuildwheel for newer python versions
- name: Build wheels with CIBuildWheel v2
# if: matrix.pyver != 'cp27' && matrix.pyver != 'cp35' && matrix.pyver != 'pp27' && matrix.pyver != 'pp36'
if: matrix.cibuildver == 'CIBuildWheel v2'
uses: pypa/cibuildwheel@v2.1.2
uses: pypa/cibuildwheel@v2.1.3
env:
SDL_VIDEODRIVER: "dummy"
SDL_AUDIODRIVER: "disk"

# CIBW_BUILD: ${{ matrix.pyver }}-*
CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* cp310-* pp37-*

Expand All @@ -100,21 +128,6 @@ jobs:
# uncomment when arm builds are working
# CIBW_ARCHS_MACOS: "x86_64 universal2"

# Install some dependencies
CIBW_BEFORE_BUILD_MACOS: |
brew install coreutils
cd buildconfig/macdependencies
./install_mac_deps.sh
cd ../..
pip install requests numpy
CIBW_BEFORE_BUILD_WINDOWS: pip install requests numpy

CIBW_TEST_COMMAND: python -m pygame.tests -v --exclude opengl,timing --time_out 300

# Increase pip debugging output
CIBW_BUILD_VERBOSITY: 2

- uses: actions/upload-artifact@v2
with:
name: pygame-wheels
Expand Down
58 changes: 58 additions & 0 deletions buildconfig/macdependencies/build_mac_deps.sh
@@ -0,0 +1,58 @@
# This uses manylinux build scripts to build dependencies
# on mac.
#
# Warning: this should probably not be run on your own mac.
# Since it will install all these deps all over the place,
# and they may conflict with existing installs you have.

set -e -x

export MACDEP_CACHE_PREFIX_PATH=${GITHUB_WORKSPACE}/pygame_mac_deps

bash ./clean_usr_local.sh
mkdir $MACDEP_CACHE_PREFIX_PATH

# to use the gnu readlink, needs `brew install coreutils`
export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"

# for great speed.
export MAKEFLAGS="-j 4"
export MACOSX_DEPLOYMENT_TARGET=10.9

cd ../manylinux-build/docker_base

# Now start installing dependencies
# ---------------------------------

# sdl_image deps
bash zlib/build-zlib.sh
bash libpng/build-png.sh # depends on zlib
bash libjpeg/build-jpeg.sh
bash libtiff/build-tiff.sh
bash libwebp/build-webp.sh

# sdl_ttf deps
# export EXTRA_CONFIG_FREETYPE=--without-harfbuzz
# bash freetype/build-freetype.sh
# bash harfbuzz/build-harfbuzz.sh
# export EXTRA_CONFIG_FREETYPE=
bash freetype/build-freetype.sh

# sdl_mixer deps
bash libmodplug/build-libmodplug.sh
bash ogg/build-ogg.sh
bash flac/build-flac.sh
bash mpg123/build-mpg123.sh

# fluidsynth (for sdl_mixer)
bash gettext/build-gettext.sh
bash glib/build-glib.sh # depends on gettext
bash sndfile/build-sndfile.sh
sudo mkdir -p /usr/local/lib64 # the install tries to put something in here
sudo mkdir -p ${MACDEP_CACHE_PREFIX_PATH}/usr/local/lib64
sudo bash fluidsynth/build-fluidsynth.sh # sudo otherwise install doesn't work.

bash sdl_libs/build-sdl2-libs.sh

# for pygame.midi
bash portmidi/build-portmidi.sh
43 changes: 43 additions & 0 deletions buildconfig/macdependencies/clean_usr_local.sh
@@ -0,0 +1,43 @@
# Cleans /usr/local for the install of mac deps, deleting things that are not
# required, or things that will be replaced with something else


# First clean up some homebrew stuff we don't want linked in
# ----------------------------------------------------------

rm -rf /usr/local/bin/curl
rm -rf /usr/local/opt/curl
rm -rf /usr/local/bin/git
rm -rf /usr/local/opt/git
# Use the apple provided curl, and git.
# The homebrew ones depend on libs we don't want to include.
# ln -s /usr/bin/curl /usr/local/bin/curl
ln -s /usr/bin/git /usr/local/bin/git

rm -rf /usr/local/lib/libtiff*
rm -rf /usr/local/lib/libsndfile*
rm -rf /usr/local/lib/glib*
rm -rf /usr/local/lib/libglib*
rm -rf /usr/local/lib/libgthread*
rm -rf /usr/local/lib/libintl*
rm -rf /usr/local/lib/libbrotlidec*
rm -rf /usr/local/lib/libopus*
rm -rf /usr/local/opt/freetype

rm -rf /usr/local/Cellar/libtiff
rm -rf /usr/local/Cellar/libsndfile
rm -rf /usr/local/Cellar/glib
rm -rf /usr/local/Cellar/brotli
rm -rf /usr/local/Cellar/pcre
rm -rf /usr/local/Cellar/opus
rm -rf /usr/local/Cellar/freetype

rm -rf /usr/local/opt/gettext

rm -rf /usr/local/share/doc/tiff-*
rm -rf /usr/local/share/doc/libsndfile
rm -rf /usr/local/share/glib-2.0
rm -rf /usr/local/share/gdb/auto-load

rm -rf /usr/local/include/glib-2.0
rm -rf /usr/local/include/gio-unix-2.0
61 changes: 61 additions & 0 deletions buildconfig/macdependencies/install_mac_deps.py
@@ -0,0 +1,61 @@
"""
A python helper script to install built (cached) mac deps into /usr/local
"""

import shutil
import sys
from pathlib import Path


def rmpath(path: Path, verbose: bool = False):
"""
Tries to remove a path of any kind
"""
if path.is_symlink():
if verbose:
print(f"- Removing existing symlink at '{path}'")

path.unlink()

elif path.is_file():
if verbose:
print(f"- Removing existing file at '{path}'")

path.unlink()

elif path.is_dir():
if verbose:
print(f"- Removing existing directory at '{path}'")

shutil.rmtree(path)


def symtree(srcdir: Path, destdir: Path, verbose: bool = False):
"""
This function creates symlinks pointing to srcdir, from destdir, such that
existing folders and files in the tree of destdir are retained
"""
if not destdir.is_dir():
# dest dir does not exist at all, create dir symlink
rmpath(destdir, verbose)
if verbose:
print(
f"- Creating directory symlink from '{destdir}' pointing to '{srcdir}'"
)

destdir.symlink_to(srcdir)
return

for path in srcdir.glob("*"):
destpath = destdir / path.name
if path.is_dir():
symtree(path, destpath, verbose)
else:
rmpath(destpath, verbose)
if verbose:
print(f"- Creating file symlink from '{destpath}' pointing to '{path}'")

destpath.symlink_to(path)


symtree(Path(sys.argv[1]), Path("/"), verbose=True)
98 changes: 6 additions & 92 deletions buildconfig/macdependencies/install_mac_deps.sh
@@ -1,95 +1,9 @@
# This uses manylinux build scripts to build dependencies
# on mac.
#
# Warning: this should probably not be run on your own mac.
# Since it will install all these deps all over the place,
# and they may conflict with existing installs you have.

# A script to install mac deps in /usr/local
set -e -x
cd ../manylinux-build/docker_base

# to use the gnu readlink, needs `brew install coreutils`
export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"

# for great speed.
export MAKEFLAGS="-j 4"
export MACOSX_DEPLOYMENT_TARGET=10.9


# First clean up some homebrew stuff we don't want linked in
# ----------------------------------------------------------

rm -rf /usr/local/bin/curl
rm -rf /usr/local/opt/curl
rm -rf /usr/local/bin/git
rm -rf /usr/local/opt/git
# Use the apple provided curl, and git.
# The homebrew ones depend on libs we don't want to include.
# ln -s /usr/bin/curl /usr/local/bin/curl
ln -s /usr/bin/git /usr/local/bin/git

rm -rf /usr/local/lib/libtiff*
rm -rf /usr/local/lib/libsndfile*
rm -rf /usr/local/lib/glib*
rm -rf /usr/local/lib/libglib*
rm -rf /usr/local/lib/libgthread*
rm -rf /usr/local/lib/libintl*
rm -rf /usr/local/lib/libbrotlidec*
rm -rf /usr/local/lib/libopus*
rm -rf /usr/local/opt/freetype

rm -rf /usr/local/Cellar/libtiff
rm -rf /usr/local/Cellar/libsndfile
rm -rf /usr/local/Cellar/glib
rm -rf /usr/local/Cellar/brotli
rm -rf /usr/local/Cellar/pcre
rm -rf /usr/local/Cellar/opus
rm -rf /usr/local/Cellar/freetype

rm -rf /usr/local/opt/gettext

rm -rf /usr/local/share/doc/tiff-*
rm -rf /usr/local/share/doc/libsndfile
rm -rf /usr/local/share/glib-2.0
rm -rf /usr/local/share/gdb/auto-load

rm -rf /usr/local/include/glib-2.0
rm -rf /usr/local/include/gio-unix-2.0


# Now start installing dependencies
# ---------------------------------

# sdl_image deps
bash zlib/build-zlib.sh
bash libpng/build-png.sh # depends on zlib
bash libjpeg/build-jpeg.sh
bash libtiff/build-tiff.sh
bash libwebp/build-webp.sh

# sdl_ttf deps
# export EXTRA_CONFIG_FREETYPE=--without-harfbuzz
# bash freetype/build-freetype.sh
# bash harfbuzz/build-harfbuzz.sh
# export EXTRA_CONFIG_FREETYPE=
bash freetype/build-freetype.sh

# sdl_mixer deps
bash libmodplug/build-libmodplug.sh
bash ogg/build-ogg.sh
bash flac/build-flac.sh
bash mpg123/build-mpg123.sh

# fluidsynth (for sdl_mixer)
bash gettext/build-gettext.sh
bash glib/build-glib.sh # depends on gettext
bash sndfile/build-sndfile.sh
sudo mkdir -p /usr/local/lib64 # the install tries to put something in here
sudo bash fluidsynth/build-fluidsynth.sh # sudo otherwise install doesn't work.

bash sdl_libs/build-sdl2-libs.sh
bash ./clean_usr_local.sh
sudo python3 install_mac_deps.py ${GITHUB_WORKSPACE}/pygame_mac_deps

# for pygame.midi
bash portmidi/build-portmidi.sh
# strangely somehow the built pygame links against the libportmidi.dylib here:
cp /usr/local/lib/libportmidi.dylib /Users/runner/work/pygame/pygame/libportmidi.dylib
# strangely somehow the built pygame links against the libportmidi.dylib here, so
# copy the dylib
cp /usr/local/lib/libportmidi.dylib ${GITHUB_WORKSPACE}/libportmidi.dylib

0 comments on commit 68de48c

Please sign in to comment.