From 7dee26cbc87445d60cbb846e751e377c2d1aebf5 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sat, 23 Mar 2019 20:53:41 +0300 Subject: [PATCH 01/16] Remove redundant language:generic --- .travis.yml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6fb537d..16b6025f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,6 @@ matrix: # default builds for MacOS - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=2.7 @@ -48,7 +47,6 @@ matrix: - ENABLE_HEADLESS=0 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.4 @@ -56,7 +54,6 @@ matrix: - ENABLE_HEADLESS=0 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.5 @@ -64,7 +61,6 @@ matrix: - ENABLE_HEADLESS=0 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.6 @@ -72,7 +68,6 @@ matrix: - ENABLE_HEADLESS=0 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.7 @@ -83,7 +78,6 @@ matrix: # headless builds for MacOS - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=2.7 @@ -91,7 +85,6 @@ matrix: - ENABLE_HEADLESS=1 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.4 @@ -99,7 +92,6 @@ matrix: - ENABLE_HEADLESS=1 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.5 @@ -107,7 +99,6 @@ matrix: - ENABLE_HEADLESS=1 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.6 @@ -115,7 +106,6 @@ matrix: - ENABLE_HEADLESS=1 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.7 @@ -126,7 +116,6 @@ matrix: # Contrib builds for MacOS - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=2.7 @@ -134,7 +123,6 @@ matrix: - ENABLE_HEADLESS=0 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.4 @@ -142,7 +130,6 @@ matrix: - ENABLE_HEADLESS=0 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.5 @@ -150,7 +137,6 @@ matrix: - ENABLE_HEADLESS=0 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.6 @@ -158,7 +144,6 @@ matrix: - ENABLE_HEADLESS=0 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.7 @@ -169,7 +154,6 @@ matrix: # headless contrib builds for MacOS - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=2.7 @@ -177,7 +161,6 @@ matrix: - ENABLE_HEADLESS=1 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.4 @@ -185,7 +168,6 @@ matrix: - ENABLE_HEADLESS=1 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.5 @@ -193,7 +175,6 @@ matrix: - ENABLE_HEADLESS=1 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.6 @@ -201,7 +182,6 @@ matrix: - ENABLE_HEADLESS=1 - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx - language: generic osx_image: xcode8 env: - MB_PYTHON_VERSION=3.7 From 59daed260ecba41deadc4c1bde9d51695fd90930 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Fri, 22 Mar 2019 00:44:02 +0300 Subject: [PATCH 02/16] Move CMAKE_OSX_DEPLOYMENT_TARGET to setup.py --- .travis.yml | 20 -------------------- setup.py | 1 + 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 16b6025f..5945c9d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,35 +45,30 @@ matrix: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - TEST_DEPENDS=numpy==1.14.5 # headless builds for MacOS @@ -83,35 +78,30 @@ matrix: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - TEST_DEPENDS=numpy==1.14.5 # Contrib builds for MacOS @@ -121,35 +111,30 @@ matrix: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - TEST_DEPENDS=numpy==1.14.5 # headless contrib builds for MacOS @@ -159,35 +144,30 @@ matrix: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - os: osx osx_image: xcode8 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - - BDIST_PARAMS="-- -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7" - TEST_DEPENDS=numpy==1.14.5 # default builds for Linux diff --git a/setup.py b/setup.py index 68bede4f..06dfe273 100644 --- a/setup.py +++ b/setup.py @@ -141,6 +141,7 @@ def main(): cmake_args.append("-DWITH_LAPACK=OFF") # Some OSX LAPACK fns are incompatible, see # https://github.com/skvark/opencv-python/issues/21 cmake_args.append("-DCMAKE_CXX_FLAGS=-stdlib=libc++") + cmake_args.append("-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.7") if sys.platform.startswith('linux'): cmake_args.append("-DWITH_IPP=OFF") # tests fail with IPP compiled with From f08d11d7bd86ef0d4a1be727f5912c7b61a8c9a8 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Fri, 22 Mar 2019 01:16:35 +0300 Subject: [PATCH 03/16] * No need for both OPENCV_PYTHON{2,3}_INSTALL_PATH in cmake args * Add explanation to more obscure cmake args --- setup.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 06dfe273..4cf0e733 100644 --- a/setup.py +++ b/setup.py @@ -102,12 +102,16 @@ def main(): # skbuild inserts PYTHON_* vars. That doesn't satisfy opencv build scripts in case of Py3 "-DPYTHON%d_EXECUTABLE=%s" % (sys.version_info[0], sys.executable), "-DBUILD_opencv_python%d=ON" % sys.version_info[0], + + # When off, adds __init__.py and a few more helper .py's. We use our own helper files with a different structure. "-DOPENCV_SKIP_PYTHON_LOADER=ON", - "-DOPENCV_PYTHON2_INSTALL_PATH=python", - "-DOPENCV_PYTHON3_INSTALL_PATH=python", + # Relative dir to install the built module to in the build tree. + # The default is generated from sysconfig, we'd rather have a constant for simplicity + "-DOPENCV_PYTHON%d_INSTALL_PATH=python" % sys.version_info[0], # Otherwise, opencv scripts would want to install `.pyd' right into site-packages, # and skbuild bails out on seeing that "-DINSTALL_CREATE_DISTRIB=ON", + # See opencv/CMakeLists.txt for options and defaults "-DBUILD_opencv_apps=OFF", "-DBUILD_SHARED_LIBS=OFF", From c04617a2c0db330420d379b7d323c91fc4657648 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 21 Mar 2019 16:43:05 +0300 Subject: [PATCH 04/16] Upgrade to Xcode 8.3 --- .travis.yml | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5945c9d5..e7e9a5f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,38 +33,38 @@ cache: - $HOME/local_bottle_metadata # `cache: ccache: true` has no effect if `language:` is not `c` or `cpp` - $HOME/.ccache - + matrix: fast_finish: true include: # default builds for MacOS - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=0 @@ -73,31 +73,31 @@ matrix: # headless builds for MacOS - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=0 @@ -106,31 +106,31 @@ matrix: # Contrib builds for MacOS - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=1 @@ -139,31 +139,31 @@ matrix: # headless contrib builds for MacOS - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - os: osx - osx_image: xcode8 + osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=1 @@ -578,7 +578,7 @@ before_install: | source multibuild_customize.sh echo $ENABLE_CONTRIB > contrib.enabled echo $ENABLE_HEADLESS > headless.enabled - + if [ -n "$IS_OSX" ]; then TAPS="$(brew --repository)/Library/Taps" if [ -e "$TAPS/caskroom/homebrew-cask" -a -e "$TAPS/homebrew/homebrew-cask" ]; then @@ -589,10 +589,10 @@ before_install: | cd $(dirname '\''{}'\'') git clean -fxd git status' \; - + brew_cache_cleanup fi - + before_install # Not interested in travis internal scripts' output set +x @@ -613,7 +613,7 @@ script: | #otherwise, Travis logic terminates prematurely #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 trap ERR - + test "$rc" -eq 0 before_cache: | @@ -627,10 +627,10 @@ before_cache: | fi brew_cache_cleanup - + fi set +x - + after_success: | # Upload wheels to pypi if requested if [ -n "$TRAVIS_TAG" ]; then From 4f4727106c00db39a67106f9083cb9d1eb3f689e Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Tue, 19 Mar 2019 15:41:56 +0300 Subject: [PATCH 05/16] Diagnostics for a `find -exec git clean` failure --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e7e9a5f0..130df767 100644 --- a/.travis.yml +++ b/.travis.yml @@ -586,9 +586,10 @@ before_install: | fi find "$TAPS" -type d -name .git -exec \ bash -xec ' - cd $(dirname '\''{}'\'') - git clean -fxd - git status' \; + cd $(dirname '\''{}'\'') || echo "status: $?" + git clean -fxd || echo "status: $?" + sleep 1 || echo "status: $?" + git status || echo "status: $?"' \; || echo "status: $?" brew_cache_cleanup fi From 61c4172a4d95e2f96f1a06d4473f83e24d9db151 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Tue, 19 Mar 2019 12:51:03 +0300 Subject: [PATCH 06/16] Fix/improve travis_osx_brew_cache & caching logic * fix built bottle & metadata cleanup when bottle is missing * -e mode for before_cache step * faster finish when already the latest version --- .travis.yml | 4 ++-- travis_osx_brew_cache.sh | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 130df767..8772517e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -619,7 +619,7 @@ script: | before_cache: | # Cleanup dirs to be cached - set -x + set -e; set -x if [ -n "$IS_OSX" ]; then # When Taps is cached, this dir causes "Error: file exists" on `brew update` @@ -630,7 +630,7 @@ before_cache: | brew_cache_cleanup fi - set +x + set +x; set +e after_success: | # Upload wheels to pypi if requested diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 47d37326..8d2401b2 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -41,6 +41,12 @@ function brew_install_and_cache_within_time_limit { local BUILD_FROM_SOURCE INCLUDE_BUILD + if brew list --versions "$PACKAGE" && ! (brew outdated | grep -qx "$PACKAGE"); then + echo "Already installed and the latest version: $PACKAGE" + return 0 + fi + + _brew_is_bottle_available "$PACKAGE" || BUILD_FROM_SOURCE=1 [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true @@ -152,7 +158,7 @@ function brew_add_local_bottles { ) fi - if [ -n "$BOTTLE" ]; then rm "$BOTTLE"; fi + if [ -n "$BOTTLE" -a -n "$BOTTLE_EXISTS" ]; then rm "$BOTTLE"; fi rm -f "$BOTTLE_LINK" rm "$JSON" @@ -295,17 +301,14 @@ function _brew_is_bottle_available { function _brew_install_and_cache { # Install bottle or make and cache bottle. - # assumes that deps were already installed. + # assumes that deps were already installed + # and not already the latest version local PACKAGE;PACKAGE="${1:?}" local USE_BOTTLE;USE_BOTTLE="${2:?}" local VERB if brew list --versions "$PACKAGE"; then - if ! (brew outdated | grep -qx "$PACKAGE"); then - echo "Already the latest version: $PACKAGE" - return 0 - fi VERB=upgrade else VERB=install From 573480899c9c79f3d6303cd6c4167f5c76e61c0e Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Tue, 19 Mar 2019 13:10:38 +0300 Subject: [PATCH 07/16] Update caching logic to support new Homebrew in OSX 10.12 * disable auto `brew cleanup` * support formulae with disabled bottles * install alongside to avoid "runtime dependents" update https://discourse.brew.sh/t/can-i-install-a-new-version-without-having-to-upgrade-runtime-dependents/4443 --- config.sh | 4 +-- travis_osx_brew_cache.sh | 59 ++++++++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/config.sh b/config.sh index fee87537..ead68645 100644 --- a/config.sh +++ b/config.sh @@ -92,14 +92,14 @@ function pre_build { if [ -n "$IS_OSX" ]; then echo "Running for OSX" - brew update --force + brew update brew_add_local_bottles # Don't query analytical info online on `brew info`, # this takes several seconds and we don't need it # see https://docs.brew.sh/Manpage , "info formula" section export HOMEBREW_NO_GITHUB_API=1 - + echo 'Installing QT4' brew tap | grep -qxF cartr/qt4 || brew tap cartr/qt4 brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin cartr/qt4 diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 8d2401b2..c62a6235 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -24,6 +24,11 @@ BREW_TIME_LIMIT=$((30*60)) BREW_TIME_HARD_LIMIT=$((40*60)) +# Auto cleanup can delete locally-built bottles +# when the caching logic isn't prepared for that +export HOMEBREW_NO_INSTALL_CLEANUP=1 + + #Public functions @@ -34,12 +39,12 @@ function brew_install_and_cache_within_time_limit { # Exit with status 2 on any other error. ( eval "$_BREW_ERREXIT" - local PACKAGE; PACKAGE="${1:?}" || return 2 - local TIME_LIMIT;TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || return 2 - local TIME_HARD_LIMIT;TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || return 2 - local TIME_START;TIME_START=${4:-$BREW_TIME_START} || return 2 + local PACKAGE; PACKAGE="${1:?}" || exit 2 + local TIME_LIMIT;TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || exit 2 + local TIME_HARD_LIMIT;TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || exit 2 + local TIME_START;TIME_START=${4:-$BREW_TIME_START} || exit 2 - local BUILD_FROM_SOURCE INCLUDE_BUILD + local BUILD_FROM_SOURCE INCLUDE_BUILD KEG_ONLY if brew list --versions "$PACKAGE" && ! (brew outdated | grep -qx "$PACKAGE"); then echo "Already installed and the latest version: $PACKAGE" @@ -47,21 +52,21 @@ function brew_install_and_cache_within_time_limit { fi - _brew_is_bottle_available "$PACKAGE" || BUILD_FROM_SOURCE=1 + _brew_is_bottle_available "$PACKAGE" KEG_ONLY || BUILD_FROM_SOURCE=1 [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true # Whitespace is illegal in package names so converting all whitespace into single spaces due to no quotes is okay. - DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD` || return 2 + DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD` || exit 2 for dep in $DEPS; do #TIME_LIMIT only has to be met if we'll be actually building the main project this iteration, i.e. after the "root" module installation #While we don't know that yet, we can make better use of Travis-given time with a laxer limit #We still can't overrun TIME_HARD_LIMIT as that would't leave time to save the cache - brew_install_and_cache_within_time_limit "$dep" $(((TIME_LIMIT+TIME_HARD_LIMIT)/2)) "$TIME_HARD_LIMIT" "$TIME_START" || return $? + brew_install_and_cache_within_time_limit "$dep" $(((TIME_LIMIT+TIME_HARD_LIMIT)/2)) "$TIME_HARD_LIMIT" "$TIME_START" || exit $? done - _brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || return $? - _brew_install_and_cache "$PACKAGE" "$([[ -z "$INCLUDE_BUILD" ]] && echo 1 || echo 0)" || return 2 - _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || return $? + _brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || exit $? + _brew_install_and_cache "$PACKAGE" "$([[ -z "$BUILD_FROM_SOURCE" ]] && echo 1 || echo 0)" "$KEG_ONLY" || exit 2 + _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || exit $? ) \ || if test $? -eq 1; then brew_go_bootstrap_mode; return 1; else return 2; fi #must run this in current process } @@ -288,8 +293,22 @@ function _brew_parse_package_info { function _brew_is_bottle_available { local PACKAGE;PACKAGE="${1:?}" - - local INFO="$(brew info "$PACKAGE" | head -n 1)" + local VAR_KEG_ONLY="$2" + + local INFO;INFO="$(brew info "$PACKAGE" | head -n 1)" + if [ -n "$VAR_KEG_ONLY" ]; then + if grep -qwF '[keg-only]' <<<"$INFO"; then + eval "${VAR_KEG_ONLY}=1" + else + eval "${VAR_KEG_ONLY}=0" + fi + fi + + if grep -qxEe '[[:space:]]*bottle :unneeded' $(brew formula "$PACKAGE"); then + echo "Bottle disabled: $PACKAGE" + return 0 + fi + if grep -qwF '(bottled)' <<<"$INFO"; then echo "Bottle available: $INFO" return 0 @@ -306,10 +325,16 @@ function _brew_install_and_cache { local PACKAGE;PACKAGE="${1:?}" local USE_BOTTLE;USE_BOTTLE="${2:?}" + local KEG_ONLY;KEG_ONLY="${3:?}" local VERB - if brew list --versions "$PACKAGE"; then - VERB=upgrade + if brew list --versions "$PACKAGE" >/dev/null; then + # Install alongside the old version to avoid to have to update "runtime dependents" + # https://discourse.brew.sh/t/can-i-install-a-new-version-without-having-to-upgrade-runtime-dependents/4443 + VERB="install --force" + if [ "$KEG_ONLY" -eq 0 ]; then + brew unlink "$PACKAGE" + fi else VERB=install fi @@ -328,8 +353,8 @@ function _brew_install_and_cache { # doesn't seem to be a documented way to get file names local BOTTLE; BOTTLE=$(grep -Ee '^./' <<<"$OUT") #proper procedure as per https://discourse.brew.sh/t/how-are-bottle-and-postinstall-related-is-it-safe-to-run-bottle-after-postinstall/3410/4 - brew uninstall "$PACKAGE" - brew install "$BOTTLE" + brew uninstall --ignore-dependencies "$PACKAGE" + brew $VERB "$BOTTLE" local JSON; JSON=$(sed -E 's/bottle(.[[:digit:]]+)?\.tar\.gz$/bottle.json/' <<<"$BOTTLE") From b5bd66108e606222e64c14048de95681d9a97978 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 21 Mar 2019 12:33:04 +0300 Subject: [PATCH 08/16] Install macpython late to avoid conflict with Homebrew python upgrade --- .travis.yml | 2 -- config.sh | 12 +++++++++--- travis_osx_brew_cache.sh | 2 ++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8772517e..632af248 100644 --- a/.travis.yml +++ b/.travis.yml @@ -594,7 +594,6 @@ before_install: | brew_cache_cleanup fi - before_install # Not interested in travis internal scripts' output set +x @@ -602,7 +601,6 @@ install: | # Build and package set -x build_wheel $REPO_DIR $PLAT - if [ -n "$USE_CCACHE" ]; then ccache --show-stats; fi set +x script: | diff --git a/config.sh b/config.sh index ead68645..22d16066 100644 --- a/config.sh +++ b/config.sh @@ -16,6 +16,7 @@ function bdist_wheel_cmd { local abs_wheelhouse=$1 python setup.py bdist_wheel $BDIST_PARAMS cp dist/*.whl $abs_wheelhouse + if [ -n "$USE_CCACHE" -a -z "$BREW_BOOTSTRAP_MODE" ]; then ccache --show-stats; fi } if [ -n "$IS_OSX" ]; then @@ -99,7 +100,7 @@ function pre_build { # this takes several seconds and we don't need it # see https://docs.brew.sh/Manpage , "info formula" section export HOMEBREW_NO_GITHUB_API=1 - + echo 'Installing QT4' brew tap | grep -qxF cartr/qt4 || brew tap cartr/qt4 brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin cartr/qt4 @@ -108,8 +109,13 @@ function pre_build { echo 'Installing FFmpeg' generate_ffmpeg_formula - brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } - + local IS_BOTTLE_AVAILABLE; _brew_is_bottle_available ffmpeg_opencv && IS_BOTTLE_AVAILABLE=1 || IS_BOTTLE_AVAILABLE=0 + _brew_install_and_cache ffmpeg_opencv "$IS_BOTTLE_AVAILABLE" + #brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } + + # Have to install macpython late to avoid conflict with Homebrew Python update + before_install + else echo "Running for linux" fi diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index c62a6235..261e1d00 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -215,6 +215,8 @@ function brew_go_bootstrap_mode { # Terminate the build but ensure saving the cache echo "Going into cache bootstrap mode" + + BREW_BOOTSTRAP_MODE=1 #Can't just `exit` because that would terminate the build without saving the cache #Have to replace further actions with no-ops From 7d87741ebb41e560d0246308c6f6a5416e286da5 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Thu, 21 Mar 2019 22:11:08 +0300 Subject: [PATCH 09/16] OSX custom ffmpeg formula: disable GPLed frei0r, librubberband --- config.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.sh b/config.sh index 22d16066..59c40ee3 100644 --- a/config.sh +++ b/config.sh @@ -70,8 +70,8 @@ if [ -n "$IS_OSX" ]; then if (!$found_blank && /^$/) {$_.="conflicts_with \"ffmpeg\"\n\n"; $found_blank=1; next;} if (!$bottle_block && /^\s*bottle do$/) { $bottle_block=1; next; } if ($bottle_block) { if (/^\s*end\s*$/) { $bottle_block=0} elsif (/^\s*sha256\s/) {$_=""} next; } -if (/^\s*depends_on "(x264|x265|xvid)"$/) {$_=""; next;} - if (/^\s*--enable-(gpl|libx264|libx265|libxvid)$/) {$_=""; next;} +if (/^\s*depends_on "(x264|x265|xvid|frei0r|rubberband)"$/) {$_=""; next;} + if (/^\s*--enable-(gpl|libx264|libx265|libxvid|frei0r|librubberband)$/) {$_=""; next;} ' <"$FF_FORMULA" >"$LFF_FORMULA" diff -u "$FF_FORMULA" "$LFF_FORMULA" || test $? -le 1 From b5b4e8473918349e1f3dfeb9fc3922056684d894 Mon Sep 17 00:00:00 2001 From: Admin Date: Fri, 22 Mar 2019 02:34:10 +0300 Subject: [PATCH 10/16] Adapt setup.py hook for skbuild 0.9.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4cf0e733..65322dbc 100644 --- a/setup.py +++ b/setup.py @@ -268,7 +268,7 @@ def _classify_installed_files_override(self, install_paths, # 'relpath'/'reldir' = relative to CMAKE_INSTALL_DIR/cmake_install_dir # 'path'/'dir' = relative to sourcetree root - cmake_install_dir = os.path.join(cls._setuptools_wrap.CMAKE_INSTALL_DIR, + cmake_install_dir = os.path.join(cls._setuptools_wrap.CMAKE_INSTALL_DIR(), cmake_install_reldir) install_relpaths = [os.path.relpath(p, cmake_install_dir) for p in install_paths] fslash_install_relpaths = [p.replace(os.path.sep, '/') for p in install_relpaths] From e93bb9376fdcda572151653a8110e60667b239e5 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Fri, 22 Mar 2019 23:04:51 +0300 Subject: [PATCH 11/16] Reduce the amount of trace output * Reduce the number of command lines executed in most used functions * List existing versions only when installing --- travis_osx_brew_cache.sh | 51 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 261e1d00..69dbf729 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -1,12 +1,5 @@ # Library to cache downloaded and locally-built Homebrew bottles in Travis OSX build. -_BREW_ERREXIT=' -set -e -o pipefail -trap '\''{ sleep 3; #if we terminale too abruptly, Travis will lose some log output - exit 2; #The trap isn''t called in the parent function, so can''t use `return` here. - #`exit` will terminate the entire build but it seems we have no choice. -}'\'' ERR -set -E' #Should be in Travis' cache BREW_LOCAL_BOTTLE_METADATA="$HOME/local_bottle_metadata" @@ -37,16 +30,17 @@ function brew_install_and_cache_within_time_limit { # use bottle if available, build and cache bottle if not. # Terminate and exit with status 1 if this takes too long. # Exit with status 2 on any other error. - ( eval "$_BREW_ERREXIT" - - local PACKAGE; PACKAGE="${1:?}" || exit 2 - local TIME_LIMIT;TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || exit 2 - local TIME_HARD_LIMIT;TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || exit 2 - local TIME_START;TIME_START=${4:-$BREW_TIME_START} || exit 2 + ( set -eE -o pipefail; trap '{ sleep 3; exit 2; }' ERR + + local PACKAGE TIME_LIMIT TIME_HARD_LIMIT TIME_START + PACKAGE="${1:?}" || exit 2 + TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || exit 2 + TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || exit 2 + TIME_START=${4:-$BREW_TIME_START} || exit 2 local BUILD_FROM_SOURCE INCLUDE_BUILD KEG_ONLY - if brew list --versions "$PACKAGE" && ! (brew outdated | grep -qx "$PACKAGE"); then + if brew list --versions "$PACKAGE" >/dev/null && ! (brew outdated | grep -qxF "$PACKAGE"); then echo "Already installed and the latest version: $PACKAGE" return 0 fi @@ -266,8 +260,9 @@ function _brew_parse_package_info { # Get and parse `brew info --json` about a package # and save data into specified variables - local PACKAGE; PACKAGE="${1:?}"; shift - local OS_CODENAME;OS_CODENAME="${1:?}"; shift + local PACKAGE OS_CODENAME + PACKAGE="${1:?}"; shift + OS_CODENAME="${1:?}"; shift local JSON_DATA; JSON_DATA=$(python2.7 -c 'if True: import sys, json, subprocess; j=json.loads(subprocess.check_output(("brew","info","--json=v1",sys.argv[1]))) @@ -325,12 +320,13 @@ function _brew_install_and_cache { # assumes that deps were already installed # and not already the latest version - local PACKAGE;PACKAGE="${1:?}" - local USE_BOTTLE;USE_BOTTLE="${2:?}" - local KEG_ONLY;KEG_ONLY="${3:?}" + local PACKAGE USE_BOTTLE KEG_ONLY + PACKAGE="${1:?}" + USE_BOTTLE="${2:?}" + KEG_ONLY="${3:?}" local VERB - if brew list --versions "$PACKAGE" >/dev/null; then + if brew list --versions "$PACKAGE"; then # Install alongside the old version to avoid to have to update "runtime dependents" # https://discourse.brew.sh/t/can-i-install-a-new-version-without-having-to-upgrade-runtime-dependents/4443 VERB="install --force" @@ -382,10 +378,11 @@ function _brew_check_elapsed_build_time { # If time limit has been reached, # arrange for further build to be skipped and return 1 - local TIME_START;TIME_START="${1:?}" - local TIME_LIMIT;TIME_LIMIT="${2:?}" + local TIME_START TIME_LIMIT ELAPSED_TIME + TIME_START="${1:?}" + TIME_LIMIT="${2:?}" - local ELAPSED_TIME;ELAPSED_TIME=$(($(date +%s) - $TIME_START)) + ELAPSED_TIME=$(($(date +%s) - $TIME_START)) echo "Elapsed time: "$(($ELAPSED_TIME/60))"m (${ELAPSED_TIME}s)" if [[ "$ELAPSED_TIME" -gt $TIME_LIMIT ]]; then @@ -400,10 +397,12 @@ function _brew_check_slow_building_ahead { #If the package's projected build completion is higher than hard limit, # skip it and arrange for further build to be skipped and return 1 - local PACKAGE="${1:?}" - local TIME_START="${2:?}" - local TIME_HARD_LIMIT="${3:?}" + local PACKAGE TIME_START TIME_HARD_LIMIT + PACKAGE="${1:?}" + TIME_START="${2:?}" + TIME_HARD_LIMIT="${3:?}" + local PROJECTED_BUILD_TIME PROJECTED_BUILD_TIME=$(echo "$BREW_SLOW_BUILIDING_PACKAGES" | awk '$1=="'"$PACKAGE"'"{print $2}') [ -z "$PROJECTED_BUILD_TIME" ] && return 0 || true From 9d020bc045f893eb500de86722f912d74caef2a5 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sat, 23 Mar 2019 02:08:15 +0300 Subject: [PATCH 12/16] Use 2 stages for MacOS builds * Add a dedicated stage to each OSX build to download/build brew packages * Remove bottle building logic and the corresponding trace output from the project build stage --- .travis.yml | 128 +++++++++++++++++++++++++++++++++------ config.sh | 35 +++++++---- travis_osx_brew_cache.sh | 6 +- 3 files changed, 135 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index 632af248..07febd6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,144 +34,232 @@ cache: # `cache: ccache: true` has no effect if `language:` is not `c` or `cpp` - $HOME/.ccache +# Add more cache stages (s2 etc) and corresponding OSX jobs like s1 +# if brew builds start to take longer than one Travis time limit +stages: + - s1 + - final + matrix: fast_finish: true include: # default builds for MacOS - - os: osx + - &osx-10 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - - os: osx + stage: final + - <<: *osx-10 + stage: s1 + - &osx-30 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - - os: osx + stage: final + - <<: *osx-30 + stage: s1 + - &osx-40 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - - os: osx + stage: final + - <<: *osx-40 + stage: s1 + - &osx-50 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - - os: osx + stage: final + - <<: *osx-50 + stage: s1 + - &osx-60 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.14.5 + stage: final + - <<: *osx-60 + stage: s1 # headless builds for MacOS - - os: osx + - &osx-70 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - - os: osx + stage: final + - <<: *osx-70 + stage: s1 + - &osx-80 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - - os: osx + stage: final + - <<: *osx-80 + stage: s1 + - &osx-90 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - - os: osx + stage: final + - <<: *osx-90 + stage: s1 + - &osx-100 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - - os: osx + stage: final + - <<: *osx-100 + stage: s1 + - &osx-110 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.14.5 + stage: final + - <<: *osx-110 + stage: s1 # Contrib builds for MacOS - - os: osx + - &osx-120 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - - os: osx + stage: final + - <<: *osx-120 + stage: s1 + - &osx-130 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - - os: osx + stage: final + - <<: *osx-130 + stage: s1 + - &osx-140 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - - os: osx + stage: final + - <<: *osx-140 + stage: s1 + - &osx-150 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - - os: osx + stage: final + - <<: *osx-150 + stage: s1 + - &osx-160 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.14.5 + stage: final + - <<: *osx-160 + stage: s1 # headless contrib builds for MacOS - - os: osx + - &osx-170 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - - os: osx + stage: final + - <<: *osx-170 + stage: s1 + - &osx-180 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - - os: osx + stage: final + - <<: *osx-180 + stage: s1 + - &osx-190 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - - os: osx + stage: final + - <<: *osx-190 + stage: s1 + - &osx-200 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - - os: osx + stage: final + - <<: *osx-200 + stage: s1 + - &osx-210 + os: osx osx_image: xcode8.3 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.14.5 + stage: final + - <<: *osx-210 + stage: s1 # default builds for Linux - os: linux + # the following jobs will use the same stage name by default + stage: s1 env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=0 diff --git a/config.sh b/config.sh index 59c40ee3..16a5ad68 100644 --- a/config.sh +++ b/config.sh @@ -37,11 +37,6 @@ if [ -n "$IS_OSX" ]; then "ffmpeg_opencv 10" \ ) - #Contrib adds significantly to project's build time - if [ "$ENABLE_CONTRIB" -eq 1 ]; then - BREW_TIME_LIMIT=$((BREW_TIME_LIMIT - 10*60)) - fi - function generate_ffmpeg_formula { local FF="ffmpeg" local LFF="ffmpeg_opencv" @@ -92,9 +87,14 @@ function pre_build { if [ -n "$IS_OSX" ]; then echo "Running for OSX" + + local CACHE_STAGE; (echo "$TRAVIS_BUILD_STAGE_NAME" | grep -qiF "final") || CACHE_STAGE=1 - brew update - brew_add_local_bottles + #after the cache stage, all bottles and Homebrew metadata should be already cached locally + if [ -n "$CACHE_STAGE" ]; then + brew update + brew_add_local_bottles + fi # Don't query analytical info online on `brew info`, # this takes several seconds and we don't need it @@ -104,14 +104,25 @@ function pre_build { echo 'Installing QT4' brew tap | grep -qxF cartr/qt4 || brew tap cartr/qt4 brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin cartr/qt4 - brew_install_and_cache_within_time_limit qt@4 || { [ $? -gt 1 ] && return 2 || return 0; } + if [ -n "$CACHE_STAGE" ]; then + brew_install_and_cache_within_time_limit qt@4 || { [ $? -gt 1 ] && return 2 || return 0; } + else + brew install qt@4 + fi echo 'Installing FFmpeg' - generate_ffmpeg_formula - local IS_BOTTLE_AVAILABLE; _brew_is_bottle_available ffmpeg_opencv && IS_BOTTLE_AVAILABLE=1 || IS_BOTTLE_AVAILABLE=0 - _brew_install_and_cache ffmpeg_opencv "$IS_BOTTLE_AVAILABLE" - #brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } + if [ -n "$CACHE_STAGE" ]; then + generate_ffmpeg_formula + brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } + else + brew install ffmpeg_opencv + fi + + if [ -n "$CACHE_STAGE" ]; then + brew_go_bootstrap_mode 0 + return 0 + fi # Have to install macpython late to avoid conflict with Homebrew Python update before_install diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 69dbf729..3eba10f6 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -10,7 +10,8 @@ BREW_TIME_START=$(($TRAVIS_TIMER_START_TIME/10**9)) # If after a package is built, elapsed time is more than this many seconds, fail the build but save Travis cache # The cutoff moment should leave enough time till Travis' job time limit to process the main project. -BREW_TIME_LIMIT=$((30*60)) +# Since we have moved deps into a separate stage, we don't need to leave time for the project any more +BREW_TIME_LIMIT=$((38*60)) # If a slow-building package is about to be built and the projected build end moment is beyond this many seconds, # skip that build, fail the Travis job and save Travis cache. # This cutoff should leave enough time for before_cache and cache save. @@ -207,6 +208,7 @@ function brew_cache_cleanup { function brew_go_bootstrap_mode { # Can be overridden # Terminate the build but ensure saving the cache + local EXIT_CODE=${1:-1} echo "Going into cache bootstrap mode" @@ -223,7 +225,7 @@ function brew_go_bootstrap_mode { # Travis runs user scripts via `eval` i.e. in the same shell process. # So have to unset errexit in order to get to cache save stage - set +e; return 1 + set +e; return '"$EXIT_CODE"' }' } From 509e5cbc75178f92d5d084b1c0e38b294471d3cd Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sat, 23 Mar 2019 22:16:33 +0300 Subject: [PATCH 13/16] Further increase time limits for the cache jobs --- travis_osx_brew_cache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 3eba10f6..87de4c78 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -11,11 +11,11 @@ BREW_TIME_START=$(($TRAVIS_TIMER_START_TIME/10**9)) # If after a package is built, elapsed time is more than this many seconds, fail the build but save Travis cache # The cutoff moment should leave enough time till Travis' job time limit to process the main project. # Since we have moved deps into a separate stage, we don't need to leave time for the project any more -BREW_TIME_LIMIT=$((38*60)) +BREW_TIME_LIMIT=$((42*60)) # If a slow-building package is about to be built and the projected build end moment is beyond this many seconds, # skip that build, fail the Travis job and save Travis cache. # This cutoff should leave enough time for before_cache and cache save. -BREW_TIME_HARD_LIMIT=$((40*60)) +BREW_TIME_HARD_LIMIT=$((43*60)) # Auto cleanup can delete locally-built bottles From c1edb270867f0fb87689e3bbc1b11331c250f677 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sat, 23 Mar 2019 22:17:44 +0300 Subject: [PATCH 14/16] x265 is not among the custom formula dependencies --- config.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/config.sh b/config.sh index 16a5ad68..c6aae3db 100644 --- a/config.sh +++ b/config.sh @@ -32,7 +32,6 @@ if [ -n "$IS_OSX" ]; then source travis_osx_brew_cache.sh BREW_SLOW_BUILIDING_PACKAGES=$(printf '%s\n' \ - "x265 20" \ "cmake 15" \ "ffmpeg_opencv 10" \ ) From f2e6647a2ade5a297b23ccab24a0a02344c0cc46 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Sat, 23 Mar 2019 22:20:33 +0300 Subject: [PATCH 15/16] HOMEBREW_NO_GITHUB_API is more relevant to the lib rather than main logic --- config.sh | 5 ----- travis_osx_brew_cache.sh | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config.sh b/config.sh index c6aae3db..c31def77 100644 --- a/config.sh +++ b/config.sh @@ -95,11 +95,6 @@ function pre_build { brew_add_local_bottles fi - # Don't query analytical info online on `brew info`, - # this takes several seconds and we don't need it - # see https://docs.brew.sh/Manpage , "info formula" section - export HOMEBREW_NO_GITHUB_API=1 - echo 'Installing QT4' brew tap | grep -qxF cartr/qt4 || brew tap cartr/qt4 brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin cartr/qt4 diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 87de4c78..43625c41 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -22,6 +22,12 @@ BREW_TIME_HARD_LIMIT=$((43*60)) # when the caching logic isn't prepared for that export HOMEBREW_NO_INSTALL_CLEANUP=1 +# Don't query analytical info online on `brew info`, +# this takes several seconds and we don't need it +# see https://docs.brew.sh/Manpage , "info formula" section +export HOMEBREW_NO_GITHUB_API=1 + + #Public functions From 23100a71d5e8797a7e3abf13d4a0c18e970599c1 Mon Sep 17 00:00:00 2001 From: Ivan Pozdeev Date: Mon, 25 Mar 2019 11:41:41 +0300 Subject: [PATCH 16/16] Fix contrib failures due to https://github.com/opencv/opencv_contrib/issues/920 --- opencv_contrib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencv_contrib b/opencv_contrib index 25221244..b7e78523 160000 --- a/opencv_contrib +++ b/opencv_contrib @@ -1 +1 @@ -Subproject commit 25221244732dcf44c1450d0f93edc2529a61c0e1 +Subproject commit b7e785233c1722c31530b59434b7d8bdcb0422f3