diff --git a/.travis.yml b/.travis.yml index 34d989f8..81d2331e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ env: - TEST_DEPENDS="numpy==1.11.1" # params to bdist_wheel. used to set osx build target. - BDIST_PARAMS="" + - CONFIG_PATH="travis_config.sh" - USE_CCACHE=1 - PLAT=x86_64 - UNICODE_WIDTH=32 @@ -663,7 +664,7 @@ before_install: | if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi source multibuild/travis_steps.sh # This sets -x - source multibuild_customize.sh + source travis_multibuild_customize.sh echo $ENABLE_CONTRIB > contrib.enabled echo $ENABLE_HEADLESS > headless.enabled diff --git a/config.sh b/travis_config.sh similarity index 99% rename from config.sh rename to travis_config.sh index c31def77..af888179 100644 --- a/config.sh +++ b/travis_config.sh @@ -16,7 +16,7 @@ function bdist_wheel_cmd { local abs_wheelhouse=$1 python setup.py bdist_wheel $BDIST_PARAMS cp dist/*.whl $abs_wheelhouse - if [ -n "$USE_CCACHE" -a -z "$BREW_BOOTSTRAP_MODE" ]; then ccache --show-stats; fi + if [ -n "$USE_CCACHE" -a -z "$BREW_BOOTSTRAP_MODE" ]; then ccache -s; fi } if [ -n "$IS_OSX" ]; then @@ -92,6 +92,7 @@ function pre_build { #after the cache stage, all bottles and Homebrew metadata should be already cached locally if [ -n "$CACHE_STAGE" ]; then brew update + generate_ffmpeg_formula brew_add_local_bottles fi @@ -107,7 +108,6 @@ function pre_build { echo 'Installing FFmpeg' if [ -n "$CACHE_STAGE" ]; then - generate_ffmpeg_formula brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } else brew install ffmpeg_opencv diff --git a/multibuild_customize.sh b/travis_multibuild_customize.sh similarity index 100% rename from multibuild_customize.sh rename to travis_multibuild_customize.sh diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh index 43625c41..e3a69cc3 100644 --- a/travis_osx_brew_cache.sh +++ b/travis_osx_brew_cache.sh @@ -27,7 +27,8 @@ export HOMEBREW_NO_INSTALL_CLEANUP=1 # see https://docs.brew.sh/Manpage , "info formula" section export HOMEBREW_NO_GITHUB_API=1 - +#Packages already installed in the current session to avoid checking them again +_BREW_ALREADY_INSTALLED='$' #$ = illegal package name; a blank line would cause macos grep to swallow everything #Public functions @@ -37,39 +38,8 @@ function brew_install_and_cache_within_time_limit { # use bottle if available, build and cache bottle if not. # Terminate and exit with status 1 if this takes too long. # Exit with status 2 on any other error. - ( set -eE -o pipefail; trap '{ sleep 3; exit 2; }' ERR - - local PACKAGE TIME_LIMIT TIME_HARD_LIMIT TIME_START - PACKAGE="${1:?}" || exit 2 - TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || exit 2 - TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || exit 2 - TIME_START=${4:-$BREW_TIME_START} || exit 2 - - local BUILD_FROM_SOURCE INCLUDE_BUILD KEG_ONLY - - if brew list --versions "$PACKAGE" >/dev/null && ! (brew outdated | grep -qxF "$PACKAGE"); then - echo "Already installed and the latest version: $PACKAGE" - return 0 - fi - - - _brew_is_bottle_available "$PACKAGE" KEG_ONLY || BUILD_FROM_SOURCE=1 - [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true - - # Whitespace is illegal in package names so converting all whitespace into single spaces due to no quotes is okay. - DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD` || exit 2 - for dep in $DEPS; do - #TIME_LIMIT only has to be met if we'll be actually building the main project this iteration, i.e. after the "root" module installation - #While we don't know that yet, we can make better use of Travis-given time with a laxer limit - #We still can't overrun TIME_HARD_LIMIT as that would't leave time to save the cache - brew_install_and_cache_within_time_limit "$dep" $(((TIME_LIMIT+TIME_HARD_LIMIT)/2)) "$TIME_HARD_LIMIT" "$TIME_START" || exit $? - done - - _brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || exit $? - _brew_install_and_cache "$PACKAGE" "$([[ -z "$BUILD_FROM_SOURCE" ]] && echo 1 || echo 0)" "$KEG_ONLY" || exit 2 - _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || exit $? - ) \ - || if test $? -eq 1; then brew_go_bootstrap_mode; return 1; else return 2; fi #must run this in current process + _brew_install_and_cache_within_time_limit $@ \ + || if test $? -eq 1; then brew_go_bootstrap_mode; return 1; else return 2; fi } function brew_add_local_bottles { @@ -106,8 +76,8 @@ function brew_add_local_bottles { local FORMULA_HAS_BOTTLE; [ -n "$FORMULA_BOTTLE_HASH" ] && FORMULA_HAS_BOTTLE=1 || true - local BOTTLE_LINK BOTTLE; BOTTLE_LINK="${JSON}.bottle.lnk"; - local BOTTLE_EXISTS BOTTLE_MISMATCH VERSION_MISMATCH + local BOTTLE_LINK BOTTLE=""; BOTTLE_LINK="${JSON}.bottle.lnk"; + local BOTTLE_EXISTS= BOTTLE_MISMATCH= VERSION_MISMATCH= # Check that the bottle file exists and is still appropriate for the formula @@ -223,12 +193,19 @@ function brew_go_bootstrap_mode { #Can't just `exit` because that would terminate the build without saving the cache #Have to replace further actions with no-ops + local MESSAGE=""; if [ "$EXIT_CODE" -ne 0 ]; then + MESSAGE='Building dependencies took too long. Restart the build in Travis UI to continue from cache.'; + fi + eval ' function '"$cmd"' { return 0; } function repair_wheelhouse { return 0; } - function install_run { - echo -e "\nBuilding dependencies took too long. Restart the build in Travis UI to continue from cache.\n" - + function install_run {'\ + "$(if [ -n "$MESSAGE" ]; then + echo \ + ' echo -e "\n'"$MESSAGE"'\n"' + fi)"\ + ' # Travis runs user scripts via `eval` i.e. in the same shell process. # So have to unset errexit in order to get to cache save stage set +e; return '"$EXIT_CODE"' @@ -239,6 +216,46 @@ function brew_go_bootstrap_mode { #Internal functions +function _brew_install_and_cache_within_time_limit { + # This fn is run with || so errexit can't be enabled + + local PACKAGE TIME_LIMIT TIME_HARD_LIMIT TIME_START MARKED_INSTALLED + PACKAGE="${1:?}" || return 2 + TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || return 2 + TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || return 2 + TIME_START=${4:-$BREW_TIME_START} || return 2 + + if grep -qxFf <(cat <<<"$_BREW_ALREADY_INSTALLED") <<<"$PACKAGE"; then + MARKED_INSTALLED=1 + fi + + if [ -n "$MARKED_INSTALLED" ] || (brew list --versions "$PACKAGE" >/dev/null && ! (brew outdated | grep -qxF "$PACKAGE")); then + echo "Already installed and the latest version: $PACKAGE" + if [ -z "$MARKED_INSTALLED" ]; then _brew_mark_installed "$PACKAGE"; fi + return 0 + fi + + local BUILD_FROM_SOURCE INCLUDE_BUILD KEG_ONLY + + _brew_is_bottle_available "$PACKAGE" KEG_ONLY || BUILD_FROM_SOURCE=1 + [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true + + # Whitespace is illegal in package names so converting all whitespace into single spaces due to no quotes is okay. + DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD` || return 2 + DEPS=`grep -vxF <(cat <<<"$_BREW_ALREADY_INSTALLED") <<<"$DEPS"` || test $? -eq 1 || return 2 + for dep in $DEPS; do + #TIME_LIMIT only has to be met if we'll be actually building the main project this iteration, i.e. after the "root" module installation + #While we don't know that yet, we can make better use of Travis-given time with a laxer limit + #We still can't overrun TIME_HARD_LIMIT as that would't leave time to save the cache + _brew_install_and_cache_within_time_limit "$dep" $(((TIME_LIMIT+TIME_HARD_LIMIT)/2)) "$TIME_HARD_LIMIT" "$TIME_START" || return $? + done + + _brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || return $? + _brew_install_and_cache "$PACKAGE" "$([[ -z "$BUILD_FROM_SOURCE" ]] && echo 1 || echo 0)" "$KEG_ONLY" || return 2 + _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || return $? +} + + function _brew_parse_bottle_json { # Parse JSON file resulting from `brew bottle --json` # and save data into specified variables @@ -278,7 +295,7 @@ function _brew_parse_package_info { revision=data["revision"] # in bottle''s json, revision is included into version; here, they are separate print data["versions"]["stable"]+("_"+str(revision) if revision else "") - bottle_data=data["bottle"]["stable"] + bottle_data=data["bottle"].get("stable",{"rebuild":"","files":{}}) print bottle_data["rebuild"] print bottle_data["files"].get(sys.argv[2],{"sha256":"!?"})["sha256"] #prevent losing trailing blank line to command substitution ' \ @@ -300,7 +317,9 @@ function _brew_is_bottle_available { local PACKAGE;PACKAGE="${1:?}" local VAR_KEG_ONLY="$2" - local INFO;INFO="$(brew info "$PACKAGE" | head -n 1)" + # `brew info` prints "Error: Broken pipe" if piped directly to `head` and the info is long + # 141 = 128 + SIGPIPE + local INFO;INFO="$((brew info "$PACKAGE" | cat || test $? -eq 141) | head -n 1)" if [ -n "$VAR_KEG_ONLY" ]; then if grep -qwF '[keg-only]' <<<"$INFO"; then eval "${VAR_KEG_ONLY}=1" @@ -377,10 +396,13 @@ function _brew_install_and_cache { echo "$CACHED_BOTTLE" >"$BOTTLE_LINK" fi + + _brew_mark_installed "$PACKAGE" } - - +function _brew_mark_installed { + _BREW_ALREADY_INSTALLED="$_BREW_ALREADY_INSTALLED"$'\n'"${1:?}" +} function _brew_check_elapsed_build_time { # If time limit has been reached,