diff --git a/.travis.yml b/.travis.yml index 93aea0c3..8c54615e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,17 @@ dist: trusty git: submodules: false +# https://docs.travis-ci.com/user/caching +cache: + directories: + # https://stackoverflow.com/questions/39930171/cache-brew-builds-with-travis-ci + - $HOME/Library/Caches/Homebrew + - /usr/local/Homebrew/ + # used in OSX custom build script dealing with local bottle caching + - $HOME/local_bottle_metadata + # `cache: ccache: true` has no effect if `language:` is not `c` or `cpp` + - $HOME/.ccache + matrix: fast_finish: true include: @@ -513,6 +524,21 @@ before_install: | source multibuild_customize.sh echo $ENABLE_CONTRIB > contrib.enabled echo $ENABLE_HEADLESS > headless.enabled + + if [ -n "$IS_OSX" ]; then + TAPS="$(brew --repository)/Library/Taps" + if [ -e "$TAPS/caskroom/homebrew-cask" -a -e "$TAPS/homebrew/homebrew-cask" ]; then + rm -rf "$TAPS/caskroom/homebrew-cask" + fi + find "$TAPS" -type d -name .git -exec \ + bash -xec ' + cd $(dirname '\''{}'\'') + git clean -fxd + git status' \; + + brew_cache_cleanup + fi + before_install # Not interested in travis internal scripts' output set +x @@ -526,9 +552,30 @@ install: | script: | # Install and run tests set -x - install_run $PLAT + install_run $PLAT && rc=$? || rc=$? set +x + #otherwise, Travis logic terminates prematurely + #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817 + trap ERR + + test "$rc" -eq 0 + +before_cache: | + # Cleanup dirs to be cached + set -x + if [ -n "$IS_OSX" ]; then + + # When Taps is cached, this dir causes "Error: file exists" on `brew update` + if [ -e "$(brew --repository)/Library/Taps/homebrew/homebrew-cask/homebrew-cask" ]; then + rm -rf "$(brew --repository)/Library/Taps/homebrew/homebrew-cask/homebrew-cask" + fi + + brew_cache_cleanup + + fi + set +x + after_success: | # Upload wheels to pypi if requested if [ -n "$TRAVIS_TAG" ]; then diff --git a/config.sh b/config.sh index 86d238e0..53c5ff2f 100644 --- a/config.sh +++ b/config.sh @@ -20,32 +20,101 @@ function bdist_wheel_cmd { if [ -n "$IS_OSX" ]; then echo " > OSX environment " + export MAKEFLAGS="-j$(sysctl -n hw.ncpu)" else echo " > Linux environment " + export MAKEFLAGS="-j$(grep -E '^processor[[:space:]]*:' /proc/cpuinfo | wc -l)" +fi + +if [ -n "$IS_OSX" ]; then + + source travis_osx_brew_cache.sh + + BREW_SLOW_BUILIDING_PACKAGES=$(printf '%s\n' \ + "x265 20" \ + "cmake 15" \ + "ffmpeg_opencv 10" \ + ) + + #Contrib adds significantly to project's build time + if [ "$ENABLE_CONTRIB" -eq 1 ]; then + BREW_TIME_LIMIT=$((BREW_TIME_LIMIT - 10*60)) + fi + + function generate_ffmpeg_formula { + local FF="ffmpeg" + local LFF="ffmpeg_opencv" + local FF_FORMULA; FF_FORMULA=$(brew formula "$FF") + local LFF_FORMULA; LFF_FORMULA="$(dirname "$FF_FORMULA")/${LFF}.rb" + + local REGENERATE + if [ -f "$LFF_FORMULA" ]; then + local UPSTREAM_VERSION VERSION + _brew_parse_package_info "$FF" " " UPSTREAM_VERSION _ _ + _brew_parse_package_info "$LFF" " " VERSION _ _ || REGENERATE=1 + #`rebuild` clause is ignored on `brew bottle` and deleted + # from newly-generated formula on `brew bottle --merge` for some reason + # so can't compare rebuild numbers + if [ "$UPSTREAM_VERSION" != "$VERSION" ]; then + REGENERATE=1 + fi + else + REGENERATE=1 + fi + if [ -n "$REGENERATE" ]; then + echo "Regenerating custom ffmpeg formula" + # Bottle block syntax: https://docs.brew.sh/Bottles#bottle-dsl-domain-specific-language + perl -wpe 'BEGIN {our ($found_blank, $bottle_block);} + if (/(^class )(Ffmpeg)(\s.*)/) {$_=$1.$2."Opencv".$3."\n"; next;} + if (!$found_blank && /^$/) {$_.="conflicts_with \"ffmpeg\"\n\n"; $found_blank=1; next;} + if (!$bottle_block && /^\s*bottle do$/) { $bottle_block=1; next; } + if ($bottle_block) { if (/^\s*end\s*$/) { $bottle_block=0} elsif (/^\s*sha256\s/) {$_=""} next; } +if (/^\s*depends_on "(x264|x265|xvid)"$/) {$_=""; next;} + if (/^\s*--enable-(gpl|libx264|libx265|libxvid)$/) {$_=""; next;} + ' <"$FF_FORMULA" >"$LFF_FORMULA" + diff -u "$FF_FORMULA" "$LFF_FORMULA" || test $? -le 1 + + ( cd "$(dirname "$LFF_FORMULA")" + # This is the official way to add a formula + # https://docs.brew.sh/Formula-Cookbook#commit + git add "$(basename "$LFF_FORMULA")" + git commit -m "add/update custom ffmpeg ${VERSION}" + ) + fi + } + fi function pre_build { echo "Starting pre-build" - set -e + set -e -o pipefail if [ -n "$IS_OSX" ]; then echo "Running for OSX" + + brew update --merge + brew_add_local_bottles + + # Don't query analytical info online on `brew info`, + # this takes several seconds and we don't need it + # see https://docs.brew.sh/Manpage , "info formula" section + export HOMEBREW_NO_GITHUB_API=1 - brew update + # https://docs.travis-ci.com/user/caching/#ccache-cache + # No need to allow rc 1 -- if this triggers a timeout, + # something is clearly wrong + brew_install_and_cache_within_time_limit ccache + export PATH="/usr/local/opt/ccache/libexec:$PATH" echo 'Installing QT4' - brew tap | grep -qxF cartr/qt4 || brew tap -v cartr/qt4 - brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin -v cartr/qt4 - brew list --versions qt@4 || brew install -v qt@4 - echo '-----------------' - echo '-----------------' + brew tap | grep -qxF cartr/qt4 || brew tap cartr/qt4 + brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin cartr/qt4 + brew_install_and_cache_within_time_limit qt@4 || { [ $? -gt 1 ] && return 2 || return 0; } + echo 'Installing FFmpeg' - # brew install does produce output regularly on a regular MacOS, - # but Travis doesn't see it for some reason - brew list --versions ffmpeg || \ - travis_wait brew install -v ffmpeg --without-x264 --without-xvid --without-gpl - brew info ffmpeg - echo '-----------------' + + generate_ffmpeg_formula + brew_install_and_cache_within_time_limit ffmpeg_opencv || { [ $? -gt 1 ] && return 2 || return 0; } else echo "Running for linux" diff --git a/travis_osx_brew_cache.sh b/travis_osx_brew_cache.sh new file mode 100644 index 00000000..47d37326 --- /dev/null +++ b/travis_osx_brew_cache.sh @@ -0,0 +1,389 @@ +# Library to cache downloaded and locally-built Homebrew bottles in Travis OSX build. + +_BREW_ERREXIT=' +set -e -o pipefail +trap '\''{ sleep 3; #if we terminale too abruptly, Travis will lose some log output + exit 2; #The trap isn''t called in the parent function, so can''t use `return` here. + #`exit` will terminate the entire build but it seems we have no choice. +}'\'' ERR +set -E' + +#Should be in Travis' cache +BREW_LOCAL_BOTTLE_METADATA="$HOME/local_bottle_metadata" + +# Starting reference point for elapsed build time; seconds since the epoch. +#TRAVIS_TIMER_START_TIME is set at the start of a log fold, in nanoseconds since the epoch +BREW_TIME_START=$(($TRAVIS_TIMER_START_TIME/10**9)) + +# If after a package is built, elapsed time is more than this many seconds, fail the build but save Travis cache +# The cutoff moment should leave enough time till Travis' job time limit to process the main project. +BREW_TIME_LIMIT=$((30*60)) +# If a slow-building package is about to be built and the projected build end moment is beyond this many seconds, +# skip that build, fail the Travis job and save Travis cache. +# This cutoff should leave enough time for before_cache and cache save. +BREW_TIME_HARD_LIMIT=$((40*60)) + + + +#Public functions + +function brew_install_and_cache_within_time_limit { + # Install the package and its dependencies one by one; + # use bottle if available, build and cache bottle if not. + # Terminate and exit with status 1 if this takes too long. + # Exit with status 2 on any other error. + ( eval "$_BREW_ERREXIT" + + local PACKAGE; PACKAGE="${1:?}" || return 2 + local TIME_LIMIT;TIME_LIMIT=${2:-$BREW_TIME_LIMIT} || return 2 + local TIME_HARD_LIMIT;TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT} || return 2 + local TIME_START;TIME_START=${4:-$BREW_TIME_START} || return 2 + + local BUILD_FROM_SOURCE INCLUDE_BUILD + + _brew_is_bottle_available "$PACKAGE" || BUILD_FROM_SOURCE=1 + [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true + + # Whitespace is illegal in package names so converting all whitespace into single spaces due to no quotes is okay. + DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD` || return 2 + for dep in $DEPS; do + #TIME_LIMIT only has to be met if we'll be actually building the main project this iteration, i.e. after the "root" module installation + #While we don't know that yet, we can make better use of Travis-given time with a laxer limit + #We still can't overrun TIME_HARD_LIMIT as that would't leave time to save the cache + brew_install_and_cache_within_time_limit "$dep" $(((TIME_LIMIT+TIME_HARD_LIMIT)/2)) "$TIME_HARD_LIMIT" "$TIME_START" || return $? + done + + _brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || return $? + _brew_install_and_cache "$PACKAGE" "$([[ -z "$INCLUDE_BUILD" ]] && echo 1 || echo 0)" || return 2 + _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || return $? + ) \ + || if test $? -eq 1; then brew_go_bootstrap_mode; return 1; else return 2; fi #must run this in current process +} + +function brew_add_local_bottles { + # Should be called after `brew update` at startup. + # Adds metadata for cached locally-built bottles to local formulas + # so that `brew` commands can find them. + # If the package was updated, removes the corresponding files + # and the bottle's entry in the formula, if any. + + # Bottle entry in formula: + # bottle do + # <...> + # sha256 "" => : + # <...> + # end + + echo "Cached bottles:" + ls "$(brew --cache)/downloads" || true #may not exist initially since it's "$(brew --cache)" that is in Travis cache + echo "Saved .json's and links:" + ls "$BREW_LOCAL_BOTTLE_METADATA" + + for JSON in "$BREW_LOCAL_BOTTLE_METADATA"/*.json; do + [ -e "$JSON" ] || break # OSX 10.11 bash has no nullglob + local PACKAGE JSON_VERSION JSON_REBUILD OS_CODENAME BOTTLE_HASH + + _brew_parse_bottle_json "$JSON" PACKAGE JSON_VERSION JSON_REBUILD OS_CODENAME BOTTLE_HASH + + echo "Adding local bottle: $PACKAGE ${JSON_VERSION}_${JSON_REBUILD}" + + local FORMULA_VERSION FORMULA_REBUILD FORMULA_BOTTLE_HASH + + _brew_parse_package_info "$PACKAGE" "$OS_CODENAME" FORMULA_VERSION FORMULA_REBUILD FORMULA_BOTTLE_HASH + + local FORMULA_HAS_BOTTLE; [ -n "$FORMULA_BOTTLE_HASH" ] && FORMULA_HAS_BOTTLE=1 || true + + + local BOTTLE_LINK BOTTLE; BOTTLE_LINK="${JSON}.bottle.lnk"; + local BOTTLE_EXISTS BOTTLE_MISMATCH VERSION_MISMATCH + + + # Check that the bottle file exists and is still appropriate for the formula + if [[ "$FORMULA_VERSION" != "$JSON_VERSION" || "$JSON_REBUILD" != "$FORMULA_REBUILD" ]]; then + VERSION_MISMATCH=1; + echo "The cached bottle is obsolete: formula ${FORMULA_VERSION}_${FORMULA_REBUILD}" + fi + if [ -f "$BOTTLE_LINK" ]; then + BOTTLE=$(cat "$BOTTLE_LINK"); + BOTTLE=$(cd "$(dirname "$BOTTLE")"; pwd)/$(basename "$BOTTLE") + + if [ -e "$BOTTLE" ]; then + BOTTLE_EXISTS=1; + + # The hash in `brew --cache $PACKAGE` entry is generated from download URL, + # which itself is generated from base URL and version + # (see Homebrew/Library/Homebrew/download_strategy.rb:cached_location). + # So if version changes, hashes will always mismatch anyway + # and we don't need a separate message about this. + # XXX: OSX doesn't have `realpath` so can't compare the entire paths + if [ -n "$FORMULA_HAS_BOTTLE" -a -z "$VERSION_MISMATCH" -a \ + "$(basename "$(brew --cache "$PACKAGE")")" != "$(basename "$BOTTLE")" ]; then + BOTTLE_MISMATCH=1; + echo "Cached bottle file doesn't correspond to formula's cache entry!" \ + "This can happen if download URL has changed." >&2 + fi + else + echo "Cached bottle file is missing!" >&2 + fi + else + echo "Link file is missing or of invalid type!" >&2 + fi + + # Delete cached bottle and all metadata if invalid + if [[ -z "$BOTTLE_EXISTS" || -n "$VERSION_MISMATCH" || -n "$BOTTLE_MISMATCH" ]]; then + echo "Deleting the cached bottle and all metadata" + + if [ "$FORMULA_BOTTLE_HASH" == "$BOTTLE_HASH" ]; then + echo "A bottle block for the cached bottle was merged into the updated formula. Removing..." + local FORMULA; FORMULA=$(brew formula "$PACKAGE") + perl -wpe 'BEGIN { our $IN_BLOCK=0; } + if ( ($IN_BLOCK==0) && /^\s*bottle\s+do\s*$/ ) { $IN_BLOCK=1; next; } + if ( ($IN_BLOCK==1) && /^\s*end\s*$/ ) { $IN_BLOCK=-1; next; } + if ( ($IN_BLOCK==1) && /^\s*sha256\s+"(\w+)"\s+=>\s+:\w+\s*$/ ) + { if ( $1 eq "'"$BOTTLE_HASH"'" ) {$_="";}; next; } + ' <"$FORMULA" >"${FORMULA}.new" + # Depending on diff version, 1 may mean differences found + # https://stackoverflow.com/questions/6971284/what-are-the-error-exit-values-for-diff + diff -u "$FORMULA" "${FORMULA}.new" || test $? -le 1 + ( cd $(dirname "$FORMULA") + FORMULA=$(basename "$FORMULA") + mv -v "${FORMULA}.new" "$FORMULA" + git commit -m "Removed obsolete local bottle ${JSON_VERSION}_${JSON_REBUILD} :${OS_CODENAME}" "$FORMULA" + ) + fi + + if [ -n "$BOTTLE" ]; then rm "$BOTTLE"; fi + rm -f "$BOTTLE_LINK" + rm "$JSON" + + #(Re)add metadata to the formula otherwise + else + if [ "$FORMULA_BOTTLE_HASH" == "$BOTTLE_HASH" ]; then + echo "The cached bottle is already present in the formula" + else + brew bottle --merge --write "$JSON" + fi + fi + done +} + + +function brew_cache_cleanup { + #Cleanup caching directories + # Is supposed to be called in before_cache + + #Lefovers from some failure probably + rm -f "$BREW_LOCAL_BOTTLE_METADATA"/*.tar.gz + + #`brew cleanup` may delete locally-built bottles that weren't needed this time + # so we're saving and restoring them + local BOTTLE_LINK BOTTLE + for BOTTLE_LINK in "$BREW_LOCAL_BOTTLE_METADATA"/*.lnk; do + [ -e "$BOTTLE_LINK" ] || break + BOTTLE=$(cat "$BOTTLE_LINK") + ln "$BOTTLE" "$BREW_LOCAL_BOTTLE_METADATA/" + done + brew cleanup + local BOTTLE_BASENAME + for BOTTLE_LINK in "$BREW_LOCAL_BOTTLE_METADATA"/*.lnk; do + [ -e "$BOTTLE_LINK" ] || break + BOTTLE=$(cat "$BOTTLE_LINK") + BOTTLE_BASENAME=$(basename "$BOTTLE") + if test ! -e "$BOTTLE"; then + echo "Restoring: $BOTTLE_BASENAME" + mv "$BREW_LOCAL_BOTTLE_METADATA/$BOTTLE_BASENAME" "$BOTTLE" + else + rm "$BREW_LOCAL_BOTTLE_METADATA/$BOTTLE_BASENAME" + fi + done +} + + +function brew_go_bootstrap_mode { +# Can be overridden +# Terminate the build but ensure saving the cache + + echo "Going into cache bootstrap mode" + + #Can't just `exit` because that would terminate the build without saving the cache + #Have to replace further actions with no-ops + + eval ' + function '"$cmd"' { return 0; } + function repair_wheelhouse { return 0; } + function install_run { + echo -e "\nBuilding dependencies took too long. Restart the build in Travis UI to continue from cache.\n" + + # Travis runs user scripts via `eval` i.e. in the same shell process. + # So have to unset errexit in order to get to cache save stage + set +e; return 1 + }' +} + + + +#Internal functions + +function _brew_parse_bottle_json { + # Parse JSON file resulting from `brew bottle --json` + # and save data into specified variables + + local JSON; JSON="${1:?}"; shift + + local JSON_DATA; JSON_DATA=$(python2.7 -c 'if True: + import sys,json; j=json.load(open(sys.argv[1],"rb")); [name]=j.keys(); [pdata]=j.values() + print name + print pdata["formula"]["pkg_version"] + print pdata["bottle"]["rebuild"] + [(tag_name, tag_dict)]=pdata["bottle"]["tags"].items() + print tag_name + print tag_dict["sha256"] + ' "$JSON") + + unset JSON + + { local i v; for i in {1..5}; do + read -r v + eval "${1:?}=\"$v\"" + shift + done } <<< "$JSON_DATA" +} + +function _brew_parse_package_info { + # Get and parse `brew info --json` about a package + # and save data into specified variables + + local PACKAGE; PACKAGE="${1:?}"; shift + local OS_CODENAME;OS_CODENAME="${1:?}"; shift + + local JSON_DATA; JSON_DATA=$(python2.7 -c 'if True: + import sys, json, subprocess; j=json.loads(subprocess.check_output(("brew","info","--json=v1",sys.argv[1]))) + data=j[0] + revision=data["revision"] + # in bottle''s json, revision is included into version; here, they are separate + print data["versions"]["stable"]+("_"+str(revision) if revision else "") + bottle_data=data["bottle"]["stable"] + print bottle_data["rebuild"] + print bottle_data["files"].get(sys.argv[2],{"sha256":"!?"})["sha256"] #prevent losing trailing blank line to command substitution + ' \ + "$PACKAGE" "$OS_CODENAME"); JSON_DATA="${JSON_DATA%\!\?}" #!? can't occur in a hash + + unset PACKAGE OS_CODENAME + + { local i v; for i in {1..3}; do + read -r v + eval "${1:?}=\"$v\"" + shift + done } <<< "$JSON_DATA" +} + + + +function _brew_is_bottle_available { + + local PACKAGE;PACKAGE="${1:?}" + + local INFO="$(brew info "$PACKAGE" | head -n 1)" + if grep -qwF '(bottled)' <<<"$INFO"; then + echo "Bottle available: $INFO" + return 0 + else + echo "Bottle not available: $INFO" + return 1 + fi +} + +function _brew_install_and_cache { + # Install bottle or make and cache bottle. + # assumes that deps were already installed. + + local PACKAGE;PACKAGE="${1:?}" + local USE_BOTTLE;USE_BOTTLE="${2:?}" + local VERB + + if brew list --versions "$PACKAGE"; then + if ! (brew outdated | grep -qx "$PACKAGE"); then + echo "Already the latest version: $PACKAGE" + return 0 + fi + VERB=upgrade + else + VERB=install + fi + + if [[ "$USE_BOTTLE" -gt 0 ]]; then + echo "Installing bottle for: $PACKAGE" + brew $VERB "$PACKAGE" + else + echo "Building bottle for: $PACKAGE" + brew $VERB --build-bottle "$PACKAGE" + exec 3>&1 + local OUT=$(brew bottle --json "$PACKAGE" | tee /dev/fd/3) + exec 3>&- + + ls "$PACKAGE"* + # doesn't seem to be a documented way to get file names + local BOTTLE; BOTTLE=$(grep -Ee '^./' <<<"$OUT") + #proper procedure as per https://discourse.brew.sh/t/how-are-bottle-and-postinstall-related-is-it-safe-to-run-bottle-after-postinstall/3410/4 + brew uninstall "$PACKAGE" + brew install "$BOTTLE" + + local JSON; JSON=$(sed -E 's/bottle(.[[:digit:]]+)?\.tar\.gz$/bottle.json/' <<<"$BOTTLE") + + #`brew bottle --merge` doesn't return nonzero on nonexisting json file + test -f "$JSON" -a -f "$BOTTLE" + + brew bottle --merge --write "$JSON" + local CACHED_BOTTLE; CACHED_BOTTLE="$(brew --cache "$PACKAGE")" + mv "$BOTTLE" "$CACHED_BOTTLE"; + local CACHED_JSON; CACHED_JSON="${BREW_LOCAL_BOTTLE_METADATA}/$(basename "$JSON")" + mv "$JSON" "$CACHED_JSON" + #Symlinks aren't cached by Travis. Will just save paths in files then. + local BOTTLE_LINK; BOTTLE_LINK="${CACHED_JSON}.bottle.lnk" + echo "$CACHED_BOTTLE" >"$BOTTLE_LINK" + + fi +} + + + + +function _brew_check_elapsed_build_time { + # If time limit has been reached, + # arrange for further build to be skipped and return 1 + + local TIME_START;TIME_START="${1:?}" + local TIME_LIMIT;TIME_LIMIT="${2:?}" + + local ELAPSED_TIME;ELAPSED_TIME=$(($(date +%s) - $TIME_START)) + echo "Elapsed time: "$(($ELAPSED_TIME/60))"m (${ELAPSED_TIME}s)" + + if [[ "$ELAPSED_TIME" -gt $TIME_LIMIT ]]; then + brew_go_bootstrap_mode + return 1 + fi + return 0 +} + +function _brew_check_slow_building_ahead { + + #If the package's projected build completion is higher than hard limit, + # skip it and arrange for further build to be skipped and return 1 + + local PACKAGE="${1:?}" + local TIME_START="${2:?}" + local TIME_HARD_LIMIT="${3:?}" + + PROJECTED_BUILD_TIME=$(echo "$BREW_SLOW_BUILIDING_PACKAGES" | awk '$1=="'"$PACKAGE"'"{print $2}') + [ -z "$PROJECTED_BUILD_TIME" ] && return 0 || true + + local PROJECTED_BUILD_END_ELAPSED_TIME + PROJECTED_BUILD_END_ELAPSED_TIME=$(( $(date +%s) - TIME_START + PROJECTED_BUILD_TIME * 60)) + + if [[ "$PROJECTED_BUILD_END_ELAPSED_TIME" -ge "$TIME_HARD_LIMIT" ]]; then + echo -e "\nProjected build end elapsed time for $PACKAGE: $((PROJECTED_BUILD_END_ELAPSED_TIME/60))m ($PROJECTED_BUILD_END_ELAPSED_TIMEs)" + brew_go_bootstrap_mode + return 1 + fi + return 0 +}