From e9c40dd16fdf3a4b954f2e320ee5ea07a469e444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Sat, 20 Nov 2021 12:40:37 -0300 Subject: [PATCH 01/10] add env cache instructions --- README.md | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/README.md b/README.md index f8f84946..81b661a6 100644 --- a/README.md +++ b/README.md @@ -528,6 +528,8 @@ jobs: ## Caching +### Caching packages + If you want to enable package caching for conda you can use the [cache action](https://github.com/actions/cache) using `~/conda_pkgs_dir` as path for conda packages. @@ -570,6 +572,73 @@ you may want to [cache those dependencies separately](https://docs.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#caching-dependencies), as they are not included in the conda package cache. +### Caching environments + +A Miniforge variant is recommended to cache deployed environments, since the +Miniconda installation path requires succesive changes of folder ownership in +order to work with the `cache` action. + +Every operating system use a different Miniforge `prefix`, so if you want to +cache the environment on all of them you must use a `matrix` strategy: + +``` + strategy: + matrix: + include: + - os: ubuntu-latest + label: linux-64 + prefix: /usr/share/miniconda3/envs/your-env-name + + - os: macos-latest + label: osx-64 + prefix: /Users/runner/miniconda3/envs/your-env-name + + - os: windows-latest + label: win-64 + prefix: C:\Miniconda3\envs\your-env-name +``` + +Then, the first installation step should setup a Miniconda variant without +specifying a environment file. + +``` + - name: Setup Mambaforge + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: foo + use-mamba: true +``` + +It's a good idea to refresh the cache every 24 hours to avoid inconsistencies +of package versions between the CI pipeline and local installations. You can +skip this step if you use an environment file product of `conda env export` +or `conda list --explicit`. + +``` + - name: Set cache date + run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV + + - uses: actions/cache@v2 + with: + path: ${{ matrix.prefix }} + key: ${{ matrix.label }}-conda-${{ hashFiles('environment.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} + env: + # Increase this value to reset cache if environment.yml has not changed + CACHE_NUMBER: 0 + id: cache +``` + +Finally, update the environment based on the environment file if the cache +does not exist. + +``` + - name: Update environment + run: mamba env update -n your-env-name -f environment.yml + if: steps.cache.outputs.cache-hit != 'true' +``` + ### Use a default shell Assuming you are using the bash shell, now adding to `shell: bash -l {0}` to From e996a97d64e5e314336f0fceb7af835ff56eaa94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Sat, 20 Nov 2021 12:42:44 -0300 Subject: [PATCH 02/10] mark code snippets as yaml --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 81b661a6..9798afd4 100644 --- a/README.md +++ b/README.md @@ -579,9 +579,9 @@ Miniconda installation path requires succesive changes of folder ownership in order to work with the `cache` action. Every operating system use a different Miniforge `prefix`, so if you want to -cache the environment on all of them you must use a `matrix` strategy: +cache the environment on all of them you must use a `matrix` strategy. -``` +```yaml strategy: matrix: include: @@ -601,7 +601,7 @@ cache the environment on all of them you must use a `matrix` strategy: Then, the first installation step should setup a Miniconda variant without specifying a environment file. -``` +```yaml - name: Setup Mambaforge uses: conda-incubator/setup-miniconda@v2 with: @@ -616,7 +616,7 @@ of package versions between the CI pipeline and local installations. You can skip this step if you use an environment file product of `conda env export` or `conda list --explicit`. -``` +```yaml - name: Set cache date run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV @@ -633,7 +633,7 @@ or `conda list --explicit`. Finally, update the environment based on the environment file if the cache does not exist. -``` +```yaml - name: Update environment run: mamba env update -n your-env-name -f environment.yml if: steps.cache.outputs.cache-hit != 'true' From a3345a3b6698049d982ef3db8139e5d982584416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Sat, 20 Nov 2021 12:44:07 -0300 Subject: [PATCH 03/10] fix env name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9798afd4..670f13e3 100644 --- a/README.md +++ b/README.md @@ -607,7 +607,7 @@ specifying a environment file. with: miniforge-variant: Mambaforge miniforge-version: latest - activate-environment: foo + activate-environment: your-env-name use-mamba: true ``` From 837cecebca923c51cbaf271b5492caeec45e77cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 23 Nov 2021 14:32:24 -0300 Subject: [PATCH 04/10] add example workflow --- .github/workflows/caching-envs-example.yml | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/caching-envs-example.yml diff --git a/.github/workflows/caching-envs-example.yml b/.github/workflows/caching-envs-example.yml new file mode 100644 index 00000000..9e2269b9 --- /dev/null +++ b/.github/workflows/caching-envs-example.yml @@ -0,0 +1,65 @@ +name: "Caching Env Example" + +on: + pull_request: + branches: + - "*" + push: + branches: + - "master" + schedule: + # Note that cronjobs run on master/main by default + - cron: "0 0 * * *" + +jobs: + caching-example: + name: Caching + + # prevent cronjobs from running on forks + if: + (github.event_name == 'schedule' && github.repository == + 'conda-incubator/setup-miniconda') || (github.event_name != 'schedule') + + strategy: + matrix: + include: + - os: ubuntu-latest + label: linux-64 + prefix: /usr/share/miniconda3/envs/anaconda-client-env + + - os: macos-latest + label: osx-64 + prefix: /Users/runner/miniconda3/envs/anaconda-client-env + + - os: windows-latest + label: win-64 + prefix: C:\Miniconda3\envs\anaconda-client-env + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + + - name: Setup Mambaforge + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: anaconda-client-env + use-mamba: true + + - name: Set cache date + run: echo "DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV + + - name: Cache conda env + uses: actions/cache@v2 + with: + path: ${{ matrix.prefix }} + key: ${{ matrix.label }}-conda-${{ hashFiles('etc/example-environment-caching.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 0 + id: cache + + - name: Update environment + run: mamba env update -n anaconda-client-env -f etc/example-environment-caching.yml + if: steps.cache.outputs.cache-hit != 'true' \ No newline at end of file From 5affed6fae1f1285841c79b4efa2395b0ec31078 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 23 Nov 2021 14:32:42 -0300 Subject: [PATCH 05/10] match readme env and yaml file names with example --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 670f13e3..7bd280fe 100644 --- a/README.md +++ b/README.md @@ -587,15 +587,15 @@ cache the environment on all of them you must use a `matrix` strategy. include: - os: ubuntu-latest label: linux-64 - prefix: /usr/share/miniconda3/envs/your-env-name + prefix: /usr/share/miniconda3/envs/anaconda-client-env - os: macos-latest label: osx-64 - prefix: /Users/runner/miniconda3/envs/your-env-name + prefix: /Users/runner/miniconda3/envs/anaconda-client-env - os: windows-latest label: win-64 - prefix: C:\Miniconda3\envs\your-env-name + prefix: C:\Miniconda3\envs\anaconda-client-env ``` Then, the first installation step should setup a Miniconda variant without @@ -607,7 +607,7 @@ specifying a environment file. with: miniforge-variant: Mambaforge miniforge-version: latest - activate-environment: your-env-name + activate-environment: anaconda-client-env use-mamba: true ``` @@ -623,9 +623,9 @@ or `conda list --explicit`. - uses: actions/cache@v2 with: path: ${{ matrix.prefix }} - key: ${{ matrix.label }}-conda-${{ hashFiles('environment.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} + key: ${{ matrix.label }}-conda-${{ hashFiles('etc/example-environment-caching.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} env: - # Increase this value to reset cache if environment.yml has not changed + # Increase this value to reset cache if etc/example-environment.yml has not changed CACHE_NUMBER: 0 id: cache ``` @@ -635,7 +635,7 @@ does not exist. ```yaml - name: Update environment - run: mamba env update -n your-env-name -f environment.yml + run: mamba env update -n anaconda-client-env -f etc/example-environment-caching.yml if: steps.cache.outputs.cache-hit != 'true' ``` From c952eb5d8f69844671c40eca319dd3432c06901b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 23 Nov 2021 14:54:13 -0300 Subject: [PATCH 06/10] test cache --- .github/workflows/caching-envs-example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/caching-envs-example.yml b/.github/workflows/caching-envs-example.yml index 9e2269b9..1d8fac12 100644 --- a/.github/workflows/caching-envs-example.yml +++ b/.github/workflows/caching-envs-example.yml @@ -62,4 +62,4 @@ jobs: - name: Update environment run: mamba env update -n anaconda-client-env -f etc/example-environment-caching.yml - if: steps.cache.outputs.cache-hit != 'true' \ No newline at end of file + if: steps.cache.outputs.cache-hit != 'true' From 796550092ef6c29864d32e2c845aa3ff852bdf4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 23 Nov 2021 15:11:34 -0300 Subject: [PATCH 07/10] try to add badge to readme --- .github/workflows/caching-envs-example.yml | 2 +- README.md | 23 +++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/caching-envs-example.yml b/.github/workflows/caching-envs-example.yml index 1d8fac12..200bd5c6 100644 --- a/.github/workflows/caching-envs-example.yml +++ b/.github/workflows/caching-envs-example.yml @@ -13,7 +13,7 @@ on: jobs: caching-example: - name: Caching + name: Caching-Env # prevent cronjobs from running on forks if: diff --git a/README.md b/README.md index 7bd280fe..738e3fe5 100644 --- a/README.md +++ b/README.md @@ -44,17 +44,18 @@ possibility of automatically activating the `test` environment on all shells. > Each of the examples below is discussed in a dedicated section below. -| Documentation | Workflow Status | -| ----------------------------------------------- | --------------------------------------------------- | -| [Basic usage](#example-1-basic-usage) | [![Basic Usage Status][ex1-badge]][ex1] | -| [Other shells](#example-2-other-shells) | [![Other Shells Status][ex2-badge]][ex2] | -| [Other options](#example-3-other-options) | [![Other Options Status][ex3-badge]][ex3] | -| [Channels](#example-4-conda-options) | [![Channels Status][ex4-badge]][ex4] | -| [Custom installer](#example-5-custom-installer) | [![Custom Installer Status][ex5-badge]][ex5] | -| [Mamba](#example-6-mamba) | [![Mamba Status][ex6-badge]][ex6] | -| [Lockfiles](#example-7-explicit-specification) | [![ Lockfiles Status][ex7-badge]][ex7] | -| [Miniforge](#example-10-miniforge) | [![Miniforge Status][ex10-badge]][ex10] | -| [Caching](#caching) | [![Caching Example Status][caching-badge]][caching] | +| Documentation | Workflow Status | +| ----------------------------------------------- | ----------------------------------------------------------------| +| [Basic usage](#example-1-basic-usage) | [![Basic Usage Status][ex1-badge]][ex1] | +| [Other shells](#example-2-other-shells) | [![Other Shells Status][ex2-badge]][ex2] | +| [Other options](#example-3-other-options) | [![Other Options Status][ex3-badge]][ex3] | +| [Channels](#example-4-conda-options) | [![Channels Status][ex4-badge]][ex4] | +| [Custom installer](#example-5-custom-installer) | [![Custom Installer Status][ex5-badge]][ex5] | +| [Mamba](#example-6-mamba) | [![Mamba Status][ex6-badge]][ex6] | +| [Lockfiles](#example-7-explicit-specification) | [![Lockfiles Status][ex7-badge]][ex7] | +| [Miniforge](#example-10-miniforge) | [![Miniforge Status][ex10-badge]][ex10] | +| [Caching packages](#caching-packages) | [![Caching Example Status][caching-badge]][caching] | +| [Caching environments](#caching-environments) | [![Caching Env Example Status][caching-env-badge]][caching-env] | [ex1]: https://github.com/conda-incubator/setup-miniconda/actions/workflows/example-1.yml?query=branch%3Amaster From 169ba140e340701e8f495b6548fb6aff9f7e5330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ezequiel=20P=C3=A1ssaro?= Date: Tue, 23 Nov 2021 15:44:59 -0300 Subject: [PATCH 08/10] improve description of cache inconsistencies --- README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 738e3fe5..93bcd8ab 100644 --- a/README.md +++ b/README.md @@ -614,8 +614,8 @@ specifying a environment file. It's a good idea to refresh the cache every 24 hours to avoid inconsistencies of package versions between the CI pipeline and local installations. You can -skip this step if you use an environment file product of `conda env export` -or `conda list --explicit`. +skip that step if you use a resolved environment file product of +`conda env export` or `conda list --explicit`. ```yaml - name: Set cache date @@ -631,6 +631,18 @@ or `conda list --explicit`. id: cache ``` +Keep in mind that hashing `etc/example-environment-caching.yml` is not the +same as hashing a resolved environment file. `conda` (and `mamba`) resolves +the dependencies declared in the YAML file according to the packages +available on the channels at installation time. Since packages are updated +all the time, you will not see these changes reflected in the cache until +the key gets updated by date. + +**This means that the same environment file can make your tests pass locally +but fail on CI, or the other way around. In that case, reset the cache +manually to see if that leads to consistent results, or use a resolved +environment file.** + Finally, update the environment based on the environment file if the cache does not exist. From 6d0e07a5235229bcb57230136485f0ed0e4d3811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 24 Nov 2021 12:05:00 +0100 Subject: [PATCH 09/10] del extra space --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 93bcd8ab..a7170daf 100644 --- a/README.md +++ b/README.md @@ -643,7 +643,7 @@ but fail on CI, or the other way around. In that case, reset the cache manually to see if that leads to consistent results, or use a resolved environment file.** -Finally, update the environment based on the environment file if the cache +Finally, update the environment based on the environment file if the cache does not exist. ```yaml From 81fac56008d6e37b1cf03cf4b5c7fdd051fe6d80 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 30 Dec 2021 13:04:45 +0200 Subject: [PATCH 10/10] Use 'npm run ...' to run scripts Fixes https://github.com/conda-incubator/setup-miniconda/runs/4663195920?check_suite_focus=true --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0f3eee9d..72894d9d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,5 +26,5 @@ jobs: - uses: actions/checkout@v2 - run: | npm install - npm check - npm build + npm run check + npm run build