ci-testing-deploy.yml

name: CI

on:
  push:
    branches:
      - "*"
    tags-ignore:
      - "*"
    paths:
      - "!*.md"
      - "!*.png"
      - "!*.svg"
      - "!docs/**"
      - "!.vscode/**"
      - "**.py"
      - "**.js"
      - "**/requirements/*.txt"
      - "**.json"
      - ".github/workflows/ci-testing-deploy.yml"
  pull_request:
    branches:
      - "*"
    paths:
      - "!*.md"
      - "!*.png"
      - "!*.svg"
      - "!docs/**"
      - "!.vscode/**"
      - "!.vscode-template/**"
      - "**.py"
      - "**.js"
      - "**/requirements/*.txt"
      - "**.json"
      - ".github/workflows/ci-testing-deploy.yml"

env:
  # secrets can be set in settings/secrets on github
  DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
  CC_TEST_REPORTER_ID: 21a72eb30476c870140b1576258873a41be6692f71bd9aebe812174b7d8f4b4e
  DEFAULT_MAX_NANO_CPUS: 1000000000
  DEFAULT_MAX_MEMORY: 268435456

jobs:
  build-test-images:
    # this step comes first, so that it is executed as first job in push calls
    # in PR calls this step is anyway skipped
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    name: "[build] docker images"
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        if: github.event_name == 'push'
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        if: github.event_name == 'push'
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: show system environs
        if: github.event_name == 'push'
        run: ./ci/helpers/show_system_versions.bash
      - name: build images
        if: github.event_name == 'push'
        run: ./ci/build/test-images.bash build_images
      - name: set owner variable
        if: github.event_name == 'push'
        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
      - name: push images
        # only pushes have access to the docker credentials
        if: github.event_name == 'push'
        run: ./ci/deploy/dockerhub-test-images.bash

  unit-test-webserver-01:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: test isolated
        run: ./ci/github/unit-testing/webserver.bash test_isolated
      - name: test
        run: ./ci/github/unit-testing/webserver.bash test_with_db 01
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-webserver-02:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: test
        run: ./ci/github/unit-testing/webserver.bash test_with_db 02
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-webserver-03:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 03"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: test
        run: ./ci/github/unit-testing/webserver.bash test_with_db 03
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-api:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] api"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "api/tests/requirements.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install api
        run: ./ci/github/unit-testing/api.bash install
      - name: test
        run: ./ci/github/unit-testing/api.bash test

  unit-test-api-server:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] api-server"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/api-server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/api-server.bash install
      - name: test
        run: ./ci/github/unit-testing/api-server.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-catalog:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] catalog"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/catalog/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/catalog.bash install
      - name: test
        run: ./ci/github/unit-testing/catalog.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/catalog/test_failures
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-datcore-adapter:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] datcore-adapter"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/datcore-adapter/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/datcore-adapter.bash install
      - name: test
        run: ./ci/github/unit-testing/datcore-adapter.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/datcore-adapter/test_failures
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-director:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] director"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        # KEEP 3.6 Development of this service is frozen
        python: [3.6]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/director/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/director.bash install
      - name: test
        run: ./ci/github/unit-testing/director.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-director-v2:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] director-v2"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/director-v2/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/director-v2.bash install
      - name: test
        run: ./ci/github/unit-testing/director-v2.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-dask-task-models-library:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dask-task-models-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/dask-task-models-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dask-task-models-library.bash install
      - name: codestyle
        run: ./ci/github/unit-testing/dask-task-models-library.bash codestyle
      - name: test
        run: ./ci/github/unit-testing/dask-task-models-library.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-dask-sidecar:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dask-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/dask-sidecar/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dask-sidecar.bash install
      - name: test
        run: ./ci/github/unit-testing/dask-sidecar.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-dynamic-sidecar:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dynamic-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/dynamic-sidecar/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dynamic-sidecar.bash install
      - name: codestyle
        run: ./ci/github/unit-testing/dynamic-sidecar.bash codestyle
      - name: test
        run: ./ci/github/unit-testing/dynamic-sidecar.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-frontend:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] frontend"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        node: [14]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - uses: actions/setup-node@v3.3.0
        with:
          node-version: ${{ matrix.node }}
          cache: "npm"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/frontend.bash install
      - name: test
        run: ./ci/github/unit-testing/frontend.bash test
    # no coverage here??
    # - uses: codecov/codecov-action@v3.1.0
    #   with:
    #     flags: unittests #optional

  unit-test-python-linting:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] python-linting"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9", "3.10"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/service-library/requirements/_test.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/python-linting.bash install
      - name: test
        run: ./ci/github/unit-testing/python-linting.bash test

  unit-test-postgres-database:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] postgres-database"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/postgres-database/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/postgres-database.bash install
      - name: test
        run: ./ci/github/unit-testing/postgres-database.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter
          ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-service-integration:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] service-integration"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/service-integration/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/service-integration.bash install
      - name: test
        run: ./ci/github/unit-testing/service-integration.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-service-library:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] service-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/service-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/service-library.bash install_all
      - name: test
        run: ./ci/github/unit-testing/service-library.bash test_all
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-settings-library:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] settings-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/settings-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/settings-library.bash install
      - name: test
        run: ./ci/github/unit-testing/settings-library.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-models-library:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] models-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/models-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/models-library.bash install
      - name: test
        run: ./ci/github/unit-testing/models-library.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-simcore-sdk:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] simcore-sdk"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: install rclone in CI
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/simcore-sdk/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/simcore-sdk.bash install
      - name: test
        run: ./ci/github/unit-testing/simcore-sdk.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  unit-test-storage:
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] storage"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/storage/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/storage.bash install
      - name: test
        run: ./ci/github/unit-testing/storage.bash test
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: unittests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  integration-test-webserver-01:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] webserver 01"
    needs: [build-test-images]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - name: set PR default variables
        # only pushes have access to the docker credentials, use a default
        if: github.event_name == 'pull_request'
        run: |
          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/webserver.bash install
      - name: test
        run: ./ci/github/integration-testing/webserver.bash test 01
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/web/server/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/webserver.bash clean_up
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: integrationtests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  integration-test-webserver-02:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] webserver 02"
    needs: [build-test-images]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - name: set PR default variables
        # only pushes have access to the docker credentials, use a default
        if: github.event_name == 'pull_request'
        run: |
          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/webserver.bash install
      - name: test
        run: ./ci/github/integration-testing/webserver.bash test 02
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/web/server/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/webserver.bash clean_up
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: integrationtests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  integration-test-director-v2-01:
    timeout-minutes: 35 # FIXME: temporary up (PC). if this timeout gets too small, then split the tests
    name: "[int] director-v2 01"
    needs: [build-test-images]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - name: set PR default variables
        # only pushes have access to the docker credentials, use a default
        if: github.event_name == 'pull_request'
        run: |
          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/director-v2/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/director-v2.bash install
      - name: test
        run: ./ci/github/integration-testing/director-v2.bash test 01
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/director-v2.bash clean_up
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: integrationtests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  integration-test-director-v2-02:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] director-v2 02"
    needs: [build-test-images]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - name: set PR default variables
        # only pushes have access to the docker credentials, use a default
        if: github.event_name == 'pull_request'
        run: |
          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker

      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup rclone docker volume plugin
        run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "services/director-v2/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/director-v2.bash install
      - name: test
        run: ./ci/github/integration-testing/director-v2.bash test 02
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/director-v2.bash clean_up
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: integrationtests #optional
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  integration-test-simcore-sdk:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] simcore-sdk"
    needs: [build-test-images]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - name: set PR default variables
        # only pushes have access to the docker credentials, use a default
        if: github.event_name == 'pull_request'
        run: |
          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: install rclone in CI
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "packages/simcore-sdk/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/simcore-sdk.bash install
      - name: test
        run: ./ci/github/integration-testing/simcore-sdk.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./packages/simcore-sdk/test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/integration-testing/simcore-sdk.bash clean_up
      - uses: codecov/codecov-action@v3.1.0
        with:
          flags: integrationtests
      - name: prepare codeclimate coverage file
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
      - name: upload codeclimate coverage
        uses: actions/upload-artifact@v3
        with:
          name: codeclimate-${{ github.job }}-coverage
          path: codeclimate.${{ github.job }}_coverage.json

  system-test-public-api:
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[sys] public api"
    needs: [build-test-images]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - name: set PR default variables
        # only pushes have access to the docker credentials, use a default
        if: github.event_name == 'pull_request'
        run: |
          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "tests/public-api/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/system-testing/public-api.bash install
      - name: test
        run: ./ci/github/system-testing/public-api.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/public-api.bash clean_up

  system-test-swarm-deploy:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] deploy simcore"
    needs: [build-test-images]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - name: set PR default variables
        # only pushes have access to the docker credentials, use a default
        if: github.event_name == 'pull_request'
        run: |
          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "tests/swarm-deploy/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/system-testing/swarm-deploy.bash install
      - name: test
        run: ./ci/github/system-testing/swarm-deploy.bash test
      - name: upload failed tests logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./test_failures
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/swarm-deploy.bash clean_up

  system-test-e2e:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] e2e"
    needs: [build-test-images]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        node: [14]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - name: set PR default variables
        # only pushes have access to the docker credentials, use a default
        if: github.event_name == 'pull_request'
        run: |
          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "tests/e2e/requirements/requirements.txt"
      - uses: actions/setup-node@v3.3.0
        with:
          node-version: ${{ matrix.node }}
          cache: "npm"
          cache-dependency-path: "tests/e2e/package-lock.json"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: setup
        run: ./ci/github/system-testing/e2e.bash install
      - name: test
        run: ./ci/github/system-testing/e2e.bash test
      - name: dump docker logs
        id: docker_logs_dump
        if: failure()
        run: ./ci/github/system-testing/e2e.bash dump_docker_logs
      - name: upload docker logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_docker_logs
          path: ./tests/e2e/test_failures
      - name: upload screenshots
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_screenshots
          path: tests/e2e/screenshots
      - name: upload e2e logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.job }}_logs
          path: tests/e2e/logs
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/e2e.bash clean_up

  system-test-environment-setup:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] environment setup"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - name: set PR default variables
        # only pushes have access to the docker credentials, use a default
        if: github.event_name == 'pull_request'
        run: |
          export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: setup python environment
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "tests/environment-setup/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/system-testing/environment-setup.bash install
      - name: test
        run: ./ci/github/system-testing/environment-setup.bash test
      - name: cleanup
        if: always()
        run: ./ci/github/system-testing/environment-setup.bash clean_up

  coverage:
    needs:
      [
        unit-test-catalog,
        unit-test-datcore-adapter,
        unit-test-director,
        unit-test-director-v2,
        unit-test-dynamic-sidecar,
        unit-test-dask-task-models-library,
        unit-test-dask-sidecar,
        unit-test-service-integration,
        unit-test-service-library,
        unit-test-settings-library,
        unit-test-models-library,
        unit-test-simcore-sdk,
        unit-test-storage,
        unit-test-webserver-01,
        unit-test-webserver-02,
        unit-test-webserver-03,
        integration-test-webserver-01,
        integration-test-webserver-02,
        integration-test-director-v2-01,
        integration-test-director-v2-02,
        integration-test-simcore-sdk,
      ]
    name: coverage
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/download-artifact@v3
        name: download all artifacts
        with:
          path: artifacts
      - name: put all artefacts together
        run: |
          mkdir --parents all_coverages
          cp artifacts/codeclimate*/*.json all_coverages
          ls --all -l all_coverages/*.json
      - name: install codeclimate test reporter
        run: |
          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
          ./cc-test-reporter sum-coverage all_coverages/codeclimate.*.json --parts $(ls all_coverages/*.json | wc --lines)
      - name: upload coverages
        run: |
          ./cc-test-reporter upload-coverage

  deploy:
    name: deploy to dockerhub
    if: github.event_name == 'push'
    needs:
      [
        unit-test-api-server,
        unit-test-api,
        unit-test-catalog,
        unit-test-dask-sidecar,
        unit-test-dask-task-models-library,
        unit-test-datcore-adapter,
        unit-test-director-v2,
        unit-test-director,
        unit-test-dynamic-sidecar,
        unit-test-frontend,
        unit-test-models-library,
        unit-test-postgres-database,
        unit-test-python-linting,
        unit-test-service-integration,
        unit-test-service-library,
        unit-test-settings-library,
        unit-test-simcore-sdk,
        unit-test-storage,
        unit-test-webserver-01,
        unit-test-webserver-02,
        unit-test-webserver-03,
        integration-test-director-v2-01,
        integration-test-director-v2-02,
        integration-test-simcore-sdk,
        integration-test-webserver-01,
        integration-test-webserver-02,
        system-test-public-api,
        system-test-swarm-deploy,
      ]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.9"]
        os: [ubuntu-20.04]
        docker_buildx: [v0.7.1]
        docker_compose: [1.29.1]
        include:
          - docker_compose: 1.29.1
            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker
      - name: setup docker-compose
        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
      - name: set owner variable
        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
      - if: github.ref == 'refs/heads/master'
        name: deploy master image
        env:
          TAG_PREFIX: master-github
        run: ./ci/deploy/dockerhub-deploy.bash
      - if: contains(github.ref, 'refs/heads/hotfix_v')
        name: deploy release hotfix image
        env:
          TAG_PREFIX: hotfix-github
        run: ./ci/deploy/dockerhub-deploy.bash
      - if: contains(github.ref, 'refs/heads/hotfix_staging_')
        name: deploy staging hotfix image
        env:
          TAG_PREFIX: hotfix-staging-github
        run: ./ci/deploy/dockerhub-deploy.bash