azure-pipelines-data.yml

trigger:
  branches:
    include:
      - master
      - 'v*'

resources:
  repositories:
    - repository: LiberTEM-live
      type: github
      endpoint: LiberTEM
      name: LiberTEM/LiberTEM-live

variables:
  - name: PIP_CACHE_DIR
    value: $(Pipeline.Workspace)/.pip
  - group: Packaging
  - group: Coverage
  - name: npm_config_cache
    value: $(Pipeline.Workspace)/.npm
  - name: tox_dir
    value: $(Build.Repository.LocalPath)/.tox

parameters:
  # by default, the tox environment is kept around, so we need this parameter
  # as a way to manually re-create it, in case dependencies change.
  - name: recreate_tox
    displayName: Recreate tox environments?
    type: boolean
    default: false

  # This parameter allows to run slower tests, this is useful for running more
  # comprehensive tests, for example before doing a release. This also enables more
  # jobs to build on usually skipped Python versions.
  - name: run_slow
    displayName: Also run slow tests
    type: boolean
    default: false

stages:
  - stage: test
    jobs:
    - job: check_example_notebooks
      pool: DataAccess
      variables:
        TESTDATA_BASE_PATH: '/data/'
      steps:
      - bash: find /data/
        displayName: 'list available data'

      - task: UsePythonVersion@0
        displayName: 'Use Python 3.9'
        inputs:
          versionSpec: '3.9'

#      - task: Cache@2
#        inputs:
#          key: 'python | "$(Agent.OS)" | test_requirements.txt'
#          restoreKeys: |
#            python | "$(Agent.OS)"
#            python
#          path: $(PIP_CACHE_DIR)
#        displayName: 'cache pip packages'

      - bash: python3.9 -m venv venv
        displayName: 'create venv'

      - bash: ./venv/bin/pip install tox tox-uv
        displayName: 'install tox'

      - ${{ if eq(parameters.recreate_tox, true) }}:
        - bash: ./venv/bin/tox -e notebooks -r -- examples/*.ipynb -v
          displayName: 'Run nbval tests $(Agent.OS) (recreating)'

      - ${{ if eq(parameters.recreate_tox, false) }}:
        - bash: ./venv/bin/tox -e notebooks -- examples/*.ipynb -v
          displayName: 'Run nbval tests $(Agent.OS)'

      - bash: ./scripts/codecov.sh -f ./coverage.xml
        displayName: 'Submit coverage to codecov.io'
        ${{ if and(not(eq(variables['Build.Reason'], 'PullRequest')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) }}:
          env:
            CODECOV_TOKEN: $(CODECOV_TOKEN)

      - task: PublishTestResults@2
        condition: succeededOrFailed()
        inputs:
          testResultsFiles: 'junit.xml'
          testRunTitle: 'Publish test results for Python $(python.version)'

    - job: data_tests
      pool: DataAccess
      strategy:
        matrix:
          Python39:
            python.version: '3.9'
            TOXENV: 'py39-data'
          Python310:
            python.version: '3.10'
            TOXENV: 'py310-data'
          Python311:
            python.version: '3.11'
            TOXENV: 'py311-data'
          Python312:
            python.version: '3.12'
            TOXENV: 'py312-data'
      variables:
        TESTDATA_BASE_PATH: '/data/'
      steps:
      - bash: find /data/
        displayName: 'list available data'

      - task: UsePythonVersion@0
        displayName: 'Use Python $(python.version)'
        inputs:
          versionSpec: '$(python.version)'

      - bash: python3.9 -m venv venv
        displayName: 'create venv'

      - bash: ./venv/bin/pip install tox tox-uv
        displayName: 'install tox (w/ uv)'

      - ${{ if eq(parameters.recreate_tox, true) }}:
        - bash: ./venv/bin/tox -r
          displayName: 'Run tox tests $(TOXENV) $(Agent.OS) (recreating)'

      - ${{ if eq(parameters.recreate_tox, false) }}:
        - bash: ./venv/bin/tox
          displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'

      - ${{ if eq(parameters.run_slow, true) }}:
        - bash: ./venv/bin/tox -- -m "slow and not dist" --cov-append
          displayName: 'Run tox slow tests $(TOXENV) $(Agent.OS)'

      - bash: ./scripts/codecov.sh -f ./coverage.xml
        displayName: 'Submit coverage to codecov.io'
        ${{ if and(not(eq(variables['Build.Reason'], 'PullRequest')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) }}:
          env:
            CODECOV_TOKEN: $(CODECOV_TOKEN)

      - task: PublishTestResults@2
        condition: succeededOrFailed()
        inputs:
          testResultsFiles: 'junit.xml'
          testRunTitle: 'Publish test results for Python $(python.version)'

    - job: numba_coverage
      pool: DataAccess
      variables:
        TOXENV: 'numba_coverage'
      steps:
      - task: UsePythonVersion@0
        displayName: 'Use Python 3.9'
        inputs:
          versionSpec: '3.9'

      - bash: pip install -U tox tox-uv
        displayName: 'install requirements'

      - ${{ if eq(parameters.recreate_tox, true) }}:
        - bash: tox -r
          displayName: 'Run tox tests $(TOXENV) $(Agent.OS) (recreating)'

      - ${{ if eq(parameters.recreate_tox, false) }}:
        - bash: tox
          displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'

      - bash: ./scripts/codecov.sh -f ./coverage.xml
        displayName: 'Submit coverage to codecov.io'
        ${{ if and(not(eq(variables['Build.Reason'], 'PullRequest')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) }}:
          env:
            CODECOV_TOKEN: $(CODECOV_TOKEN)

      - task: PublishTestResults@2
        condition: succeededOrFailed()
        inputs:
          testResultsFiles: 'junit.xml'
          testRunTitle: 'Publish test results for numba coverage on Python 3.9'

    - job: lint
      pool: DataAccess
      strategy:
        matrix:
          docs-check:
            TOXENV: 'docs-check'
      variables:
        TOXENV: '$(TOXENV)'
      steps:
      - task: UsePythonVersion@0
        displayName: 'Use Python 3.9'
        inputs:
          versionSpec: '3.9'
      - bash: apt-get update && apt-get install -y pandoc graphviz
        displayName: install required debian packages

      - bash: pip install -U tox tox-uv
        displayName: 'install requirements'

      - ${{ if eq(parameters.recreate_tox, true) }}:
        - bash: tox -r
          displayName: 'Run tox tests $(TOXENV) $(Agent.OS) (recreating)'

      - ${{ if eq(parameters.recreate_tox, false) }}:
        - bash: tox
          displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'

    - job: integration_live
      pool: DataAccess
      variables:
        TESTDATA_BASE_PATH: '/data/'
      steps:
      - task: UsePythonVersion@0
        displayName: 'Use Python 3.9'
        inputs:
          versionSpec: '3.9'

      - checkout: self
      - checkout: LiberTEM-live

      - bash: python3 -m venv $(Agent.TempDirectory)/venv/
        displayName: create virtualenv

      - bash: $(Agent.TempDirectory)/venv/bin/pip install uv
        displayName: install uv

      # FIXME: install with uv, which bugs out with the path right now...
      - bash: $(Agent.TempDirectory)/venv/bin/pip install $(Build.SourcesDirectory)/LiberTEM/
        displayName: install LiberTEM

      - bash: $(Agent.TempDirectory)/venv/bin/pip install "$(Build.SourcesDirectory)/LiberTEM-live/[dectris]"
        displayName: install LiberTEM-live

      - bash: . $(Agent.TempDirectory)/venv/bin/activate &&  $(Agent.TempDirectory)/venv/bin/uv pip install -r $(Build.SourcesDirectory)/LiberTEM-live/test_requirements.txt
        displayName: install LiberTEM-live test requirements

      - bash: cd $(Build.SourcesDirectory)/LiberTEM-live/ && $(Agent.TempDirectory)/venv/bin/pytest
        displayName: run LiberTEM-live tests