ci.yml

# GitHub Actions CI definition for TensorBoard.
#
# YAML schema for GitHub Actions:
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
#
# Helpful YAML parser to clarify YAML syntax:
# https://yaml-online-parser.appspot.com/

name: CI

on:
  push:
    branches:
      - master
      - '[0-9]+.*'
      - 'ci-*'
    tags:
      - '*'
  pull_request: {}
  schedule:
    # 13:00 UTC is 05:00 in Pacific standard time (UTC-8), which is well
    # after nightly TensorFlow wheels are released (around 2--3 AM) and
    # just after nightly TensorBoard wheels are released (around 04:15).
    # (cron syntax: minute hour day-of-month month day-of-week)
    - cron: '0 13 * * *'

env:
  # Keep this Bazel version in sync with the `versions.check` directive
  # in our WORKSPACE file.
  BAZEL_VERSION: '4.0.0'
  BAZEL_SHA256SUM: '7bee349a626281fc8b8d04a7a0b0358492712377400ab12533aeb39c2eb2b901'
  BUILDTOOLS_VERSION: '3.0.0'
  BUILDIFIER_SHA256SUM: 'e92a6793c7134c5431c58fbc34700664f101e5c9b1c1fcd93b97978e8b7f88db'
  BUILDOZER_SHA256SUM: '3d58a0b6972e4535718cdd6c12778170ea7382de7c75bc3728f5719437ffb84d'
  TENSORFLOW_VERSION: 'tf-nightly'

jobs:
  build:
    runs-on: ubuntu-18.04
    needs: lint-python-flake8 # fail fast in case of "undefined variable" errors
    strategy:
      fail-fast: false
      matrix:
        tf_version_id: ['tf', 'notf']
        python_version: ['3.7']
    steps:
      - uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e
      - uses: actions/setup-python@152ba7c4dd6521b8e9c93f72d362ce03bf6c4f20
        with:
          python-version: ${{ matrix.python_version }}
          architecture: 'x64'
      - name: 'Set up Bazel'
        run: |
          ci/download_bazel.sh "${BAZEL_VERSION}" "${BAZEL_SHA256SUM}" ~/bazel
          sudo mv ~/bazel /usr/local/bin/bazel
          sudo chmod +x /usr/local/bin/bazel
          cp ./ci/bazelrc ~/.bazelrc
      - name: 'Configure build cache write credentials'
        env:
          CREDS: ${{ secrets.BAZEL_CACHE_SERVICE_ACCOUNT_CREDS }}
          EVENT_TYPE: ${{ github.event_name }}
        run: |
          if [ -z "${CREDS}" ]; then
            printf 'Using read-only cache (no credentials)\n'
            exit
          fi
          if [ "${EVENT_TYPE}" = pull_request ]; then
            printf 'Using read-only cache (PR build)\n'
            exit
          fi
          printf 'Using writable cache\n'
          creds_file=/tmp/service_account_creds.json
          printf '%s\n' "${CREDS}" >"${creds_file}"
          printf '%s\n' >>~/.bazelrc \
            "common --google_credentials=${creds_file}" \
            "common --remote_upload_local_results=true" \
            ;
      - name: 'Install TensorFlow'
        run: pip install "${TENSORFLOW_VERSION}"
        if: matrix.tf_version_id != 'notf'
      - name: 'Install Python dependencies'
        run: |
          python -m pip install -U pip
          pip install \
            -r ./tensorboard/pip_package/requirements.txt \
            -r ./tensorboard/pip_package/requirements_dev.txt \
            ;
      - name: 'Check Pip state'
        run: pip freeze --all
      - name: 'Bazel: fetch'
        run: bazel fetch //tensorboard/...
      - name: 'Bazel: build'
        run: bazel build //tensorboard/...
      - name: 'Bazel: test (with TensorFlow support)'
        run: bazel test //tensorboard/...
        if: matrix.tf_version_id != 'notf'
      - name: 'Bazel: test (non-TensorFlow only)'
        run: bazel test //tensorboard/... --test_tag_filters="support_notf"
        if: matrix.tf_version_id == 'notf'
      - name: 'Bazel: run Pip package test (with TensorFlow support)'
        run: |
          bazel run //tensorboard/pip_package:test_pip_package -- \
            --tf-version "${TENSORFLOW_VERSION}"
        if: matrix.tf_version_id != 'notf'
      - name: 'Bazel: run Pip package test (non-TensorFlow only)'
        run: |
          bazel run //tensorboard/pip_package:test_pip_package -- \
            --tf-version notf
        if: matrix.tf_version_id == 'notf'
      - name: 'Bazel: run manual tests'
        run: |
          bazel test //tensorboard/compat/tensorflow_stub:gfile_s3_test &&
          bazel test //tensorboard/summary/writer:event_file_writer_s3_test &&
          bazel test //tensorboard/compat/tensorflow_stub:gfile_fsspec_test &&
          bazel test //tensorboard/summary/writer:event_file_writer_fsspec_test

  build-data-server-pip:
    runs-on: ${{ matrix.platform }}
    strategy:
      fail-fast: false
      matrix:
        mode: ['native']
        platform: ['ubuntu-18.04', 'macos-10.15']
        rust_version: ['1.58.1']
        include:
          - mode: 'universal'
            platform: 'ubuntu-18.04'
            rust_version: '1.58.1'
    steps:
      - uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e
      - uses: actions/setup-python@152ba7c4dd6521b8e9c93f72d362ce03bf6c4f20
        with:
          python-version: '3.8'
          architecture: 'x64'
      - name: 'Cache Cargo artifacts'
        if: matrix.mode == 'native'
        uses: actions/cache@1a9e2138d905efd099035b49d8b7a3888c653ca8
        with:
          path: |
            tensorboard/data/server/target/
            # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            # Needed if we `cargo install` binaries (at time of writing, this
            # job doesn't but the files are tiny, and this will mitigate
            # confusion if we need to `cargo install` later)
            ~/.cargo/.crates.toml
            ~/.cargo/.crates2.json
          key: build-data-server-pip-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }}
      - name: 'Install Rust toolchain'
        if: matrix.mode == 'native'
        uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af
        with:
          toolchain: ${{ matrix.rust_version }}
          default: true
          components: rustfmt
      - name: 'Install Python packaging deps'
        run: |
          python -m pip install -U pip
          pip install \
            -c ./tensorboard/pip_package/requirements.txt \
            -c ./tensorboard/pip_package/requirements_dev.txt \
            setuptools wheel
      - name: 'Build'
        if: matrix.mode == 'native'
        run: cd tensorboard/data/server/ && cargo build --release
      - name: 'Test'
        if: matrix.mode == 'native'
        run: cd tensorboard/data/server/ && cargo test --release
      - name: 'Package (native)'
        if: matrix.mode == 'native'
        run: |
          rm -rf /tmp/pip_package && mkdir /tmp/pip_package
          python tensorboard/data/server/pip_package/build.py \
            --server-binary tensorboard/data/server/target/release/rustboard \
            --out-dir /tmp/pip_package \
            ;
      - name: 'Package (universal)'
        if: matrix.mode == 'universal'
        run: |
          rm -rf /tmp/pip_package && mkdir /tmp/pip_package
          python tensorboard/data/server/pip_package/build.py \
            --universal \
            --out-dir /tmp/pip_package \
            ;
      - name: 'Upload'
        uses: actions/upload-artifact@ee69f02b3dfdecd58bb31b4d133da38ba6fe3700
        with:
          name: tensorboard-data-server
          path: /tmp/pip_package/*

  lint-python-flake8:
    runs-on: ubuntu-18.04
    strategy:
      fail-fast: false
      matrix:
        # flake8 should run on each Python version that we target,
        # because the errors and warnings can differ due to language
        # changes, and we want to catch them all.
        python_version: ['3.6', '3.7']
    steps:
      - uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e
      - uses: actions/setup-python@152ba7c4dd6521b8e9c93f72d362ce03bf6c4f20
        with:
          python-version: ${{ matrix.python_version }}
          architecture: 'x64'
      - name: 'Install flake8'
        run: |
          python -m pip install -U pip
          pip install flake8 -c ./tensorboard/pip_package/requirements_dev.txt
      - run: pip freeze --all
      - name: 'Lint Python code for errors with flake8'
        # See: http://flake8.pycqa.org/en/3.7.8/user/error-codes.html
        # Use the comment '# noqa: <error code>' to suppress.
        run: flake8 . --count --select=E9,F63,F7,F82,F401 --show-source --statistics

  lint-python-yaml-docs:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e
      - uses: actions/setup-python@152ba7c4dd6521b8e9c93f72d362ce03bf6c4f20
        with:
          python-version: '3.6'
          architecture: 'x64'
      - name: 'Install black, yamllint, and the TensorFlow docs notebook tools'
        run: |
          python -m pip install -U pip
          nbfmt_version="174c9a5c1cc51a3af1de98d84824c811ecd49029"
          pip install black yamllint -c ./tensorboard/pip_package/requirements_dev.txt
          pip install -U git+https://github.com/tensorflow/docs@${nbfmt_version}
      - run: pip freeze --all
      - name: 'Lint Python code for style with Black'
        # You can run `black .` to fix all Black complaints.
        run: black --check --diff .
      - name: 'Lint YAML for gotchas with yamllint'
        # Use '# yamllint disable-line rule:foo' to suppress.
        run: yamllint -c docs/.yamllint docs docs/.yamllint
      - name: 'Lint Colab notebooks for formatting with nbfmt'
        run: git ls-files -z '*.ipynb' | xargs -0 python3 -m tensorflow_docs.tools.nbfmt --test

  lint-rust:
    runs-on: ubuntu-18.04
    strategy:
      matrix:
        rust_version: ['1.58.1']
        cargo_raze_version: ['0.13.0']
    steps:
      - uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e
      - name: 'Cache Cargo artifacts'
        uses: actions/cache@1a9e2138d905efd099035b49d8b7a3888c653ca8
        with:
          path: |
            tensorboard/data/server/target/
            # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            # Needed for installing binaries (`cargo-raze`) with cache
            ~/.cargo/.crates.toml
            ~/.cargo/.crates2.json
          key: lint-rust-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ matrix.cargo_raze_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }}
      - name: 'Install Rust toolchain'
        uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af
        with:
          toolchain: ${{ matrix.rust_version }}
          default: true
          components: rustfmt, clippy
      - name: 'Install cargo-raze'
        run: cargo install cargo-raze --version ${{ matrix.cargo_raze_version }}
      - name: 'Run Rustfmt'
        run: (cd tensorboard/data/server/ && cargo fmt -- --check)
      - name: 'Run Clippy'
        uses: actions-rs/clippy-check@b5b5f21f4797c02da247df37026fcd0a5024aa4d
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          args: --tests --manifest-path tensorboard/data/server/Cargo.toml
      - name: 'Check cargo-raze freshness'
        run: |
          rm -rf third_party/rust/
          (cd tensorboard/data/server/ && cargo fetch && cargo raze)
          git add .
          git diff --staged --exit-code

  lint-frontend:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e
      - uses: actions/setup-node@56899e050abffc08c2b3b61f3ec6a79a9dc3223d
        with:
          # default on setup-node@1 is v10.24.1.
          version: '12.21.0'
      - run: yarn install --ignore-engines
        # You can run `yarn fix-lint` to fix all Prettier complaints.
      - run: yarn lint
        # Make sure no tests are skipped with "focused" tests.
      - run: |
          ! git grep -E 'f(it|describe)\(' 'tensorboard/*_test.ts'
        # Make sure no one depends on Angular material and CDK directly. Please
        # import the indirection in //tensorboard/webapp/angular.
      - run: |
          ! git grep -E '"@npm//@angular/material"|"@npm//@angular/cdk"' 'tensorboard/*/BUILD' ':!tensorboard/webapp/BUILD' ':!tensorboard/webapp/angular/BUILD'
        # Cannot directly depend on d3 in webapp. Must depend on
        # `//tensorboard/webapp/third_party:d3` instead.
      - run: |
          ! git grep -E '"@npm//d3"|"@npm//@types/d3"' 'tensorboard/webapp/**/*BUILD' ':!tensorboard/webapp/third_party/**'
      - run: |
          ! git grep -E '"@npm_angular_bazel//:index.bzl"' 'tensorboard/**/BUILD'
      - run: |
          ! git grep -E 'mat-color|$mat-' 'tensorboard/**/*.scss'

  lint-misc: # build, protos, etc.
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e
      - name: 'Set up Buildifier'
        run: |
          ci/download_buildifier.sh "${BUILDTOOLS_VERSION}" "${BUILDIFIER_SHA256SUM}" ~/buildifier
          sudo mv ~/buildifier /usr/local/bin/buildifier
      - name: 'Set up Buildozer'
        run: |
          ci/download_buildozer.sh "${BUILDTOOLS_VERSION}" "${BUILDOZER_SHA256SUM}" ~/buildozer
          sudo mv ~/buildozer /usr/local/bin/buildozer
      - name: 'Lint BUILD files'
        # TODO(tensorboard-team): address all lint warnings and remove the exemption.
        run:
          git ls-files -z '*BUILD' third_party/js.bzl third_party/workspace.bzl WORKSPACE | xargs -0 buildifier --mode=check --lint=warn
          --warnings=-native-py,-native-java
      - run: ./tensorboard/tools/mirror_urls_test.sh
      - name: 'Lint for no py2 BUILD targets'
        # Use | to start a literal so YAML doesn't complain about the '!' character.
        run: |
          ! git grep 'python_version = "PY2"' '*BUILD'
      - name: 'No comments on licenses rule'
        # Assert buildozer error code for 'success, when no changes were made'.
        # https://github.com/bazelbuild/buildtools/blob/master/buildozer/README.md#error-code
        run: |
          buildozer '//tensorboard/...:%licenses' remove_comment && false || test $? = 3
      - name: clang-format lint
        uses: DoozyX/clang-format-lint-action@0138140a2adaafd3032a3ff37f66366fd7dc88e0
        with:
          source: ./tensorboard
          # Exclude tensorboard/compat because the source of truth is TensorFlow.
          exclude: ./tensorboard/compat/proto
          extensions: 'proto'
          clangFormatVersion: 9
      - run: ./tensorboard/tools/do_not_submit_test.sh
      - run: ./tensorboard/tools/license_test.sh
      - run: ./tensorboard/tools/whitespace_hygiene_test.py