Split CI into Ubuntu 16.04 and Ubuntu 20.04 runs (#579)

* Bump mxnet from 1.6.0 to 1.7.0.post1 Bumps [mxnet](https://github.com/apache/incubator-mxnet) from 1.6.0 to 1.7.0.post1. - [Release notes](https://github.com/apache/incubator-mxnet/releases) - [Changelog](https://github.com/apache/incubator-mxnet/blob/master/NEWS.md) - [Commits](https://github.com/apache/incubator-mxnet/commits) Signed-off-by: dependabot[bot] <support@github.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * CI updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Test run_test.sh Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Test run_test.sh Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Test run_test.sh Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Test run_test.sh Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Test run_test.sh Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Test run_test.sh Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Test run_test.sh Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates for mxnet 1.7 Signed-off-by: Beat Buesser <beat.buesser@ie.ibm.com> * Updates Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Beat Buesser <49047826+beat-buesser@users.noreply.github.com> Co-authored-by: Beat Buesser <beat.buesser@ie.ibm.com>
Trusted-AI · Mar 17, 2021 · a160861 · a160861
1 parent 71aece3
commit a160861
Show file tree

Hide file tree

Showing 3 changed files with 121 additions and 27 deletions.
diff --git a/.github/workflows/ci-mxnet.yml b/.github/workflows/ci-mxnet.yml
@@ -0,0 +1,80 @@
+name: Continuous Integration
+on:
+  # Run on manual trigger
+  workflow_dispatch:
+
+  # Run on pull requests
+  pull_request:
+    paths-ignore:
+      - '*.md'
+
+  # Run when pushing to main or dev branches
+  push:
+    branches:
+      - main
+      - dev*
+
+  # Run scheduled CI flow daily
+  schedule:
+    - cron: '0 8 * * 0'
+
+jobs:
+  test:
+    runs-on: ubuntu-16.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: mxnet (Python 3.7)
+            framework: mxnet
+            python: 3.7
+          - name: legacy (TensorFlow 2.3.2 Keras 2.4.3  PyTorch 1.7.1 scikit-learn 0.22.2 Python 3.7)
+            framework: legacy
+            python: 3.7
+            tensorflow: 2.3.2
+            keras: 2.4.3
+            scikit-learn: 0.22.2
+            torch: 1.7.1+cpu
+            torchvision: 0.8.2+cpu
+            torchaudio: 0.7.2
+          - name: legacy (TensorFlow 2.4.1 Keras 2.4.3 PyTorch 1.7.1 scikit-learn 0.23.2 Python 3.7)
+            framework: legacy
+            python: 3.7
+            tensorflow: 2.4.1
+            keras: 2.4.3
+            torch: 1.7.1+cpu
+            torchvision: 0.8.2+cpu
+            torchaudio: 0.7.2
+            scikit-learn: 0.23.2
+
+    name: Run ${{ matrix.name }} Tests
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -y -q install ffmpeg libavcodec-extra
+          python -m pip install --upgrade pip setuptools wheel
+          pip install tensorflow==2.4.1
+          pip install keras==2.4.3
+          pip3 install -q -r requirements.txt
+          pip list
+      - name: Pre-install legacy
+        if: ${{ matrix.framework == 'legacy' }}
+        run: |
+          pip install tensorflow==${{ matrix.tensorflow }}
+          pip install keras==${{ matrix.keras }}
+          pip install scikit-learn==${{ matrix.scikit-learn }}
+          pip install torch==${{ matrix.torch }} -f https://download.pytorch.org/whl/torch_stable.html
+          pip install torchvision==${{ matrix.torchvision }} -f https://download.pytorch.org/whl/torch_stable.html
+          pip install torchaudio==${{ matrix.torchaudio }} -f https://download.pytorch.org/whl/torch_stable.html
+          pip list
+      - name: Run ${{ matrix.name }} Tests
+        run: ./run_tests.sh ${{ matrix.framework }}
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -62,6 +62,9 @@ jobs:
           - name: PyTorch (Python 3.7)
             framework: pytorch
             python: 3.7
+            torch: 1.7.1+cpu
+            torchvision: 0.8.2+cpu
+            torchaudio: 0.7.2
           - name: scikit-learn 0.22.2 (Python 3.7)
             framework: scikitlearn
             scikit-learn: 0.22.2
@@ -70,28 +73,27 @@ jobs:
             framework: scikitlearn
             scikit-learn: 0.23.2
             python: 3.8
-          - name: mxnet (Python 3.7)
-            framework: mxnet
-            python: 3.7
-          - name: TensorFlow+Lingvo 2.1.0v1 (Keras 2.3.1 Python 3.6)
-            framework: tensorflow2v1
-            python: 3.6
-            tensorflow: 2.1.0
-            tf_version: v2
-            keras: 2.3.1
-            lingvo: 0.6.4
-          - name: legacy (TensorFlow 2.2.1 Keras 2.3.1 scikit-learn 0.22.2 Python 3.7)
-            framework: legacy
-            python: 3.7
-            tensorflow: 2.2.1
-            keras: 2.3.1
-            scikit-learn: 0.22.2
-          - name: legacy (TensorFlow 2.3.2 Keras 2.4.3 scikit-learn 0.23.2 Python 3.8)
-            framework: legacy
-            python: 3.8
-            tensorflow: 2.3.2
-            keras: 2.4.3
-            scikit-learn: 0.23.2
+#          - name: mxnet (Python 3.7)
+#            framework: mxnet
+#            python: 3.7
+#          - name: legacy (TensorFlow 2.3.2 Keras 2.4.3  PyTorch 1.7.1 scikit-learn 0.22.2 Python 3.7)
+#            framework: legacy
+#            python: 3.7
+#            tensorflow: 2.3.2
+#            keras: 2.4.3
+#            scikit-learn: 0.22.2
+#            torch: 1.7.1+cpu
+#            torchvision: 0.8.2+cpu
+#            torchaudio: 0.7.2
+#          - name: legacy (TensorFlow 2.4.1 Keras 2.4.3 PyTorch 1.7.1 scikit-learn 0.23.2 Python 3.7)
+#            framework: legacy
+#            python: 3.7
+#            tensorflow: 2.4.1
+#            keras: 2.4.3
+#            torch: 1.7.1+cpu
+#            torchvision: 0.8.2+cpu
+#            torchaudio: 0.7.2
+#            scikit-learn: 0.23.2
 
     name: Run ${{ matrix.name }} Tests
     steps:
@@ -106,6 +108,8 @@ jobs:
           sudo apt-get update
           sudo apt-get -y -q install ffmpeg libavcodec-extra
           python -m pip install --upgrade pip setuptools wheel
+          pip install tensorflow==2.4.1
+          pip install keras==2.4.3
           pip3 install -q -r requirements.txt
           pip list
       - name: Pre-install legacy
@@ -114,6 +118,9 @@ jobs:
           pip install tensorflow==${{ matrix.tensorflow }}
           pip install keras==${{ matrix.keras }}
           pip install scikit-learn==${{ matrix.scikit-learn }}
+          pip install torch==${{ matrix.torch }} -f https://download.pytorch.org/whl/torch_stable.html
+          pip install torchvision==${{ matrix.torchvision }} -f https://download.pytorch.org/whl/torch_stable.html
+          pip install torchaudio==${{ matrix.torchaudio }} -f https://download.pytorch.org/whl/torch_stable.html
           pip list
       - name: Pre-install tensorflow
         if: ${{ matrix.framework == 'tensorflow' || matrix.framework == 'keras' || matrix.framework == 'kerastf' || matrix.framework == 'tensorflow2v1' }}
@@ -126,6 +133,12 @@ jobs:
         run: |
           pip install scikit-learn==${{ matrix.scikit-learn }}
           pip list
+      - name: Pre-install torch
+        if: ${{ matrix.framework == 'pytorch' }}
+        run: |
+          pip install torch==${{ matrix.torch }} -f https://download.pytorch.org/whl/torch_stable.html
+          pip install torchvision==${{ matrix.torchvision }} -f https://download.pytorch.org/whl/torch_stable.html
+          pip install torchaudio==${{ matrix.torchaudio }} -f https://download.pytorch.org/whl/torch_stable.html
       - name: Pre-install Lingvo ASR
         if: ${{ matrix.tensorflow == '2.1.0' }}
         run: |

diff --git a/requirements.txt b/requirements.txt
@@ -21,12 +21,13 @@ h5py==2.10.0
 # supported versions: (tensorflow==2.2.0 with keras==2.3.1) or (tensorflow==1.15.4 with keras==2.2.5)
 tensorflow>=1.15.4
 keras>=2.2.5
-tensorflow-addons~=0.11.0
+tensorflow-addons~=0.11.2
+# using mxnet-native for reproducible test results on CI machines without Intel Architecture Processors, but mxnet is fully supported by ART
 mxnet==1.6.0
 --find-links https://download.pytorch.org/whl/torch_stable.html
-torch==1.8.0+cpu
-torchaudio~=0.8.0
-torchvision~=0.9.0
+torch==1.7.1+cpu -f https://download.pytorch.org/whl/torch_stable.html
+torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html
+torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html
 catboost==0.24.4
 GPy==1.9.9
 lightgbm==3.0.0