[WIP] Add Jukebox model

.gitattributes

@@ -1,3 +1,4 @@
 *.py	eol=lf
 *.rst	eol=lf
-*.md	eol=lf
+*.md	eol=lf
+*.mdx   eol=lf

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,5 +1,5 @@
 name: "\U0001F41B Bug Report"
-description: Submit a bug report to help us import transformers
+description: Submit a bug report to help us improve transformers
 labels: [ "bug" ]
 body:
   - type: textarea
@@ -118,4 +118,4 @@ body:
     attributes:
       label: Expected behavior
       description: "A clear and concise description of what you would expect to happen."
-      render: shell
+      render: shell

.github/ISSUE_TEMPLATE/config.yml

@@ -1,9 +1,12 @@
 blank_issues_enabled: true
 version: 2.1
 contact_links:
+  - name: Model checkpoints on the Hugging Face Hub
+    url: https://huggingface.co/models
+    about: Open a Pull request / Discussion related to a specific model checkpoint directly on the Hugging Face Hub
   - name: Website Related
     url: https://github.com/huggingface/hub-docs/issues
     about: Feature requests and bug reports related to the website
   - name: Forum
     url: https://discuss.huggingface.co/
-    about: General usage questions and community discussions
+    about: General usage questions and community discussions

.github/workflows/add-model-like.yml

@@ -27,14 +27,27 @@ jobs:
         id: cache
         with:
           path: ~/venv/
-          key: v2-tests_model_like-${{ hashFiles('setup.py') }}
+          key: v4-tests_model_like-${{ hashFiles('setup.py') }}
 
       - name: Create virtual environment on cache miss
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
           python -m venv ~/venv && . ~/venv/bin/activate
           pip install --upgrade pip!=21.3
-          pip install .[dev]
+          pip install -e .[dev]
+
+      - name: Check transformers location
+        # make `transformers` available as package (required since we use `-e` flag) and check it's indeed from the repo.
+        run: |
+          . ~/venv/bin/activate
+          python setup.py develop
+          transformer_loc=$(pip show transformers | grep "Location: " | cut -c11-)
+          transformer_repo_loc=$(pwd .)
+          if [ "$transformer_loc" != "$transformer_repo_loc/src" ]; then
+              echo "transformers is from $transformer_loc but it shoud be from $transformer_repo_loc/src."
+              echo "A fix is required. Stop testing."
+              exit 1
+          fi
 
       - name: Create model files
         run: |

.github/workflows/build-docker-images.yml

@@ -39,9 +39,35 @@ jobs:
           push: true
           tags: huggingface/transformers-all-latest-gpu
 
+  latest-with-torch-nightly-docker:
+    name: "Nightly PyTorch + Stable TensorFlow"
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Check out code
+        uses: actions/checkout@v2
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker/transformers-all-latest-gpu
+          build-args: |
+            REF=main
+            PYTORCH=pre
+          push: true
+          tags: huggingface/transformers-all-latest-torch-nightly-gpu
+
   latest-torch-deepspeed-docker:
     name: "Latest PyTorch + DeepSpeed"
-    needs: latest-docker
     runs-on: ubuntu-latest
     steps:
       -
@@ -66,6 +92,32 @@ jobs:
           push: true
           tags: huggingface/transformers-pytorch-deepspeed-latest-gpu
 
+  nightly-torch-deepspeed-docker:
+    name: "Nightly PyTorch + DeepSpeed"
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Check out code
+        uses: actions/checkout@v2
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker/transformers-pytorch-deepspeed-nightly-gpu
+          build-args: |
+            REF=main
+          push: true
+          tags: huggingface/transformers-pytorch-deepspeed-nightly-gpu
+
   doc-builder:
     name: "Doc builder"
     runs-on: ubuntu-latest
@@ -93,7 +145,6 @@ jobs:
   latest-pytorch:
     name: "Latest PyTorch [dev]"
     runs-on: ubuntu-latest
-    needs: latest-torch-deepspeed-docker
     steps:
       -
         name: Set up Docker Buildx
@@ -118,7 +169,6 @@ jobs:
           tags: huggingface/transformers-pytorch-gpu
 
   latest-tensorflow:
-    needs: latest-pytorch
     name: "Latest TensorFlow [dev]"
     runs-on: ubuntu-latest
     steps:

.github/workflows/build_documentation.yml

@@ -15,6 +15,6 @@ jobs:
       commit_sha: ${{ github.sha }}
       package: transformers
       notebook_folder: transformers_doc
-      languages: en es
+      languages: en es it pt
     secrets:
       token: ${{ secrets.HUGGINGFACE_PUSH }}

.github/workflows/build_pr_documentation.yml

@@ -14,4 +14,4 @@ jobs:
       commit_sha: ${{ github.event.pull_request.head.sha }}
       pr_number: ${{ github.event.number }}
       package: transformers
-      languages: en es
+      languages: en es it pt

.github/workflows/doctests.yml

@@ -32,9 +32,7 @@ jobs:
 
       - name: GPU visibility
         run: |
-          utils/print_env_pt.py
-          TF_CPP_MIN_LOG_LEVEL=3 python3 -c "import tensorflow as tf; print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))"
-          TF_CPP_MIN_LOG_LEVEL=3 python3 -c "import tensorflow as tf; print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))"
+          python3 utils/print_env.py
 
       - name: Prepare files for doctests
         run: |

.github/workflows/model-templates.yml

@@ -1,28 +1,16 @@
 name: Model templates runner
 
 on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - "src/**"
-      - "tests/**"
-      - ".github/**"
-      - "templates/**"
-    types: [assigned, opened, synchronize, reopened]
+  repository_dispatch:
+  schedule:
+    - cron: "0 2 * * *"
 
 jobs:
   run_tests_templates:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v1
-
-      - name: Install Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.6
+        uses: actions/checkout@v2
 
       - name: Install dependencies
         run: |
@@ -33,14 +21,27 @@ jobs:
         id: cache
         with:
           path: ~/venv/
-          key: v2-tests_templates-${{ hashFiles('setup.py') }}
+          key: v4-tests_templates-${{ hashFiles('setup.py') }}
 
       - name: Create virtual environment on cache miss
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
           python -m venv ~/venv && . ~/venv/bin/activate
           pip install --upgrade pip!=21.3
-          pip install .[dev]
+          pip install -e .[dev]
+
+      - name: Check transformers location
+        # make `transformers` available as package (required since we use `-e` flag) and check it's indeed from the repo.
+        run: |
+          . ~/venv/bin/activate
+          python setup.py develop
+          transformer_loc=$(pip show transformers | grep "Location: " | cut -c11-)
+          transformer_repo_loc=$(pwd .)
+          if [ "$transformer_loc" != "$transformer_repo_loc/src" ]; then
+              echo "transformers is from $transformer_loc but it shoud be from $transformer_repo_loc/src."
+              echo "A fix is required. Stop testing."
+              exit 1
+          fi
 
       - name: Create model files
         run: |
@@ -65,7 +66,6 @@ jobs:
 
       - name: Run style changes
         run: |
-          git fetch origin main:main
           . ~/venv/bin/activate
           make style && make quality && make repo-consistency