diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 70c69cff9..758b3162d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -26,21 +26,12 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Load pip cache
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/test-requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Install dependencies
-        run: |
-          pip install --upgrade pip
-          pip install -r test-requirements.txt
-
       - name: Test typing_extensions
-        run: pytest typing_extensions/src_py3
+        run: |
+          # Be wary of running `pip install` here, since it becomes easy for us to
+          # accidentally pick up typing_extensions as installed by a dependency 
+          cd typing_extensions/src_py3
+          python -m unittest test_typing_extensions.py
 
   linting:
     name: Lint
diff --git a/test-requirements.txt b/test-requirements.txt
index e29654168..658ae0a52 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,4 +1,3 @@
 flake8
 flake8-bugbear
 flake8-pyi
-pytest