diff --git a/.github/issue-labeler.yml b/.github/issue-labeler.yml
new file mode 100644
index 000000000..0821bc8fe
--- /dev/null
+++ b/.github/issue-labeler.yml
@@ -0,0 +1,2 @@
+needs_triage:
+  - '.*'
diff --git a/.github/labeler.yml b/.github/labeler.yml
deleted file mode 100644
index 75adb1d02..000000000
--- a/.github/labeler.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-needs_triage:
-    - '.*'
-    - '.*/*'
-    - '*'
-    - '*/*'
-
-docs:
-    - docs/*
-
-test:
-    - 'test/*'
diff --git a/.github/pr-labeler.yml b/.github/pr-labeler.yml
new file mode 100644
index 000000000..4364b3239
--- /dev/null
+++ b/.github/pr-labeler.yml
@@ -0,0 +1,11 @@
+needs_triage:
+  - '.*'
+  - '.*/*'
+  - '*'
+  - '*/*'
+
+docs:
+  - docs/*
+
+test:
+  - 'test/*'
diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml
index 7ded537a4..edc88c4ca 100644
--- a/.github/workflows/triage.yml
+++ b/.github/workflows/triage.yml
@@ -1,18 +1,12 @@
 name: Triage
 
-permissions:
-  contents: read
-  pull-requests: write
-  issues: write
-
 on:
   issues:
     types:
       - opened
+      - reopened
 
-  pull_request:
-    types:
-      - opened
+  pull_request_target:
 
 jobs:
   triage:
@@ -20,7 +14,20 @@ jobs:
     name: Label
 
     steps:
-      - name: Apply labels
+      - name: Label pull requests
         uses: actions/labeler@v3
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          configuration-path: .github/pr-labeler.yml
+        if: github.event_name == 'pull_request_target'
+
+      - name: Label issues
+        # This is the latest commit in the v2.4 branch. Waiting for a 2.4.1 or newer release.
+        # https://github.com/github/issue-labeler/issues/24
+        uses: github/issue-labeler@99b4c5dda477c65f4ef08486b73c5787e1e33601
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          not-before: 2021-12-07T07:00:00Z
+          configuration-path: .github/issue-labeler.yml
+          enable-versioned-regex: 0
+        if: github.event_name == 'issues'