diff --git a/README.md b/README.md
index 1b64d130..ada93a84 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,6 @@ For example:
 feature: ['feature/*', 'feat/*']
 fix: fix/*
 chore: chore/*
-fixed-branch: fixed-branch-name
 ```
 
 Then if a pull request is opened with the branch name `feature/218-add-emoji-support` the Action will automatically apply the `feature` label.
diff --git a/__tests__/action.test.ts b/__tests__/action.test.ts
index 475f150d..bfc5e125 100644
--- a/__tests__/action.test.ts
+++ b/__tests__/action.test.ts
@@ -76,6 +76,22 @@ describe('pr-labeler-action', () => {
     expect.assertions(1)
   })
 
+  it("adds only one label if the branch matches a negative pattern", async () => {
+    nock('https://api.github.com')
+        .get('/repos/Codertocat/Hello-World/contents/.github/pr-labeler.yml?ref=release%2Fskip-this-one')
+        .reply(200, configFixture())
+        .post('/repos/Codertocat/Hello-World/issues/1/labels', body => {
+          expect(body).toMatchObject({
+            labels: ['skip-release']
+          })
+          return true
+        })
+        .reply(200)
+
+    await action(new MockContext(pullRequestOpenedFixture({ ref: 'release/skip-this-one' })))
+    expect.assertions(1)
+  })
+
   it("adds no labels if the branch doesn't match any patterns", async () => {
     nock('https://api.github.com')
       .get('/repos/Codertocat/Hello-World/contents/.github/pr-labeler.yml?ref=hello_world')
diff --git a/__tests__/fixtures/config.yml b/__tests__/fixtures/config.yml
index 04574e6b..236599f4 100644
--- a/__tests__/fixtures/config.yml
+++ b/__tests__/fixtures/config.yml
@@ -1,4 +1,5 @@
 '🎉 feature': ['feature/*', 'feat/*']
 fix: fix/*
 chore: chore/*
-release: release/*
+release: ['release/*', 'hotfix/*', '!release/skip-*']
+skip-release: release/skip-*
diff --git a/src/action.ts b/src/action.ts
index 09a6b19d..54e63d38 100644
--- a/src/action.ts
+++ b/src/action.ts
@@ -3,11 +3,12 @@ import * as github from '@actions/github'
 import { Context } from '@actions/github/lib/context'
 import matcher from 'matcher'
 import getConfig, { Config } from './utils/config'
+import { arrayify } from './utils/arrayify'
 
 const defaultConfig = {
   feature: ['feature/*', 'feat/*'],
   fix: 'fix/*',
-  chore: 'chore/*'
+  chore: 'chore/*',
 }
 
 async function action(context: Context = github.context) {
@@ -30,7 +31,7 @@ async function action(context: Context = github.context) {
       await octokit.issues.addLabels({
         ...context.repo,
         number: context.payload.pull_request.number,
-        labels: labelsToAdd
+        labels: labelsToAdd,
       })
     }
   } catch (error) {
@@ -44,20 +45,18 @@ async function action(context: Context = github.context) {
 }
 
 function getLabelsToAdd(config: Config, branchName: string): string[] {
-  return Object.entries(config).reduce(
-    (labels, [label, patterns]) => {
-      if (
-        Array.isArray(patterns)
-          ? patterns.some(pattern => matcher.isMatch(branchName, pattern))
-          : matcher.isMatch(branchName, patterns)
-      ) {
-        labels.push(label)
-      }
-
-      return labels
-    },
-    [] as string[]
-  )
+  const labelsToAdd: string[] = []
+
+  for (const label in config) {
+    const patterns = arrayify(config[label])
+    const matches = matcher([branchName], patterns)
+
+    if (matches.length > 0) {
+      labelsToAdd.push(label)
+    }
+  }
+
+  return labelsToAdd
 }
 
 export default action
diff --git a/src/utils/arrayify.ts b/src/utils/arrayify.ts
new file mode 100644
index 00000000..10f7bb63
--- /dev/null
+++ b/src/utils/arrayify.ts
@@ -0,0 +1,3 @@
+export function arrayify<T>(x: T | T[]): T[] {
+  return Array.isArray(x) ? x : [x]
+}