diff --git a/README.md b/README.md
index e5531054..ec9e74fc 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,6 @@ For example:
 feature: ['feature/*', 'feat/*']
 fix: fix/*
 chore: chore/*
-fixed-branch: fixed-branch-name
 ```
 
 Then if a pull request is opened with the branch name `feature/218-add-emoji-support` the Action will automatically apply the `feature` label.
@@ -45,21 +44,6 @@ Then if a pull request is opened with the branch name `feature/218-add-emoji-sup
 
 You can use `*` as a wildcard for matching multiple branch names. See https://www.npmjs.com/package/matcher for more information about wildcard options.
 
-### Negative matcher
-
-You can also use `!` to add a label on all branches that does NOT match the pattern.
-
-Used in a list of patterns, then, the branch name will have to match all patterns.
-
-For example:
-
-```yml
-patch: ['update/*', '!update/sbt-*']
-no-release: update/sbt-*
-```
-
-In this scenario, a branch named `update/sbt-native-package` will only have the `no-release` label
-
 ### Default configuration
 
 When no configuration is provided, the following defaults will be used:
diff --git a/__tests__/fixtures/config.yml b/__tests__/fixtures/config.yml
index a9580659..236599f4 100644
--- a/__tests__/fixtures/config.yml
+++ b/__tests__/fixtures/config.yml
@@ -1,5 +1,5 @@
 '🎉 feature': ['feature/*', 'feat/*']
 fix: fix/*
 chore: chore/*
-release: ['release/*', '!release/skip-*']
+release: ['release/*', 'hotfix/*', '!release/skip-*']
 skip-release: release/skip-*
diff --git a/src/action.ts b/src/action.ts
index a2f46daf..54e63d38 100644
--- a/src/action.ts
+++ b/src/action.ts
@@ -3,11 +3,12 @@ import * as github from '@actions/github'
 import { Context } from '@actions/github/lib/context'
 import matcher from 'matcher'
 import getConfig, { Config } from './utils/config'
+import { arrayify } from './utils/arrayify'
 
 const defaultConfig = {
   feature: ['feature/*', 'feat/*'],
   fix: 'fix/*',
-  chore: 'chore/*'
+  chore: 'chore/*',
 }
 
 async function action(context: Context = github.context) {
@@ -30,7 +31,7 @@ async function action(context: Context = github.context) {
       await octokit.issues.addLabels({
         ...context.repo,
         number: context.payload.pull_request.number,
-        labels: labelsToAdd
+        labels: labelsToAdd,
       })
     }
   } catch (error) {
@@ -44,23 +45,18 @@ async function action(context: Context = github.context) {
 }
 
 function getLabelsToAdd(config: Config, branchName: string): string[] {
-  return Object.entries(config).reduce(
+  const labelsToAdd: string[] = []
 
-    (labels, [label, pattern]) => {
-      const patterns = Array.isArray(pattern) ? pattern : [pattern]
+  for (const label in config) {
+    const patterns = arrayify(config[label])
+    const matches = matcher([branchName], patterns)
 
-      if (
-        patterns.some(pattern => pattern.startsWith("!"))
-            ? patterns.every(pattern => matcher.isMatch(branchName, pattern))
-            : patterns.some(pattern => matcher.isMatch(branchName, pattern))
-      ) {
-        labels.push(label)
-      }
+    if (matches.length > 0) {
+      labelsToAdd.push(label)
+    }
+  }
 
-      return labels
-    },
-    [] as string[]
-  )
+  return labelsToAdd
 }
 
 export default action
diff --git a/src/utils/arrayify.ts b/src/utils/arrayify.ts
new file mode 100644
index 00000000..10f7bb63
--- /dev/null
+++ b/src/utils/arrayify.ts
@@ -0,0 +1,3 @@
+export function arrayify<T>(x: T | T[]): T[] {
+  return Array.isArray(x) ? x : [x]
+}