Merge pull request #215 from Hexiro/Hexiro-patch-2

Remove duplicate regexes, add test to spot duplicate regexes

pywhat/Data/regex.json

@@ -627,21 +627,6 @@
          "LinkedIn"
       ]
    },
-   {
-      "Name": "GitHub Access Token",
-      "Regex": "(?i)^([a-zA-Z0-9_-]*:[a-zA-Z0-9_-]+@github\\.com)$",
-      "plural_name": false,
-      "Description": null,
-      "Exploit": null,
-      "Rarity": 1,
-      "URL": null,
-      "Tags": [
-         "API Keys",
-         "Bug Bounty",
-         "Credentials",
-         "GitHub"
-      ]
-   },
    {
       "Name": "Stripe Restricted API Token",
       "Regex": "(?i)^(rk_live_[0-9a-zA-Z]{24})$",
@@ -672,21 +657,6 @@
          "Stripe"
       ]
    },
-   {
-      "Name": "Square Access Token",
-      "Regex": "(?i)^(sqOatp-[0-9A-Za-z\\-_]{22})$",
-      "plural_name": false,
-      "Description": null,
-      "Exploit": "Use the command below to verify that the access token is valid:\n  $ curl \"https://squareup.com/oauth2/revoke\" -d '{\"access_token\":\"[RANDOM_STRING]\",\"client_id\":\"[APP_ID]\"}'  -H \"Content-Type: application/json\" -H \"Authorization: Client TOKEN_HERE\"\n",
-      "Rarity": 1,
-      "URL": null,
-      "Tags": [
-         "API Keys",
-         "Bug Bounty",
-         "Credentials",
-         "Square"
-      ]
-   },
    {
       "Name": "Square OAuth Token",
       "Regex": "(?i)^(sq0csp-[ 0-9A-Za-z\\-_]{43})$",

tests/test_regex_database.py

@@ -70,3 +70,11 @@ def test_sorted_by_rarity():
     assert rarity_num == sorted(
         rarity_num, reverse=True
     ), "Regexes should be sorted by rarity in 'regex.json'. Regexes with rarity '1' are at the top of the file and '0' is at the bottom."
+
+
+def test_no_duplicate_regexes():
+    names = [regex["Name"] for regex in database]
+    duplicate_names = {name for name in names if names.count(name) > 1}
+    assert duplicate_names == set(), (
+        ", ".join(duplicate_names) + " present in 'regex.json' more than once."
+    )