diff --git a/CHANGELOG.md b/CHANGELOG.md
index 43c540b..d2c39cc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,10 @@
   for dependency management. There apparently is an issue no-binary installs of
   packages build with Poetry. This fixes
   [#148](https://github.com/sybrenstuvel/python-rsa/issues/148)
+- Limited SHA3 support to those Python versions (3.6+) that support it natively.
+  The third-party library that adds support for this to Python 3.5 is a binary
+  package, and thus breaks the pure-Python nature of Python-RSA.
+  This should fix [#147](https://github.com/sybrenstuvel/python-rsa/issues/147).
 
 
 ## Version 4.1 - released 2020-06-10
diff --git a/Pipfile b/Pipfile
index 31a1ada..a6f846c 100644
--- a/Pipfile
+++ b/Pipfile
@@ -5,7 +5,6 @@ name = "pypi"
 
 [packages]
 "pyasn1" = ">=0.1.3"
-"pysha3" = {version = "~=1.0, >=1.0",markers = "python_version < '3.6'"}
 
 [dev-packages]
 coveralls = "~=1.8, >=1.8"
diff --git a/Pipfile.lock b/Pipfile.lock
index 77a2030..702edae 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "9f3ee557d56ac9a111d2af3bd82a9c8cfc3969723398e26b95c565ea868f2735"
+            "sha256": "4df253faa2a1f6d6665fddc4c13f5e278a4127c27d7b76e59607a8154f96b1ab"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -34,33 +34,6 @@
             ],
             "index": "pypi",
             "version": "==0.4.8"
-        },
-        "pysha3": {
-            "hashes": [
-                "sha256:0060a66be16665d90c432f55a0ba1f6480590cfb7d2ad389e688a399183474f0",
-                "sha256:11a2ba7a2e1d9669d0052fc8fb30f5661caed5512586ecbeeaf6bf9478ab5c48",
-                "sha256:386998ee83e313b6911327174e088021f9f2061cbfa1651b97629b761e9ef5c4",
-                "sha256:41be70b06c8775a9e4d4eeb52f2f6a3f356f17539a54eac61f43a29e42fd453d",
-                "sha256:4416f16b0f1605c25f627966f76873e432971824778b369bd9ce1bb63d6566d9",
-                "sha256:571a246308a7b63f15f5aa9651f99cf30f2a6acba18eddf28f1510935968b603",
-                "sha256:59111c08b8f34495575d12e5f2ce3bafb98bea470bc81e70c8b6df99aef0dd2f",
-                "sha256:5ec8da7c5c70a53b5fa99094af3ba8d343955b212bc346a0d25f6ff75853999f",
-                "sha256:684cb01d87ed6ff466c135f1c83e7e4042d0fc668fa20619f581e6add1d38d77",
-                "sha256:68c3a60a39f9179b263d29e221c1bd6e01353178b14323c39cc70593c30f21c5",
-                "sha256:6e6a84efb7856f5d760ee55cd2b446972cb7b835676065f6c4f694913ea8f8d9",
-                "sha256:827b308dc025efe9b6b7bae36c2e09ed0118a81f792d888548188e97b9bf9a3d",
-                "sha256:93abd775dac570cb9951c4e423bcb2bc6303a9d1dc0dc2b7afa2dd401d195b24",
-                "sha256:9c778fa8b161dc9348dc5cc361e94d54aa5ff18413788f4641f6600d4893a608",
-                "sha256:9fdd28884c5d0b4edfed269b12badfa07f1c89dbc5c9c66dd279833894a9896b",
-                "sha256:c7c2adcc43836223680ebdf91f1d3373543dc32747c182c8ca2e02d1b69ce030",
-                "sha256:c93a2676e6588abcfaecb73eb14485c81c63b94fca2000a811a7b4fb5937b8e8",
-                "sha256:cd5c961b603bd2e6c2b5ef9976f3238a561c58569945d4165efb9b9383b050ef",
-                "sha256:f9046d59b3e72aa84f6dae83a040bd1184ebd7fef4e822d38186a8158c89e3cf",
-                "sha256:fd7e66999060d079e9c0e8893e78d8017dad4f59721f6fe0be6307cd32127a07",
-                "sha256:fe988e73f2ce6d947220624f04d467faf05f1bbdbc64b0a201296bb3af92739e"
-            ],
-            "markers": "python_version < '3.6'",
-            "version": "==1.0.2"
         }
     },
     "develop": {
diff --git a/rsa/pkcs1.py b/rsa/pkcs1.py
index 408bc5b..57b0276 100644
--- a/rsa/pkcs1.py
+++ b/rsa/pkcs1.py
@@ -33,12 +33,6 @@
 
 from . import common, transform, core, key
 
-if sys.version_info < (3, 6):
-    # Python 3.6 and newer have SHA-3 support. For Python 3.5 we need a third party library.
-    # This library monkey-patches the hashlib module so that it looks like Python actually
-    # supports SHA-3 natively.
-    import sha3  # noqa: F401
-
 # ASN.1 codes that describe the hash algorithm used.
 HASH_ASN1 = {
     'MD5': b'\x30\x20\x30\x0c\x06\x08\x2a\x86\x48\x86\xf7\x0d\x02\x05\x05\x00\x04\x10',
@@ -47,9 +41,6 @@
     'SHA-256': b'\x30\x31\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20',
     'SHA-384': b'\x30\x41\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x02\x05\x00\x04\x30',
     'SHA-512': b'\x30\x51\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x03\x05\x00\x04\x40',
-    'SHA3-256': b'\x30\x31\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x08\x05\x00\x04\x20',
-    'SHA3-384': b'\x30\x41\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x09\x05\x00\x04\x30',
-    'SHA3-512': b'\x30\x51\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x0a\x05\x00\x04\x40',
 }
 
 HASH_METHODS = {
@@ -59,12 +50,24 @@
     'SHA-256': hashlib.sha256,
     'SHA-384': hashlib.sha384,
     'SHA-512': hashlib.sha512,
-    'SHA3-256': hashlib.sha3_256,
-    'SHA3-384': hashlib.sha3_384,
-    'SHA3-512': hashlib.sha3_512,
 }
 
 
+if sys.version_info >= (3, 6):
+    # Python 3.6 introduced SHA3 support.
+    HASH_ASN1.update({
+        'SHA3-256': b'\x30\x31\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x08\x05\x00\x04\x20',
+        'SHA3-384': b'\x30\x41\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x09\x05\x00\x04\x30',
+        'SHA3-512': b'\x30\x51\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x0a\x05\x00\x04\x40',
+    })
+
+    HASH_METHODS.update({
+        'SHA3-256': hashlib.sha3_256,
+        'SHA3-384': hashlib.sha3_384,
+        'SHA3-512': hashlib.sha3_512,
+    })
+
+
 class CryptoError(Exception):
     """Base class for all exceptions in this module."""
 
diff --git a/tests/test_pkcs1.py b/tests/test_pkcs1.py
index 702ce2d..f7baf7f 100644
--- a/tests/test_pkcs1.py
+++ b/tests/test_pkcs1.py
@@ -15,6 +15,7 @@
 """Tests string operations."""
 
 import struct
+import sys
 import unittest
 
 import rsa
@@ -101,6 +102,12 @@ def test_sign_verify(self):
         signature = pkcs1.sign(message, self.priv, 'SHA-256')
         self.assertEqual('SHA-256', pkcs1.verify(message, signature, self.pub))
 
+
+    @unittest.skipIf(sys.version_info < (3, 6), "SHA3 requires Python 3.6+")
+    def test_sign_verify_sha3(self):
+        """Test happy flow of sign and verify with SHA3-256"""
+
+        message = b'je moeder'
         signature = pkcs1.sign(message, self.priv, 'SHA3-256')
         self.assertEqual('SHA3-256', pkcs1.verify(message, signature, self.pub))