Add support for external modules to web3 instance

- added web3.attach_module() to attach a single module to a web3 instance e.g. w3.attach_module('module1', ModuleClass1)
- added support for initializing the instance with external modules e.g. w3 = Web3(..., external_modules={'module1': ModuleClass1})
- modules can be passed in without needing to be a tuple or list if they don't contain submodules i.e. pass in the class by itself instead a tuple with 1 argument, (ModuleClass1,)
- added tests for the above

docs/providers.rst

@@ -387,7 +387,7 @@ AsyncHTTPProvider
         >>> from web3.net import AsyncNet
 
         >>> w3 = Web3(AsyncHTTPProvider("http://127.0.0.1:8545"),
-        ...           modules={'eth': (AsyncEth,), 'net': (AsyncNet,)},
+        ...           modules={'eth': AsyncEth, 'net': AsyncNet},
         ...           middlewares=[])  # See supported middleware section below for middleware options
 
     Under the hood, the ``AsyncHTTPProvider`` uses the python

newsfragments/2288.feature.rst

@@ -0,0 +1 @@
+Support for attaching external modules to the ``Web3`` instance when instantiating the ``Web3`` instance, via the ``external_modules`` argument, or via the new ``attach_module()`` method

tests/core/conftest.py

@@ -0,0 +1,41 @@
+import pytest
+
+from web3.module import (
+    Module,
+)
+
+
+@pytest.fixture(scope='module')
+def module1():
+    class Module1(Module):
+        a = 'a'
+
+        @property
+        def b(self):
+            return 'b'
+    return Module1
+
+
+@pytest.fixture(scope='module')
+def module2():
+    class Module2(Module):
+        c = 'c'
+
+        @staticmethod
+        def d():
+            return 'd'
+    return Module2
+
+
+@pytest.fixture(scope='module')
+def module3():
+    class Module3(Module):
+        e = 'e'
+    return Module3
+
+
+@pytest.fixture(scope='module')
+def module4():
+    class Module4(Module):
+        f = 'f'
+    return Module4

tests/core/method-class/test_method.py

@@ -265,7 +265,7 @@ class FakeModule(Module):
 def dummy_w3():
     return Web3(
         EthereumTesterProvider(),
-        modules={'fake': (FakeModule,)},
+        modules={'fake': FakeModule},
         middlewares=[])
 
 

tests/core/method-class/test_result_formatters.py

@@ -46,7 +46,7 @@ def dummy_w3():
     w3 = Web3(
         DummyProvider(),
         middlewares=[result_middleware],
-        modules={"module": (ModuleForTest,)})
+        modules={"module": ModuleForTest})
     return w3
 
 

tests/core/utilities/test_attach_modules.py

@@ -1,5 +1,9 @@
 import pytest
 
+from eth_utils import (
+    is_integer,
+)
+
 from web3 import Web3
 from web3._utils.module import (
     attach_modules,
@@ -37,10 +41,10 @@ def unlock_account(self):
 def test_attach_modules():
     mods = {
         "geth": (MockGeth, {
-            "personal": (MockGethPersonal,),
-            "admin": (MockGethAdmin,),
+            "personal": MockGethPersonal,
+            "admin": MockGethAdmin,
         }),
-        "eth": (MockEth,),
+        "eth": MockEth,
     }
     w3 = Web3(EthereumTesterProvider(), modules={})
     attach_modules(w3, mods)
@@ -51,10 +55,10 @@ def test_attach_modules():
 
 def test_attach_modules_multiple_levels_deep():
     mods = {
-        "eth": (MockEth,),
+        "eth": MockEth,
         "geth": (MockGeth, {
             "personal": (MockGethPersonal, {
-                "admin": (MockGethAdmin,),
+                "admin": MockGethAdmin,
             }),
         }),
     }
@@ -76,9 +80,45 @@ def test_attach_modules_with_wrong_module_format():
 
 def test_attach_modules_with_existing_modules():
     mods = {
-        "eth": (MockEth,),
+        "eth": MockEth,
     }
     w3 = Web3(EthereumTesterProvider, modules=mods)
     with pytest.raises(AttributeError,
                        match="The web3 object already has an attribute with that name"):
         attach_modules(w3, mods)
+
+
+def test_attach_external_modules_multiple_levels(module1, module2, module3, module4):
+    w3 = Web3(
+        EthereumTesterProvider(),
+        external_modules=({
+            'module1': module1,
+            'module2': (module2, {
+                'submodule1': (module3, {
+                    'submodule2': module4,
+                }),
+            })
+        })
+    )
+
+    assert w3.isConnected()
+
+    # assert instantiated with default modules
+    assert hasattr(w3, 'geth')
+    assert hasattr(w3, 'eth')
+    assert is_integer(w3.eth.chain_id)
+
+    # assert instantiated with module1
+    assert hasattr(w3, 'module1')
+    assert w3.module1.a == 'a'
+    assert w3.module1.b == 'b'
+
+    # assert instantiated with module2 + submodules
+    assert hasattr(w3, 'module2')
+    assert w3.module2.c == 'c'
+    assert w3.module2.d() == 'd'
+
+    assert hasattr(w3.module2, 'submodule1')
+    assert w3.module2.submodule1.e == 'e'
+    assert hasattr(w3.module2.submodule1, 'submodule2')
+    assert w3.module2.submodule1.submodule2.f == 'f'

tests/core/web3-module/test_attach_module.py

@@ -0,0 +1,35 @@
+from eth_utils import (
+    is_integer,
+)
+
+
+def test_attach_module(web3, module1, module2, module3, module4):
+    web3.attach_module('module1', module1)
+    web3.attach_module(
+        'module2',
+        (module2, {
+            'submodule1': (module3, {
+                'submodule2': module4,
+            })
+        })
+    )
+
+    # assert module1 attached
+    assert hasattr(web3, 'module1')
+    assert web3.module1.a == 'a'
+    assert web3.module1.b == 'b'
+
+    # assert module2 + submodules attached
+    assert hasattr(web3, 'module2')
+    assert web3.module2.c == 'c'
+    assert web3.module2.d() == 'd'
+
+    assert hasattr(web3.module2, 'submodule1')
+    assert web3.module2.submodule1.e == 'e'
+    assert hasattr(web3.module2.submodule1, 'submodule2')
+    assert web3.module2.submodule1.submodule2.f == 'f'
+
+    # assert default modules intact
+    assert hasattr(web3, 'geth')
+    assert hasattr(web3, 'eth')
+    assert is_integer(web3.eth.chain_id)

web3/_utils/module.py

@@ -18,11 +18,13 @@
 
 def attach_modules(
     parent_module: Union["Web3", "Module"],
-    module_definitions: Dict[str, Sequence[Any]],
+    module_definitions: Dict[str, Any],
     w3: Optional[Union["Web3", "Module"]] = None
 ) -> None:
     for module_name, module_info in module_definitions.items():
-        module_class = module_info[0]
+        module_info_is_list_like = isinstance(module_info, Sequence)
+
+        module_class = module_info[0] if module_info_is_list_like else module_info
 
         if hasattr(parent_module, module_name):
             raise AttributeError(
@@ -36,9 +38,10 @@ def attach_modules(
         else:
             setattr(parent_module, module_name, module_class(w3))
 
-        if len(module_info) == 2:
-            submodule_definitions = module_info[1]
-            module = getattr(parent_module, module_name)
-            attach_modules(module, submodule_definitions, w3)
-        elif len(module_info) != 1:
-            raise ValidationError("Module definitions can only have 1 or 2 elements.")
+        if module_info_is_list_like:
+            if len(module_info) == 2:
+                submodule_definitions = module_info[1]
+                module = getattr(parent_module, module_name)
+                attach_modules(module, submodule_definitions, w3)
+            elif len(module_info) != 1:
+                raise ValidationError("Module definitions can only have 1 or 2 elements.")

web3/eth.py

@@ -458,7 +458,7 @@ async def call(
         return await self._call(transaction, block_identifier, state_override)
 
 
-class Eth(BaseEth, Module):
+class Eth(BaseEth):
     account = Account()
     defaultContractFactory: Type[Union[Contract, ConciseContract, ContractCaller]] = Contract  # noqa: E704,E501
     iban = Iban

web3/main.py

@@ -28,6 +28,7 @@
     List,
     Optional,
     Sequence,
+    Type,
     TYPE_CHECKING,
     Union,
     cast,
@@ -86,6 +87,9 @@
 from web3.manager import (
     RequestManager as DefaultRequestManager,
 )
+from web3.module import (
+    Module,
+)
 from web3.net import (
     AsyncNet,
     Net,
@@ -128,21 +132,21 @@
     from web3.pm import PM  # noqa: F401
 
 
-def get_default_modules() -> Dict[str, Sequence[Any]]:
+def get_default_modules() -> Dict[str, Union[Type[Module], Sequence[Any]]]:
     return {
-        "eth": (Eth,),
-        "net": (Net,),
-        "version": (Version,),
+        "eth": Eth,
+        "net": Net,
+        "version": Version,
         "parity": (Parity, {
-            "personal": (ParityPersonal,),
+            "personal": ParityPersonal,
         }),
         "geth": (Geth, {
-            "admin": (GethAdmin,),
-            "miner": (GethMiner,),
-            "personal": (GethPersonal,),
-            "txpool": (GethTxPool,),
+            "admin": GethAdmin,
+            "miner": GethMiner,
+            "personal": GethPersonal,
+            "txpool": GethTxPool,
         }),
-        "testing": (Testing,),
+        "testing": Testing,
     }
 
 
@@ -232,7 +236,8 @@ def __init__(
         self,
         provider: Optional[BaseProvider] = None,
         middlewares: Optional[Sequence[Any]] = None,
-        modules: Optional[Dict[str, Sequence[Any]]] = None,
+        modules: Optional[Dict[str, Union[Type[Module], Sequence[Any]]]] = None,
+        external_modules: Optional[Dict[str, Union[Type[Module], Sequence[Any]]]] = None,
         ens: ENS = cast(ENS, empty)
     ) -> None:
         self.manager = self.RequestManager(self, provider, middlewares)
@@ -245,6 +250,9 @@ def __init__(
 
         attach_modules(self, modules)
 
+        if external_modules is not None:
+            attach_modules(self, external_modules)
+
         self.ens = ens
 
     @property
@@ -323,6 +331,35 @@ def solidityKeccak(cls, abi_types: List[TypeStr], values: List[Any]) -> bytes:
         )))
         return cls.keccak(hexstr=hex_string)
 
+    def attach_module(
+        self,
+        module_name: str,
+        module: Union[Type[Module], Sequence[Any]]
+    ) -> None:
+        """
+        Attach a module to the `Web3` instance. Modules should inherit from the `web3.module.Module`
+        class.
+
+        Attaching a simple module:
+
+            >>> w3.attach_module('module1', ModuleClass1)
+            >>> w3.module1.return_one
+            1
+
+        Attaching a module with submodules:
+
+            >>> w3.attach_module(
+            ...     'module2',
+            ...     (ModuleClass2, {
+            ...         'submodule1': SubModuleClass1,
+            ...         'submodule2': SubModuleClass2,
+            ...     })
+            ... )
+            >>> w3.module2.submodule1.return_zero
+            0
+        """
+        attach_modules(self, {module_name: module})
+
     def isConnected(self) -> bool:
         return self.provider.isConnected()
 

web3/module.py

@@ -79,7 +79,7 @@ async def caller(*args: Any, **kwargs: Any) -> RPCResponse:
 
 #  Module should no longer have access to the full web3 api.
 #  Only the calling functions need access to the request methods.
-#  Any "re-entrant" shinanigans can go in the middlewares, which do
+#  Any "re-entrant" shenanigans can go in the middlewares, which do
 #  have web3 access.
 class Module:
     is_async = False

web3/tools/benchmark/main.py

@@ -75,7 +75,7 @@ async def build_async_w3_http(endpoint_uri: str) -> Web3:
     _web3 = Web3(
         AsyncHTTPProvider(endpoint_uri),  # type: ignore
         middlewares=[async_gas_price_strategy_middleware, async_buffered_gas_estimate_middleware],
-        modules={"eth": (AsyncEth,)},
+        modules={"eth": AsyncEth},
     )
     return _web3