Fix evaluation loader and add tests

catalyst/runners/runner.py

@@ -677,7 +677,7 @@ def evaluate_loader(
         model: Optional[Model] = None,
         seed: int = 42,
         verbose: bool = False,
-    ) -> Dict:
+    ) -> Dict[str, Any]:
         """
         Evaluates data from loader with given model and returns obtained metrics. # noqa: DAR401
 
@@ -707,7 +707,7 @@ def evaluate_loader(
 
         if model is None:
             model = self.model
-        assert self.model is not None
+        assert model is not None
 
         self.train(
             model=model,

tests/catalyst/runners/test_runner.py

@@ -1,9 +1,11 @@
 from typing import Tuple
 
+import pytest
 import torch
 from torch import nn, Tensor
 from torch.utils.data import DataLoader, Dataset
 
+from catalyst import dl
 from catalyst.core.runner import RunnerException
 from catalyst.dl import SupervisedRunner
 
@@ -58,3 +60,51 @@ def test_cathing_empty_loader() -> None:
         run_train_with_empty_loader()
     except RunnerException:
         pass
+
+
+def test_evaluation_loader_metrics() -> None:
+    """
+    Test if metrics in evaluate loader works properly.
+    """
+    dataset = DummyDataset()
+    model = nn.Linear(in_features=dataset.features_dim, out_features=dataset.out_dim)
+    loader = DataLoader(dataset=dataset, batch_size=1)
+    callbacks = [dl.AccuracyCallback(input_key="logits", target_key="targets", topk_args=(1,))]
+    runner = SupervisedRunner()
+    runner.train(
+        loaders={"train": loader, "valid": loader},
+        model=model,
+        num_epochs=1,
+        criterion=nn.BCEWithLogitsLoss(),
+        callbacks=callbacks,
+    )
+    runner_internal_metrics = runner.loader_metrics
+    evaluate_loader_metrics = runner.evaluate_loader(loader=loader, callbacks=callbacks)
+    assert runner_internal_metrics["accuracy"] == evaluate_loader_metrics["accuracy"]
+
+
+def test_evaluation_loader_empty_model() -> None:
+    """
+    Test if there is no model was given, assertion raises.
+    """
+    with pytest.raises(AssertionError) as record:
+        dataset = DummyDataset()
+        loader = DataLoader(dataset=dataset, batch_size=1)
+        callbacks = [dl.AccuracyCallback(input_key="logits", target_key="targets", topk_args=(1,))]
+        runner = SupervisedRunner()
+        runner.evaluate_loader(loader=loader, callbacks=callbacks, model=None)
+        if not record:
+            pytest.fail("Expected assertion bacuase model is empty!")
+
+
+def test_evaluation_loader_custom_model() -> None:
+    """
+    Test if evaluate loader works with custom model.
+    """
+    dataset = DummyDataset()
+    model = nn.Linear(in_features=dataset.features_dim, out_features=dataset.out_dim)
+    loader = DataLoader(dataset=dataset, batch_size=1)
+    callbacks = [dl.AccuracyCallback(input_key="logits", target_key="targets", topk_args=(1,))]
+    runner = SupervisedRunner()
+
+    runner.evaluate_loader(loader=loader, callbacks=callbacks, model=model)