diff --git a/CHANGELOG.md b/CHANGELOG.md index ac6fb0702d..194ccc85b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - make `expdir` in `catalyst-dl run` optional ([#1249](https://github.com/catalyst-team/catalyst/pull/1249)) - Bump neptune-client from 0.9.5 to 0.9.8 in `requirements-neptune.txt` ([#1251](https://github.com/catalyst-team/catalyst/pull/1251)) - automatic merge for master (with [Mergify](https://mergify.io/)) fixed ([#1250](https://github.com/catalyst-team/catalyst/pull/1250)) - +- Evaluate loader custom model bug was fixed [#1254](https://github.com/catalyst-team/catalyst/pull/1254) ## [21.06] - 2021-06-29 diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index ddd23d6131..dce75e9596 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -677,7 +677,7 @@ def evaluate_loader( model: Optional[Model] = None, seed: int = 42, verbose: bool = False, - ) -> Dict: + ) -> Dict[str, Any]: """ Evaluates data from loader with given model and returns obtained metrics. # noqa: DAR401 @@ -707,7 +707,7 @@ def evaluate_loader( if model is None: model = self.model - assert self.model is not None + assert model is not None self.train( model=model, diff --git a/tests/catalyst/runners/test_runner.py b/tests/catalyst/runners/test_runner.py index 67027e14f2..e4012ac02d 100644 --- a/tests/catalyst/runners/test_runner.py +++ b/tests/catalyst/runners/test_runner.py @@ -1,9 +1,11 @@ from typing import Tuple +import pytest import torch from torch import nn, Tensor from torch.utils.data import DataLoader, Dataset +from catalyst import dl from catalyst.core.runner import RunnerException from catalyst.dl import SupervisedRunner @@ -58,3 +60,51 @@ def test_cathing_empty_loader() -> None: run_train_with_empty_loader() except RunnerException: pass + + +def test_evaluation_loader_metrics() -> None: + """ + Test if metrics in evaluate loader works properly. + """ + dataset = DummyDataset() + model = nn.Linear(in_features=dataset.features_dim, out_features=dataset.out_dim) + loader = DataLoader(dataset=dataset, batch_size=1) + callbacks = [dl.AccuracyCallback(input_key="logits", target_key="targets", topk_args=(1,))] + runner = SupervisedRunner() + runner.train( + loaders={"train": loader, "valid": loader}, + model=model, + num_epochs=1, + criterion=nn.BCEWithLogitsLoss(), + callbacks=callbacks, + ) + runner_internal_metrics = runner.loader_metrics + evaluate_loader_metrics = runner.evaluate_loader(loader=loader, callbacks=callbacks) + assert runner_internal_metrics["accuracy"] == evaluate_loader_metrics["accuracy"] + + +def test_evaluation_loader_empty_model() -> None: + """ + Test if there is no model was given, assertion raises. + """ + with pytest.raises(AssertionError) as record: + dataset = DummyDataset() + loader = DataLoader(dataset=dataset, batch_size=1) + callbacks = [dl.AccuracyCallback(input_key="logits", target_key="targets", topk_args=(1,))] + runner = SupervisedRunner() + runner.evaluate_loader(loader=loader, callbacks=callbacks, model=None) + if not record: + pytest.fail("Expected assertion bacuase model is empty!") + + +def test_evaluation_loader_custom_model() -> None: + """ + Test if evaluate loader works with custom model. + """ + dataset = DummyDataset() + model = nn.Linear(in_features=dataset.features_dim, out_features=dataset.out_dim) + loader = DataLoader(dataset=dataset, batch_size=1) + callbacks = [dl.AccuracyCallback(input_key="logits", target_key="targets", topk_args=(1,))] + runner = SupervisedRunner() + + runner.evaluate_loader(loader=loader, callbacks=callbacks, model=model)