Merge pull request #971 from elephantmipt/metrics

Metrics

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - extra functions for classification metrics ([#966](https://github.com/catalyst-team/catalyst/pull/966))
 - `OneOf` and `OneOfV2` batch transforms ([#951](https://github.com/catalyst-team/catalyst/pull/951))
+- ``precision_recall_fbeta_support`` metric ([#971](https://github.com/catalyst-team/catalyst/pull/971))
 
 ### Changed
 

catalyst/metrics/__init__.py

@@ -3,7 +3,10 @@
 from catalyst.metrics.auc import auc
 from catalyst.metrics.cmc_score import cmc_score, cmc_score_count
 from catalyst.metrics.dice import dice, calculate_dice
-from catalyst.metrics.f1_score import f1_score
+from catalyst.metrics.f1_score import f1_score, fbeta_score
+from catalyst.metrics.classification import precision_recall_fbeta_support
+from catalyst.metrics.precision import precision
+from catalyst.metrics.recall import recall
 from catalyst.metrics.focal import sigmoid_focal_loss, reduced_focal_loss
 from catalyst.metrics.functional import (
     process_multilabel_components,

catalyst/metrics/classification.py

@@ -0,0 +1,43 @@
+from typing import Optional, Tuple
+
+import torch
+
+from catalyst.metrics.functional import get_multiclass_statistics
+
+
+def precision_recall_fbeta_support(
+    outputs: torch.Tensor,
+    targets: torch.Tensor,
+    beta: float = 1,
+    eps: float = 1e-6,
+    argmax_dim: int = -1,
+    num_classes: Optional[int] = None,
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+    """
+    Counts precision, recall, fbeta_score.
+
+    Args:
+        outputs: A list of predicted elements
+        targets:  A list of elements that are to be predicted
+        beta: beta param for f_score
+        eps: epsilon to avoid zero division
+        argmax_dim: int, that specifies dimension for argmax transformation
+            in case of scores/probabilities in ``outputs``
+        num_classes: int, that specifies number of classes if it known.
+
+    Returns:
+        tuple of precision, recall, fbeta_score
+    """
+    tn, fp, fn, tp, support = get_multiclass_statistics(
+        outputs=outputs,
+        targets=targets,
+        argmax_dim=argmax_dim,
+        num_classes=num_classes,
+    )
+    precision = (tp + eps) / (fp + tp + eps)
+    recall = (tp + eps) / (fn + tp + eps)
+    numerator = (1 + beta ** 2) * precision * recall
+    denominator = beta ** 2 * precision + recall
+    fbeta = numerator / denominator
+
+    return precision, recall, fbeta, support

catalyst/metrics/f1_score.py

@@ -1,53 +1,84 @@
 """
 F1 score.
 """
+from typing import Optional, Union
+
 import torch
 
-from catalyst.utils.torch import get_activation_fn
+from catalyst.metrics.classification import precision_recall_fbeta_support
 
 
-def f1_score(
+def fbeta_score(
     outputs: torch.Tensor,
     targets: torch.Tensor,
     beta: float = 1.0,
     eps: float = 1e-7,
-    threshold: float = None,
-    activation: str = "Sigmoid",
-):
+    argmax_dim: int = -1,
+    num_classes: Optional[int] = None,
+) -> Union[float, torch.Tensor]:
     """
+    Counts fbeta score for given ``outputs`` and ``targets``.
+
     Args:
         outputs: A list of predicted elements
         targets:  A list of elements that are to be predicted
-        eps: epsilon to avoid zero division
         beta: beta param for f_score
-        threshold: threshold for outputs binarization
-        activation: An torch.nn activation applied to the outputs.
-            Must be one of ["none", "Sigmoid", "Softmax2d"]
+        eps: epsilon to avoid zero division
+        argmax_dim: int, that specifies dimension for argmax transformation
+            in case of scores/probabilities in ``outputs``
+        num_classes: int, that specifies number of classes if it known
+
+    Raises:
+        Exception: If ``beta`` is a negative number.
 
     Returns:
-        float: F_1 score
+        float: F_1 score.
     """
-    activation_fn = get_activation_fn(activation)
+    if beta < 0:
+        raise Exception("beta parameter should be non-negative")
+
+    _p, _r, fbeta, _ = precision_recall_fbeta_support(
+        outputs=outputs,
+        targets=targets,
+        beta=beta,
+        eps=eps,
+        argmax_dim=argmax_dim,
+        num_classes=num_classes,
+    )
+    return fbeta
 
-    outputs = activation_fn(outputs)
 
-    if threshold is not None:
-        outputs = (outputs > threshold).float()
+def f1_score(
+    outputs: torch.Tensor,
+    targets: torch.Tensor,
+    eps: float = 1e-7,
+    argmax_dim: int = -1,
+    num_classes: Optional[int] = None,
+) -> Union[float, torch.Tensor]:
+    """
+    Fbeta_score with beta=1.
 
-    true_positive = torch.sum(targets * outputs)
-    false_positive = torch.sum(outputs) - true_positive
-    false_negative = torch.sum(targets) - true_positive
+    Args:
+        outputs: A list of predicted elements
+        targets:  A list of elements that are to be predicted
+        eps: epsilon to avoid zero division
+        argmax_dim: int, that specifies dimension for argmax transformation
+            in case of scores/probabilities in ``outputs``
+        num_classes: int, that specifies number of classes if it known
 
-    precision_plus_recall = (
-        (1 + beta ** 2) * true_positive
-        + beta ** 2 * false_negative
-        + false_positive
-        + eps
+    Returns:
+        float: F_1 score
+    """
+    score = fbeta_score(
+        outputs=outputs,
+        targets=targets,
+        beta=1,
+        eps=eps,
+        argmax_dim=argmax_dim,
+        num_classes=num_classes,
     )
 
-    score = ((1 + beta ** 2) * true_positive + eps) / precision_plus_recall
-
     return score
 
 
-__all__ = ["f1_score"]
+__all__ = ["f1_score", "fbeta_score"]

catalyst/metrics/functional.py

@@ -1,6 +1,8 @@
 from typing import Callable, Dict, Optional, Sequence, Tuple
 from functools import partial
 
+import numpy as np
+
 import torch
 from torch import Tensor
 from torch.nn import functional as F
@@ -11,6 +13,74 @@
 # as a baseline
 
 
+def process_multiclass_components(
+    outputs: torch.Tensor,
+    targets: torch.Tensor,
+    argmax_dim: int = -1,
+    num_classes: Optional[int] = None,
+) -> Tuple[torch.Tensor, torch.Tensor, int]:
+    """
+    Preprocess input in case multiclass classification task.
+
+    Args:
+        outputs: estimated targets as predicted by a model
+            with shape [bs; ..., (num_classes or 1)]
+        targets: ground truth (correct) target values
+            with shape [bs; ..., 1]
+        argmax_dim: int, that specifies dimension for argmax transformation
+            in case of scores/probabilities in ``outputs``
+        num_classes: int, that specifies number of classes if it known
+
+    Returns:
+        preprocessed outputs, targets and num_classes
+    """
+    # @TODO: better multiclass preprocessing, label -> class_id mapping
+    if not torch.is_tensor(outputs):
+        outputs = torch.from_numpy(np.array(outputs))
+    if not torch.is_tensor(targets):
+        targets = torch.from_numpy(np.array(targets))
+
+    if outputs.dim() == targets.dim() + 1:
+        # looks like we have scores/probabilities in our outputs
+        # let's convert them to final model predictions
+        num_classes = max(
+            outputs.shape[argmax_dim], int(targets.max().detach().item() + 1)
+        )
+        outputs = torch.argmax(outputs, dim=argmax_dim)
+    if num_classes is None:
+        # as far as we expect the outputs/targets tensors to be int64
+        # we could find number of classes as max available number
+        num_classes = max(
+            int(outputs.max().detach().item() + 1),
+            int(targets.max().detach().item() + 1),
+        )
+
+    if outputs.dim() == 1:
+        outputs = outputs.view(-1, 1)
+    elif outputs.dim() == 2 and outputs.size(0) == 1:
+        # transpose case
+        outputs.permute(1, 0)
+    else:
+        assert outputs.size(1) == 1 and outputs.dim() == 2, (
+            "Wrong `outputs` shape, "
+            "expected 1D or 2D with size 1 in the second dim "
+            "got {}".format(outputs.shape)
+        )
+
+    if targets.dim() == 1:
+        targets = targets.view(-1, 1)
+    elif targets.dim() == 2 and targets.size(0) == 1:
+        # transpose case
+        targets.permute(1, 0)
+    else:
+        assert targets.size(1) == 1 and targets.dim() == 2, (
+            "Wrong `outputs` shape, "
+            "expected 1D or 2D with size 1 in the second dim"
+        )
+
+    return outputs, targets, num_classes
+
+
 def process_multilabel_components(
     outputs: torch.Tensor,
     targets: torch.Tensor,
@@ -127,8 +197,8 @@ def get_multiclass_statistics(
             with shape [bs; ..., (num_classes or 1)]
         targets: ground truth (correct) target values
             with shape [bs; ..., 1]
-        argmax_dim: int, that specifies dimention for argmax transformation
-            in case of scores/probabilites in ``outputs``
+        argmax_dim: int, that specifies dimension for argmax transformation
+            in case of scores/probabilities in ``outputs``
         num_classes: int, that specifies number of classes if it known
 
     Returns:
@@ -143,21 +213,12 @@ def get_multiclass_statistics(
         tensor([0., 0., 0., 1., 1.]), tensor([1., 1., 0., 0., 0.]),
         tensor([1., 1., 0., 1., 1.])
     """
-    # @TODO: move to process_multiclass_components ?
-    if outputs.dim() == targets.dim() + 1:
-        # looks like we have scores/probabilities in our outputs
-        # let's convert them to final model predictions
-        num_classes = max(
-            outputs.shape[argmax_dim], int(targets.max().detach().item() + 1)
-        )
-        outputs = torch.argmax(outputs, dim=argmax_dim)
-    if num_classes is None:
-        # as far as we expect the outputs/targets tensors to be int64
-        # we could find number of classes as max available number
-        num_classes = max(
-            int(outputs.max().detach().item() + 1),
-            int(targets.max().detach().item() + 1),
-        )
+    outputs, targets, num_classes = process_multiclass_components(
+        outputs=outputs,
+        targets=targets,
+        argmax_dim=argmax_dim,
+        num_classes=num_classes,
+    )
 
     tn = torch.zeros((num_classes,), device=outputs.device)
     fp = torch.zeros((num_classes,), device=outputs.device)

catalyst/metrics/precision.py

@@ -1,7 +1,8 @@
-from typing import Optional
+from typing import Optional, Union
 
 import torch
 
+from catalyst.metrics import precision_recall_fbeta_support
 from catalyst.metrics.functional import process_multilabel_components
 
 
@@ -66,4 +67,37 @@ def average_precision(
     return ap
 
 
-__all__ = ["average_precision"]
+def precision(
+    outputs: torch.Tensor,
+    targets: torch.Tensor,
+    argmax_dim: int = -1,
+    eps: float = 1e-7,
+    num_classes: Optional[int] = None,
+) -> Union[float, torch.Tensor]:
+    """
+    Multiclass precision metric.
+
+    Args:
+        outputs: estimated targets as predicted by a model
+            with shape [bs; ..., (num_classes or 1)]
+        targets: ground truth (correct) target values
+            with shape [bs; ..., 1]
+        argmax_dim: int, that specifies dimension for argmax transformation
+            in case of scores/probabilities in ``outputs``
+        eps: float. Epsilon to avoid zero division.
+        num_classes: int, that specifies number of classes if it known
+
+    Returns:
+        Tensor:
+    """
+    precision_score, _, _, _, = precision_recall_fbeta_support(
+        outputs=outputs,
+        targets=targets,
+        argmax_dim=argmax_dim,
+        eps=eps,
+        num_classes=num_classes,
+    )
+    return precision_score
+
+
+__all__ = ["average_precision", "precision"]

catalyst/metrics/recall.py

@@ -0,0 +1,39 @@
+from typing import Optional, Union
+
+import torch
+
+from catalyst.metrics import precision_recall_fbeta_support
+
+
+def recall(
+    outputs: torch.Tensor,
+    targets: torch.Tensor,
+    argmax_dim: int = -1,
+    eps: float = 1e-7,
+    num_classes: Optional[int] = None,
+) -> Union[float, torch.Tensor]:
+    """
+    Multiclass precision metric.
+
+    Args:
+        outputs: estimated targets as predicted by a model
+            with shape [bs; ..., (num_classes or 1)]
+        targets: ground truth (correct) target values
+            with shape [bs; ..., 1]
+        argmax_dim: int, that specifies dimension for argmax transformation
+            in case of scores/probabilities in ``outputs``
+        eps: float. Epsilon to avoid zero division.
+        num_classes: int, that specifies number of classes if it known.
+
+    Returns:
+        Tensor: recall for every class
+    """
+    _, recall_score, _, _ = precision_recall_fbeta_support(
+        outputs=outputs,
+        targets=targets,
+        argmax_dim=argmax_dim,
+        eps=eps,
+        num_classes=num_classes,
+    )
+
+    return recall_score