From 471bb0e22307436f7afae6ada88102da09f46842 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Mon, 31 Jan 2022 14:31:45 +0100 Subject: [PATCH 001/141] implementaed IOU with segmentation masks and MAP for instance segmentation --- torchmetrics/detection/mean_ap.py | 76 ++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index c4ebec228d8..1b4aeea56ef 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -80,6 +80,27 @@ class COCOMetricResults(BaseMetricResults): ) +def segm_iou(inputs, targets, smooth=1): + + n_inputs = inputs.shape[0] + n_targets = targets.shape[0] + # flatten label and prediction tensors + inputs = inputs.view(n_inputs, -1).repeat_interleave(n_targets, 0) + targets = targets.view(n_targets, -1).repeat(n_inputs, 1) + + # i1 * t1 + # i1 * t2 + # i2 * t1 + # i2 * t2 + + # intersection is equivalent to True Positive count + # union is the mutually inclusive area of all labels & predictions + intersections = (inputs * targets).sum(1, keepdims=True) + unions = (inputs + targets).sum(1, keepdims=True) + + return ((intersections + smooth) / (unions + smooth)).view(n_inputs, n_targets) + + def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]]) -> None: """Ensure the correct input format of `preds` and `targets`""" if not isinstance(preds, Sequence): @@ -223,10 +244,13 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruth_boxes: List[Tensor] groundtruth_labels: List[Tensor] + groundtruth_masks: List[Tensor] + detection_masks: List[Tensor] def __init__( self, box_format: str = "xyxy", + iou_type: str = "bbox", iou_thresholds: Optional[List[float]] = None, rec_thresholds: Optional[List[float]] = None, max_detection_thresholds: Optional[List[int]] = None, @@ -243,6 +267,7 @@ def __init__( ) allowed_box_formats = ("xyxy", "xywh", "cxcywh") + allowed_iou_types = ("segm", "bbox") if box_format not in allowed_box_formats: raise ValueError(f"Expected argument `box_format` to be one of {allowed_box_formats} but got {box_format}") self.box_format = box_format @@ -250,11 +275,14 @@ def __init__( self.rec_thresholds = rec_thresholds or torch.linspace(0.0, 1.00, round(1.00 / 0.01) + 1).tolist() max_det_thr, _ = torch.sort(IntTensor(max_detection_thresholds or [1, 10, 100])) self.max_detection_thresholds = max_det_thr.tolist() + if iou_type not in allowed_iou_types: + raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") + self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): @@ -264,8 +292,10 @@ def __init__( self.add_state("detection_boxes", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -316,12 +346,16 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detection_boxes.append(boxes) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) + if "masks" in item: + self.detection_masks.append(item["masks"]) for item in target: boxes = _fix_empty_tensors(item["boxes"]) boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") self.groundtruth_boxes.append(boxes) self.groundtruth_labels.append(item["labels"]) + if "masks" in item: + self.groundtruth_masks.append(item["masks"]) def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -329,7 +363,17 @@ def _get_classes(self) -> List: return torch.cat(self.detection_labels + self.groundtruth_labels).unique().tolist() return [] - def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: + def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: + if self.iou_type == "segm": + return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) + elif self.iou_type == "bbox": + return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + else: + raise Exception(f"IOU type {self.iou_type} is not supported") + + def _compute_iou_impl( + self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable + ) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -341,10 +385,13 @@ def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: max_det: Maximum number of evaluated detection bounding boxes """ - gt = self.groundtruth_boxes[idx] - det = self.detection_boxes[idx] - gt_label_mask = self.groundtruth_labels[idx] == class_id - det_label_mask = self.detection_labels[idx] == class_id + + gt = ground_truths[id] + det = detections[id] + + gt_label_mask = self.groundtruth_labels[id] == class_id + det_label_mask = self.detection_labels[id] == class_id + if len(gt_label_mask) == 0 or len(det_label_mask) == 0: return Tensor([]) gt = gt[gt_label_mask] @@ -360,8 +407,7 @@ def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: if len(det) > max_det: det = det[:max_det] - # generalized_box_iou - ious = box_iou(det, gt) + ious = compute_iou(det, gt) return ious def __evaluate_image_gt_no_preds( @@ -761,6 +807,14 @@ def compute(self) -> dict: - map_per_class: ``torch.Tensor`` (-1 if class metrics are disabled) - mar_100_per_class: ``torch.Tensor`` (-1 if class metrics are disabled) """ + + # move everything to CPU, as we are faster here + self.detections = [box.cpu() for box in self.detection_boxes] + self.detection_labels = [label.cpu() for label in self.detection_labels] + self.detection_scores = [score.cpu() for score in self.detection_scores] + self.groundtruths = [box.cpu() for box in self.groundtruth_boxes] + self.groundtruth_labels = [label.cpu() for label in self.groundtruth_labels] + classes = self._get_classes() precisions, recalls = self._calculate(classes) map_val, mar_val = self._summarize_results(precisions, recalls) From 6ec2c5caedb1d7e07e878dc67fa1d4dab1f61fda Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Sat, 5 Feb 2022 11:48:30 +0100 Subject: [PATCH 002/141] rebase; ddp test still failing --- torchmetrics/detection/mean_ap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 1b4aeea56ef..78c20cc091d 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -841,4 +841,5 @@ def compute(self) -> dict: metrics.update(mar_val) metrics.map_per_class = map_per_class_values metrics[f"mar_{self.max_detection_thresholds[-1]}_per_class"] = mar_max_dets_per_class_values + return metrics From f0ab08335881af14d3a36599fd1584b55bdc4253 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:30:56 +0100 Subject: [PATCH 003/141] checking pr --- torchmetrics/detection/mean_ap.py | 73 ++++++++++++++++++------------- torchmetrics/metric.py | 1 + 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 78c20cc091d..1be1b0f2cf2 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -# Copyright The PyTorch Lightning team. +n # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -101,7 +101,9 @@ def segm_iou(inputs, targets, smooth=1): return ((intersections + smooth) / (unions + smooth)).view(n_inputs, n_targets) -def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]]) -> None: +def _input_validator( + preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]], iou_type: str = "bbox" +) -> None: """Ensure the correct input format of `preds` and `targets`""" if not isinstance(preds, Sequence): raise ValueError("Expected argument `preds` to be of type Sequence") @@ -109,37 +111,38 @@ def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[ raise ValueError("Expected argument `target` to be of type Sequence") if len(preds) != len(targets): raise ValueError("Expected argument `preds` and `target` to have the same length") + iou_attribute = "boxes" if iou_type == "bbox" else "masks" - for k in ["boxes", "scores", "labels"]: + for k in [iou_attribute, "scores", "labels"]: if any(k not in p for p in preds): raise ValueError(f"Expected all dicts in `preds` to contain the `{k}` key") - for k in ["boxes", "labels"]: + for k in [iou_attribute, "labels"]: if any(k not in p for p in targets): raise ValueError(f"Expected all dicts in `target` to contain the `{k}` key") - if any(type(pred["boxes"]) is not Tensor for pred in preds): - raise ValueError("Expected all boxes in `preds` to be of type Tensor") + if any(type(pred[iou_attribute]) is not Tensor for pred in preds): + raise ValueError(f"Expected all {iou_attribute} in `preds` to be of type Tensor") if any(type(pred["scores"]) is not Tensor for pred in preds): raise ValueError("Expected all scores in `preds` to be of type Tensor") if any(type(pred["labels"]) is not Tensor for pred in preds): raise ValueError("Expected all labels in `preds` to be of type Tensor") - if any(type(target["boxes"]) is not Tensor for target in targets): - raise ValueError("Expected all boxes in `target` to be of type Tensor") + if any(type(target[iou_attribute]) is not Tensor for target in targets): + raise ValueError(f"Expected all {iou_attribute} in `target` to be of type Tensor") if any(type(target["labels"]) is not Tensor for target in targets): raise ValueError("Expected all labels in `target` to be of type Tensor") for i, item in enumerate(targets): - if item["boxes"].size(0) != item["labels"].size(0): + if item[iou_attribute].size(0) != item["labels"].size(0): raise ValueError( - f"Input boxes and labels of sample {i} in targets have a" - f" different length (expected {item['boxes'].size(0)} labels, got {item['labels'].size(0)})" + f"Input {iou_attribute} and labels of sample {i} in targets have a" + f" different length (expected {item[iou_attribute].size(0)} labels, got {item['labels'].size(0)})" ) for i, item in enumerate(preds): - if not (item["boxes"].size(0) == item["labels"].size(0) == item["scores"].size(0)): + if not (item[iou_attribute].size(0) == item["labels"].size(0) == item["scores"].size(0)): raise ValueError( - f"Input boxes, labels and scores of sample {i} in predictions have a" - f" different length (expected {item['boxes'].size(0)} labels and scores," + f"Input {iou_attribute}, labels and scores of sample {i} in predictions have a" + f" different length (expected {item[iou_attribute].size(0)} labels and scores," f" got {item['labels'].size(0)} labels and {item['scores'].size(0)})" ) @@ -292,9 +295,9 @@ def __init__( self.add_state("detection_boxes", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore @@ -341,21 +344,29 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] _input_validator(preds, target) for item in preds: - boxes = _fix_empty_tensors(item["boxes"]) - boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + boxes, masks = self._get_safe_item_values(item) self.detection_boxes.append(boxes) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - if "masks" in item: - self.detection_masks.append(item["masks"]) + self.detection_masks.append(masks) for item in target: - boxes = _fix_empty_tensors(item["boxes"]) - boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + boxes, masks = self._get_safe_item_values(item) self.groundtruth_boxes.append(boxes) self.groundtruth_labels.append(item["labels"]) - if "masks" in item: - self.groundtruth_masks.append(item["masks"]) + self.groundtruth_masks.append(masks) + + def _get_safe_item_values(self, item): + if self.iou_type == "bbox": + boxes = _fix_empty_tensors(item["boxes"]) + boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + masks = _fix_empty_tensors(torch.Tensor()) + elif self.iou_type == "masks": + masks = _fix_empty_tensors(item["masks"]) + boxes = _fix_empty_tensors(torch.Tensor()) + else: + raise Exception(f"IOU type {self.iou_type} is not supported") + return boxes, masks def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -364,12 +375,14 @@ def _get_classes(self) -> List: return [] def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: - if self.iou_type == "segm": - return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) - elif self.iou_type == "bbox": - return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) - else: - raise Exception(f"IOU type {self.iou_type} is not supported") + return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + + # if self.iou_type == "segm": + # return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) + # elif self.iou_type == "bbox": + + # else: + # raise Exception(f"IOU type {self.iou_type} is not supported") def _compute_iou_impl( self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable @@ -407,7 +420,7 @@ def _compute_iou_impl( if len(det) > max_det: det = det[:max_det] - ious = compute_iou(det, gt) + ious = box_iou(det, gt) return ious def __evaluate_image_gt_no_preds( diff --git a/torchmetrics/metric.py b/torchmetrics/metric.py index 37db9015f92..1b3d193c4f0 100644 --- a/torchmetrics/metric.py +++ b/torchmetrics/metric.py @@ -257,6 +257,7 @@ def _sync_dist(self, dist_sync_fn: Callable = gather_all_tensors, process_group: for attr, reduction_fn in self._reductions.items(): # pre-processing ops (stack or flatten for inputs) + if isinstance(output_dict[attr][0], Tensor): output_dict[attr] = torch.stack(output_dict[attr]) elif isinstance(output_dict[attr][0], list): From 8564c4ab4c2b10c65bfec8b21458be30275da595 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:52:44 +0100 Subject: [PATCH 004/141] clean API interface; working on tests for iou_type SEGM --- tests/detection/test_map.py | 20 ++++++++++ torchmetrics/detection/mean_ap.py | 66 +++++++++++++------------------ 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 01524eae167..21649ec7c5b 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -344,6 +344,26 @@ def test_missing_gt(): assert result["map"] < 1, "MAP cannot be 1, as there is an image with no ground truth, but some predictions." +@pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") +def test_segm_iou_empty_mask(): + """Test empty ground truths.""" + metric = MeanAveragePrecision(iou_type="segm") + + metric.update( + [ + dict( + masks=torch.randint(0, 1, (1, 10, 10)), + scores=torch.Tensor([0.5]), + labels=torch.IntTensor([4]), + ), + ], + [ + dict(masks=torch.Tensor([]), labels=torch.IntTensor([])), + ], + ) + metric.compute() + + @pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") def test_error_on_wrong_input(): """Test class input validation.""" diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 1be1b0f2cf2..8fb961ac38e 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -242,13 +242,11 @@ class MeanAveragePrecision(Metric): If ``class_metrics`` is not a boolean """ - detection_boxes: List[Tensor] + detections: List[Tensor] detection_scores: List[Tensor] detection_labels: List[Tensor] - groundtruth_boxes: List[Tensor] + groundtruths: List[Tensor] groundtruth_labels: List[Tensor] - groundtruth_masks: List[Tensor] - detection_masks: List[Tensor] def __init__( self, @@ -292,13 +290,11 @@ def __init__( raise ValueError("Expected argument `class_metrics` to be a boolean") self.class_metrics = class_metrics - self.add_state("detection_boxes", default=[], dist_reduce_fx=None) + self.add_state("detections", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) + self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -341,32 +337,29 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] ValueError: If any score is not type float and of length 1 """ - _input_validator(preds, target) + _input_validator(preds, target, iou_type=self.iou_type) for item in preds: - boxes, masks = self._get_safe_item_values(item) - self.detection_boxes.append(boxes) + detections = self._get_safe_item_values(item) + self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - self.detection_masks.append(masks) for item in target: - boxes, masks = self._get_safe_item_values(item) - self.groundtruth_boxes.append(boxes) + groundtruths = self._get_safe_item_values(item) + self.groundtruths.append(groundtruths) self.groundtruth_labels.append(item["labels"]) - self.groundtruth_masks.append(masks) def _get_safe_item_values(self, item): if self.iou_type == "bbox": boxes = _fix_empty_tensors(item["boxes"]) boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") - masks = _fix_empty_tensors(torch.Tensor()) - elif self.iou_type == "masks": + return boxes + elif self.iou_type == "segm": masks = _fix_empty_tensors(item["masks"]) - boxes = _fix_empty_tensors(torch.Tensor()) + return masks else: raise Exception(f"IOU type {self.iou_type} is not supported") - return boxes, masks def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -375,18 +368,11 @@ def _get_classes(self) -> List: return [] def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: - return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + iou_func = box_iou if self.iou_type == "bbox" else segm_iou - # if self.iou_type == "segm": - # return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) - # elif self.iou_type == "bbox": + return self._compute_iou_impl(id, class_id, max_det, iou_func) - # else: - # raise Exception(f"IOU type {self.iou_type} is not supported") - - def _compute_iou_impl( - self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable - ) -> Tensor: + def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: Callable) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -399,8 +385,9 @@ def _compute_iou_impl( Maximum number of evaluated detection bounding boxes """ - gt = ground_truths[id] - det = detections[id] + # if self.iou_type == "bbox": + gt = self.groundtruths[id] + det = self.detections[id] gt_label_mask = self.groundtruth_labels[id] == class_id det_label_mask = self.detection_labels[id] == class_id @@ -413,14 +400,14 @@ def _compute_iou_impl( return Tensor([]) # Sort by scores and use only max detections - scores = self.detection_scores[idx] - scores_filtered = scores[self.detection_labels[idx] == class_id] + scores = self.detection_scores[id] + scores_filtered = scores[self.detection_labels[id] == class_id] inds = torch.argsort(scores_filtered, descending=True) det = det[inds] if len(det) > max_det: det = det[:max_det] - ious = box_iou(det, gt) + ious = compute_iou(det, gt) return ious def __evaluate_image_gt_no_preds( @@ -494,8 +481,9 @@ def _evaluate_image( ious: IoU results for image and class. """ - gt = self.groundtruth_boxes[idx] - det = self.detection_boxes[idx] + + gt = self.groundtruths[idx] + det = self.detections[idx] gt_label_mask = self.groundtruth_labels[idx] == class_id det_label_mask = self.detection_labels[idx] == class_id @@ -649,7 +637,7 @@ def _calculate(self, class_ids: List) -> Tuple[MAPMetricResults, MARMetricResult class_ids: List of label class Ids. """ - img_ids = range(len(self.groundtruth_boxes)) + img_ids = range(len(self.groundtruths)) max_detections = self.max_detection_thresholds[-1] area_ranges = self.bbox_area_ranges.values() @@ -822,10 +810,10 @@ def compute(self) -> dict: """ # move everything to CPU, as we are faster here - self.detections = [box.cpu() for box in self.detection_boxes] + self.detections = [box.cpu() for box in self.detections] self.detection_labels = [label.cpu() for label in self.detection_labels] self.detection_scores = [score.cpu() for score in self.detection_scores] - self.groundtruths = [box.cpu() for box in self.groundtruth_boxes] + self.groundtruths = [box.cpu() for box in self.groundtruths] self.groundtruth_labels = [label.cpu() for label in self.groundtruth_labels] classes = self._get_classes() From 2eef6662d35a132e6450fa8772f5a6f6f5b5f19a Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Wed, 6 Apr 2022 16:25:07 +0200 Subject: [PATCH 005/141] fixing import --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 8fb961ac38e..ecd9a8b8e50 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple import torch from torch import IntTensor, Tensor From 0ede5e8f525394d769f7d3fbbcbda7c407816f4f Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:14:04 +0200 Subject: [PATCH 006/141] map test --- tests/detection/test_map.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 21649ec7c5b..d9bc85d7eb2 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -23,7 +23,9 @@ Input = namedtuple("Input", ["preds", "target"]) -_inputs = Input( +_inputs_masks = Input(preds=[], target=[]) + +_inputs_bboxes = Input( preds=[ [ dict( @@ -137,7 +139,7 @@ def _compare_fn(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results All classes Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.706 Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.901 @@ -214,8 +216,8 @@ def test_map(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs.preds, - target=_inputs.target, + preds=_inputs_bboxes.preds, + target=_inputs_bboxes.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From 65453f09544131ece158618cd840c6c2878b749b Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:42:25 +0200 Subject: [PATCH 007/141] fixes --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index ecd9a8b8e50..bd9bffb7106 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -n # Copyright The PyTorch Lightning team. +# Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From b4ce67685e0daf67266085c4b1590fafa77798f5 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 15:42:27 +0200 Subject: [PATCH 008/141] test added (required native RLE encoding from COCO); fixed IOU --- tests/detection/test_map.py | 138 +++++++++++++++++++++++++++++- torchmetrics/detection/mean_ap.py | 39 ++++++++- 2 files changed, 171 insertions(+), 6 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index d9bc85d7eb2..b8c731d8199 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -14,8 +14,10 @@ from collections import namedtuple +import numpy as np import pytest import torch +from pycocotools import mask from tests.helpers.testers import MetricTester from torchmetrics.detection.mean_ap import MeanAveragePrecision @@ -23,7 +25,87 @@ Input = namedtuple("Input", ["preds", "target"]) -_inputs_masks = Input(preds=[], target=[]) +_inputs_masks = Input( + preds=[ + [ + dict( + masks=torch.Tensor( + mask.decode( + { + "size": [478, 640], + "counts": "VQi31m>0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR dict: } +def _compare_fn_segm(preds, target) -> dict: + """Comparison function for map implementation. + + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + + """ + return { + "map": torch.Tensor([0.352]), + "map_50": torch.Tensor([0.742]), + "map_75": torch.Tensor([0.252]), + "map_small": torch.Tensor([-1]), + "map_medium": torch.Tensor([-1]), + "map_large": torch.Tensor([0.352]), + "mar_1": torch.Tensor([0.35]), + "mar_10": torch.Tensor([0.35]), + "mar_100": torch.Tensor([0.35]), + "mar_small": torch.Tensor([-1]), + "mar_medium": torch.Tensor([-1]), + "mar_large": torch.Tensor([0.35]), + "map_per_class": torch.Tensor([0.4039604, -1.0, 0.3]), + "mar_100_per_class": torch.Tensor([0.4, -1.0, 0.3]), + } + + _pytest_condition = not (_TORCHVISION_AVAILABLE and _TORCHVISION_GREATER_EQUAL_0_8) @@ -212,7 +330,8 @@ class TestMAP(MetricTester): atol = 1e-1 @pytest.mark.parametrize("ddp", [False, True]) - def test_map(self, compute_on_cpu, ddp): + def test_map_bbox(self, compute_on_cpu, ddp): + """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, @@ -225,6 +344,21 @@ def test_map(self, compute_on_cpu, ddp): metric_args={"class_metrics": True, "compute_on_cpu": compute_on_cpu}, ) + @pytest.mark.parametrize("ddp", [False]) + def test_map_segm(self, ddp): + """Test modular implementation for correctness.""" + + self.run_class_metric_test( + ddp=ddp, + preds=_inputs_masks.preds, + target=_inputs_masks.target, + metric_class=MeanAveragePrecision, + sk_metric=_compare_fn_segm, + dist_sync_on_step=False, + check_batch=False, + metric_args={"class_metrics": True, "iou_type": "segm"}, + ) + # noinspection PyTypeChecker @pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index bd9bffb7106..6615fecc3b4 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -29,6 +29,24 @@ log = logging.getLogger(__name__) +def mask_area(input): + n_inputs = len(input) + + return input.reshape(n_inputs, -1).sum(1) + + +def compute_area(input, type="bbox"): + if len(input) == 0: + + return torch.Tensor([]).to(input.device) + + if type == "bbox": + + return box_area(input) + else: + return mask_area(input) + + class BaseMetricResults(dict): """Base metric class, that allows fields for pre-defined metrics.""" @@ -80,13 +98,23 @@ class COCOMetricResults(BaseMetricResults): ) +def _segm_iou(mask1, mask2): + + intersection = (mask1 * mask2).sum() + if intersection == 0: + return 0.0 + union = torch.logical_or(mask1, mask2).to(torch.int).sum() + return (intersection / union).unsqueeze(0) + + def segm_iou(inputs, targets, smooth=1): n_inputs = inputs.shape[0] n_targets = targets.shape[0] # flatten label and prediction tensors - inputs = inputs.view(n_inputs, -1).repeat_interleave(n_targets, 0) - targets = targets.view(n_targets, -1).repeat(n_inputs, 1) + + inputs = inputs.reshape(n_inputs, -1).repeat_interleave(n_targets, 0) + targets = targets.reshape(n_targets, -1).repeat(n_inputs, 1) # i1 * t1 # i1 * t2 @@ -408,6 +436,7 @@ def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: C det = det[:max_det] ious = compute_iou(det, gt) + return ious def __evaluate_image_gt_no_preds( @@ -506,13 +535,14 @@ def _evaluate_image( if gt.numel() == 0 and det.numel() == 0: return None - areas = box_area(gt) + areas = compute_area(gt, self.iou_type) ignore_area = (areas < area_range[0]) | (areas > area_range[1]) # sort dt highest score first, sort gt ignore last ignore_area_sorted, gtind = torch.sort(ignore_area.to(torch.uint8)) # Convert to uint8 temporarily and back to bool, because "Sort currently does not support bool dtype on CUDA" ignore_area_sorted = ignore_area_sorted.to(torch.bool) + gt = gt[gtind] scores = self.detection_scores[idx] scores_filtered = scores[det_label_mask] @@ -542,12 +572,13 @@ def _evaluate_image( gt_matches[idx_iou, m] = 1 # set unmatched detections outside of area range to ignore - det_areas = box_area(det) + det_areas = compute_area(det, self.iou_type) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.logical_or( det_ignore, torch.logical_and(det_matches == 0, torch.repeat_interleave(ar, nb_iou_thrs, 0)) ) + return { "dtMatches": det_matches.to(self.device), "gtMatches": gt_matches.to(self.device), From cf0c77505d587ee2fa6af3e505bb7f2dcba78d3d Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:13:34 +0200 Subject: [PATCH 009/141] rebased and fixed GPU device --- tests/detection/test_map.py | 13 +++++++------ torchmetrics/detection/mean_ap.py | 10 ++++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index b8c731d8199..1e55d286e49 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -107,7 +107,7 @@ ) -_inputs_bboxes = Input( +_inputs = Input( preds=[ [ dict( @@ -335,8 +335,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs_bboxes.preds, - target=_inputs_bboxes.target, + preds=_inputs.preds, + target=_inputs.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, @@ -345,7 +345,7 @@ def test_map_bbox(self, compute_on_cpu, ddp): ) @pytest.mark.parametrize("ddp", [False]) - def test_map_segm(self, ddp): + def test_map_segm(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( @@ -356,7 +356,7 @@ def test_map_segm(self, ddp): sk_metric=_compare_fn_segm, dist_sync_on_step=False, check_batch=False, - metric_args={"class_metrics": True, "iou_type": "segm"}, + metric_args={"class_metrics": True, "compute_on_cpu": compute_on_cpu, "iou_type": "segm"}, ) @@ -488,7 +488,7 @@ def test_segm_iou_empty_mask(): metric.update( [ dict( - masks=torch.randint(0, 1, (1, 10, 10)), + masks=torch.randint(0, 1, (1, 10, 10)).bool(), scores=torch.Tensor([0.5]), labels=torch.IntTensor([4]), ), @@ -497,6 +497,7 @@ def test_segm_iou_empty_mask(): dict(masks=torch.Tensor([]), labels=torch.IntTensor([])), ], ) + metric.compute() diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 6615fecc3b4..490151e022c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -469,6 +469,7 @@ def __evaluate_image_preds_no_gt( """Some predictions but no GT.""" # GTs nb_gt = 0 + gt_ignore = torch.zeros(nb_gt, dtype=torch.bool, device=self.device) # Detections @@ -480,7 +481,7 @@ def __evaluate_image_preds_no_gt( if len(det) > max_det: det = det[:max_det] nb_det = len(det) - det_areas = box_area(det).to(self.device) + det_areas = compute_area(det, type=self.iou_type).to(self.device) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.repeat_interleave(ar, nb_iou_thrs, 0) @@ -488,9 +489,9 @@ def __evaluate_image_preds_no_gt( return { "dtMatches": torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device), "gtMatches": torch.zeros((nb_iou_thrs, nb_gt), dtype=torch.bool, device=self.device), - "dtScores": scores_sorted, - "gtIgnore": gt_ignore, - "dtIgnore": det_ignore, + "dtScores": scores_sorted.to(self.device), + "gtIgnore": gt_ignore.to(self.device), + "dtIgnore": det_ignore.to(self.device), } def _evaluate_image( @@ -768,6 +769,7 @@ def __calculate_recall_precision_scores( img_eval_cls_bbox = [e for e in img_eval_cls_bbox if e is not None] if not img_eval_cls_bbox: return recall, precision, scores + det_scores = torch.cat([e["dtScores"][:max_det] for e in img_eval_cls_bbox]) # different sorting method generates slightly different results. From 5fb5b6e31e4d8245ecc6af427200baebe90af94c Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:17:20 +0200 Subject: [PATCH 010/141] docstring and smooth fix --- torchmetrics/detection/mean_ap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 490151e022c..bd9d0035253 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -107,7 +107,7 @@ def _segm_iou(mask1, mask2): return (intersection / union).unsqueeze(0) -def segm_iou(inputs, targets, smooth=1): +def segm_iou(inputs, targets, smooth=1e-5): n_inputs = inputs.shape[0] n_targets = targets.shape[0] @@ -209,6 +209,8 @@ class MeanAveragePrecision(Metric): Args: box_format: Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. + iou_type: + Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. From b56688c5c9fbb0ce1a144172c1bebd86fed0f33f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:14 +0000 Subject: [PATCH 011/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 1 - torchmetrics/detection/mean_ap.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 1e55d286e49..f85b386e812 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -294,7 +294,6 @@ def _compare_fn_segm(preds, target) -> dict: Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 - """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index bd9d0035253..8da70b3219f 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -310,10 +310,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 0b1104e5aa53ba0c02c84aa732384a480237a29c Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:27:37 +0200 Subject: [PATCH 012/141] Added pycocotools as detection requirements for test --- requirements/detection.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/detection.txt b/requirements/detection.txt index cbec7260775..da90ea549b7 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1 +1,2 @@ torchvision>=0.8 +pycocotools From 83ed880fd36cab5c55962772a032774516770a77 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:28:18 +0200 Subject: [PATCH 013/141] pycocotools test req --- requirements/detection.txt | 1 - requirements/test.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/detection.txt b/requirements/detection.txt index da90ea549b7..cbec7260775 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1,2 +1 @@ torchvision>=0.8 -pycocotools diff --git a/requirements/test.txt b/requirements/test.txt index dfffc35433d..78c17457c98 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -16,3 +16,4 @@ fire cloudpickle>=1.3 scikit-learn>=0.24 +pycocotools From a4d60a8216998266feb0e8776d2df825062ce3ee Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Mon, 31 Jan 2022 14:31:45 +0100 Subject: [PATCH 014/141] implementaed IOU with segmentation masks and MAP for instance segmentation --- torchmetrics/detection/mean_ap.py | 76 ++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 1ef6f5ae69c..520c0a580f3 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -80,6 +80,27 @@ class COCOMetricResults(BaseMetricResults): ) +def segm_iou(inputs, targets, smooth=1): + + n_inputs = inputs.shape[0] + n_targets = targets.shape[0] + # flatten label and prediction tensors + inputs = inputs.view(n_inputs, -1).repeat_interleave(n_targets, 0) + targets = targets.view(n_targets, -1).repeat(n_inputs, 1) + + # i1 * t1 + # i1 * t2 + # i2 * t1 + # i2 * t2 + + # intersection is equivalent to True Positive count + # union is the mutually inclusive area of all labels & predictions + intersections = (inputs * targets).sum(1, keepdims=True) + unions = (inputs + targets).sum(1, keepdims=True) + + return ((intersections + smooth) / (unions + smooth)).view(n_inputs, n_targets) + + def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]]) -> None: """Ensure the correct input format of `preds` and `targets`""" if not isinstance(preds, Sequence): @@ -223,10 +244,13 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruth_boxes: List[Tensor] groundtruth_labels: List[Tensor] + groundtruth_masks: List[Tensor] + detection_masks: List[Tensor] def __init__( self, box_format: str = "xyxy", + iou_type: str = "bbox", iou_thresholds: Optional[List[float]] = None, rec_thresholds: Optional[List[float]] = None, max_detection_thresholds: Optional[List[int]] = None, @@ -243,6 +267,7 @@ def __init__( ) allowed_box_formats = ("xyxy", "xywh", "cxcywh") + allowed_iou_types = ("segm", "bbox") if box_format not in allowed_box_formats: raise ValueError(f"Expected argument `box_format` to be one of {allowed_box_formats} but got {box_format}") self.box_format = box_format @@ -250,11 +275,14 @@ def __init__( self.rec_thresholds = rec_thresholds or torch.linspace(0.0, 1.00, round(1.00 / 0.01) + 1).tolist() max_det_thr, _ = torch.sort(IntTensor(max_detection_thresholds or [1, 10, 100])) self.max_detection_thresholds = max_det_thr.tolist() + if iou_type not in allowed_iou_types: + raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") + self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): @@ -264,8 +292,10 @@ def __init__( self.add_state("detection_boxes", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -316,12 +346,16 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detection_boxes.append(boxes) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) + if "masks" in item: + self.detection_masks.append(item["masks"]) for item in target: boxes = _fix_empty_tensors(item["boxes"]) boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") self.groundtruth_boxes.append(boxes) self.groundtruth_labels.append(item["labels"]) + if "masks" in item: + self.groundtruth_masks.append(item["masks"]) def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -329,7 +363,17 @@ def _get_classes(self) -> List: return torch.cat(self.detection_labels + self.groundtruth_labels).unique().tolist() return [] - def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: + def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: + if self.iou_type == "segm": + return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) + elif self.iou_type == "bbox": + return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + else: + raise Exception(f"IOU type {self.iou_type} is not supported") + + def _compute_iou_impl( + self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable + ) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -341,10 +385,13 @@ def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: max_det: Maximum number of evaluated detection bounding boxes """ - gt = self.groundtruth_boxes[idx] - det = self.detection_boxes[idx] - gt_label_mask = self.groundtruth_labels[idx] == class_id - det_label_mask = self.detection_labels[idx] == class_id + + gt = ground_truths[id] + det = detections[id] + + gt_label_mask = self.groundtruth_labels[id] == class_id + det_label_mask = self.detection_labels[id] == class_id + if len(gt_label_mask) == 0 or len(det_label_mask) == 0: return Tensor([]) gt = gt[gt_label_mask] @@ -360,8 +407,7 @@ def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: if len(det) > max_det: det = det[:max_det] - # generalized_box_iou - ious = box_iou(det, gt) + ious = compute_iou(det, gt) return ious def __evaluate_image_gt_no_preds( @@ -764,6 +810,14 @@ def compute(self) -> dict: - map_per_class: ``torch.Tensor`` (-1 if class metrics are disabled) - mar_100_per_class: ``torch.Tensor`` (-1 if class metrics are disabled) """ + + # move everything to CPU, as we are faster here + self.detections = [box.cpu() for box in self.detection_boxes] + self.detection_labels = [label.cpu() for label in self.detection_labels] + self.detection_scores = [score.cpu() for score in self.detection_scores] + self.groundtruths = [box.cpu() for box in self.groundtruth_boxes] + self.groundtruth_labels = [label.cpu() for label in self.groundtruth_labels] + classes = self._get_classes() precisions, recalls = self._calculate(classes) map_val, mar_val = self._summarize_results(precisions, recalls) From d4b0c67e7cf28df830acd8000ab38cc6687adb80 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Sat, 5 Feb 2022 11:48:30 +0100 Subject: [PATCH 015/141] rebase; ddp test still failing --- torchmetrics/detection/mean_ap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 520c0a580f3..6668e92fcbd 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -844,4 +844,5 @@ def compute(self) -> dict: metrics.update(mar_val) metrics.map_per_class = map_per_class_values metrics[f"mar_{self.max_detection_thresholds[-1]}_per_class"] = mar_max_dets_per_class_values + return metrics From ff74a0fc9b98306512499791249743100e03ee7c Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:30:56 +0100 Subject: [PATCH 016/141] checking pr --- torchmetrics/detection/mean_ap.py | 73 ++++++++++++++++++------------- torchmetrics/metric.py | 1 + 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 6668e92fcbd..fd04c67f67c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -# Copyright The PyTorch Lightning team. +n # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -101,7 +101,9 @@ def segm_iou(inputs, targets, smooth=1): return ((intersections + smooth) / (unions + smooth)).view(n_inputs, n_targets) -def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]]) -> None: +def _input_validator( + preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]], iou_type: str = "bbox" +) -> None: """Ensure the correct input format of `preds` and `targets`""" if not isinstance(preds, Sequence): raise ValueError("Expected argument `preds` to be of type Sequence") @@ -109,37 +111,38 @@ def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[ raise ValueError("Expected argument `target` to be of type Sequence") if len(preds) != len(targets): raise ValueError("Expected argument `preds` and `target` to have the same length") + iou_attribute = "boxes" if iou_type == "bbox" else "masks" - for k in ["boxes", "scores", "labels"]: + for k in [iou_attribute, "scores", "labels"]: if any(k not in p for p in preds): raise ValueError(f"Expected all dicts in `preds` to contain the `{k}` key") - for k in ["boxes", "labels"]: + for k in [iou_attribute, "labels"]: if any(k not in p for p in targets): raise ValueError(f"Expected all dicts in `target` to contain the `{k}` key") - if any(type(pred["boxes"]) is not Tensor for pred in preds): - raise ValueError("Expected all boxes in `preds` to be of type Tensor") + if any(type(pred[iou_attribute]) is not Tensor for pred in preds): + raise ValueError(f"Expected all {iou_attribute} in `preds` to be of type Tensor") if any(type(pred["scores"]) is not Tensor for pred in preds): raise ValueError("Expected all scores in `preds` to be of type Tensor") if any(type(pred["labels"]) is not Tensor for pred in preds): raise ValueError("Expected all labels in `preds` to be of type Tensor") - if any(type(target["boxes"]) is not Tensor for target in targets): - raise ValueError("Expected all boxes in `target` to be of type Tensor") + if any(type(target[iou_attribute]) is not Tensor for target in targets): + raise ValueError(f"Expected all {iou_attribute} in `target` to be of type Tensor") if any(type(target["labels"]) is not Tensor for target in targets): raise ValueError("Expected all labels in `target` to be of type Tensor") for i, item in enumerate(targets): - if item["boxes"].size(0) != item["labels"].size(0): + if item[iou_attribute].size(0) != item["labels"].size(0): raise ValueError( - f"Input boxes and labels of sample {i} in targets have a" - f" different length (expected {item['boxes'].size(0)} labels, got {item['labels'].size(0)})" + f"Input {iou_attribute} and labels of sample {i} in targets have a" + f" different length (expected {item[iou_attribute].size(0)} labels, got {item['labels'].size(0)})" ) for i, item in enumerate(preds): - if not (item["boxes"].size(0) == item["labels"].size(0) == item["scores"].size(0)): + if not (item[iou_attribute].size(0) == item["labels"].size(0) == item["scores"].size(0)): raise ValueError( - f"Input boxes, labels and scores of sample {i} in predictions have a" - f" different length (expected {item['boxes'].size(0)} labels and scores," + f"Input {iou_attribute}, labels and scores of sample {i} in predictions have a" + f" different length (expected {item[iou_attribute].size(0)} labels and scores," f" got {item['labels'].size(0)} labels and {item['scores'].size(0)})" ) @@ -292,9 +295,9 @@ def __init__( self.add_state("detection_boxes", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore @@ -341,21 +344,29 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] _input_validator(preds, target) for item in preds: - boxes = _fix_empty_tensors(item["boxes"]) - boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + boxes, masks = self._get_safe_item_values(item) self.detection_boxes.append(boxes) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - if "masks" in item: - self.detection_masks.append(item["masks"]) + self.detection_masks.append(masks) for item in target: - boxes = _fix_empty_tensors(item["boxes"]) - boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + boxes, masks = self._get_safe_item_values(item) self.groundtruth_boxes.append(boxes) self.groundtruth_labels.append(item["labels"]) - if "masks" in item: - self.groundtruth_masks.append(item["masks"]) + self.groundtruth_masks.append(masks) + + def _get_safe_item_values(self, item): + if self.iou_type == "bbox": + boxes = _fix_empty_tensors(item["boxes"]) + boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + masks = _fix_empty_tensors(torch.Tensor()) + elif self.iou_type == "masks": + masks = _fix_empty_tensors(item["masks"]) + boxes = _fix_empty_tensors(torch.Tensor()) + else: + raise Exception(f"IOU type {self.iou_type} is not supported") + return boxes, masks def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -364,12 +375,14 @@ def _get_classes(self) -> List: return [] def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: - if self.iou_type == "segm": - return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) - elif self.iou_type == "bbox": - return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) - else: - raise Exception(f"IOU type {self.iou_type} is not supported") + return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + + # if self.iou_type == "segm": + # return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) + # elif self.iou_type == "bbox": + + # else: + # raise Exception(f"IOU type {self.iou_type} is not supported") def _compute_iou_impl( self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable @@ -407,7 +420,7 @@ def _compute_iou_impl( if len(det) > max_det: det = det[:max_det] - ious = compute_iou(det, gt) + ious = box_iou(det, gt) return ious def __evaluate_image_gt_no_preds( diff --git a/torchmetrics/metric.py b/torchmetrics/metric.py index 5e4484ef119..d0d50c862a4 100644 --- a/torchmetrics/metric.py +++ b/torchmetrics/metric.py @@ -257,6 +257,7 @@ def _sync_dist(self, dist_sync_fn: Callable = gather_all_tensors, process_group: for attr, reduction_fn in self._reductions.items(): # pre-processing ops (stack or flatten for inputs) + if isinstance(output_dict[attr][0], Tensor): output_dict[attr] = torch.stack(output_dict[attr]) elif isinstance(output_dict[attr][0], list): From 069196538da3f290deb5485d6ce424304c758b72 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:52:44 +0100 Subject: [PATCH 017/141] clean API interface; working on tests for iou_type SEGM --- tests/detection/test_map.py | 20 ++++++++++ torchmetrics/detection/mean_ap.py | 66 +++++++++++++------------------ 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index f63e3d80814..3d1a347eac2 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -363,6 +363,26 @@ def test_missing_gt(): assert result["map"] < 1, "MAP cannot be 1, as there is an image with no ground truth, but some predictions." +@pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") +def test_segm_iou_empty_mask(): + """Test empty ground truths.""" + metric = MeanAveragePrecision(iou_type="segm") + + metric.update( + [ + dict( + masks=torch.randint(0, 1, (1, 10, 10)), + scores=torch.Tensor([0.5]), + labels=torch.IntTensor([4]), + ), + ], + [ + dict(masks=torch.Tensor([]), labels=torch.IntTensor([])), + ], + ) + metric.compute() + + @pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") def test_error_on_wrong_input(): """Test class input validation.""" diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index fd04c67f67c..47ad4209f1c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -242,13 +242,11 @@ class MeanAveragePrecision(Metric): If ``class_metrics`` is not a boolean """ - detection_boxes: List[Tensor] + detections: List[Tensor] detection_scores: List[Tensor] detection_labels: List[Tensor] - groundtruth_boxes: List[Tensor] + groundtruths: List[Tensor] groundtruth_labels: List[Tensor] - groundtruth_masks: List[Tensor] - detection_masks: List[Tensor] def __init__( self, @@ -292,13 +290,11 @@ def __init__( raise ValueError("Expected argument `class_metrics` to be a boolean") self.class_metrics = class_metrics - self.add_state("detection_boxes", default=[], dist_reduce_fx=None) + self.add_state("detections", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) + self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -341,32 +337,29 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] ValueError: If any score is not type float and of length 1 """ - _input_validator(preds, target) + _input_validator(preds, target, iou_type=self.iou_type) for item in preds: - boxes, masks = self._get_safe_item_values(item) - self.detection_boxes.append(boxes) + detections = self._get_safe_item_values(item) + self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - self.detection_masks.append(masks) for item in target: - boxes, masks = self._get_safe_item_values(item) - self.groundtruth_boxes.append(boxes) + groundtruths = self._get_safe_item_values(item) + self.groundtruths.append(groundtruths) self.groundtruth_labels.append(item["labels"]) - self.groundtruth_masks.append(masks) def _get_safe_item_values(self, item): if self.iou_type == "bbox": boxes = _fix_empty_tensors(item["boxes"]) boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") - masks = _fix_empty_tensors(torch.Tensor()) - elif self.iou_type == "masks": + return boxes + elif self.iou_type == "segm": masks = _fix_empty_tensors(item["masks"]) - boxes = _fix_empty_tensors(torch.Tensor()) + return masks else: raise Exception(f"IOU type {self.iou_type} is not supported") - return boxes, masks def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -375,18 +368,11 @@ def _get_classes(self) -> List: return [] def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: - return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + iou_func = box_iou if self.iou_type == "bbox" else segm_iou - # if self.iou_type == "segm": - # return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) - # elif self.iou_type == "bbox": + return self._compute_iou_impl(id, class_id, max_det, iou_func) - # else: - # raise Exception(f"IOU type {self.iou_type} is not supported") - - def _compute_iou_impl( - self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable - ) -> Tensor: + def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: Callable) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -399,8 +385,9 @@ def _compute_iou_impl( Maximum number of evaluated detection bounding boxes """ - gt = ground_truths[id] - det = detections[id] + # if self.iou_type == "bbox": + gt = self.groundtruths[id] + det = self.detections[id] gt_label_mask = self.groundtruth_labels[id] == class_id det_label_mask = self.detection_labels[id] == class_id @@ -413,14 +400,14 @@ def _compute_iou_impl( return Tensor([]) # Sort by scores and use only max detections - scores = self.detection_scores[idx] - scores_filtered = scores[self.detection_labels[idx] == class_id] + scores = self.detection_scores[id] + scores_filtered = scores[self.detection_labels[id] == class_id] inds = torch.argsort(scores_filtered, descending=True) det = det[inds] if len(det) > max_det: det = det[:max_det] - ious = box_iou(det, gt) + ious = compute_iou(det, gt) return ious def __evaluate_image_gt_no_preds( @@ -494,8 +481,9 @@ def _evaluate_image( ious: IoU results for image and class. """ - gt = self.groundtruth_boxes[idx] - det = self.detection_boxes[idx] + + gt = self.groundtruths[idx] + det = self.detections[idx] gt_label_mask = self.groundtruth_labels[idx] == class_id det_label_mask = self.detection_labels[idx] == class_id @@ -649,7 +637,7 @@ def _calculate(self, class_ids: List) -> Tuple[MAPMetricResults, MARMetricResult class_ids: List of label class Ids. """ - img_ids = range(len(self.groundtruth_boxes)) + img_ids = range(len(self.groundtruths)) max_detections = self.max_detection_thresholds[-1] area_ranges = self.bbox_area_ranges.values() @@ -825,10 +813,10 @@ def compute(self) -> dict: """ # move everything to CPU, as we are faster here - self.detections = [box.cpu() for box in self.detection_boxes] + self.detections = [box.cpu() for box in self.detections] self.detection_labels = [label.cpu() for label in self.detection_labels] self.detection_scores = [score.cpu() for score in self.detection_scores] - self.groundtruths = [box.cpu() for box in self.groundtruth_boxes] + self.groundtruths = [box.cpu() for box in self.groundtruths] self.groundtruth_labels = [label.cpu() for label in self.groundtruth_labels] classes = self._get_classes() From faa709dfaa53da4f7dd31197566d94a6a88d0a81 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Wed, 6 Apr 2022 16:25:07 +0200 Subject: [PATCH 018/141] fixing import --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 47ad4209f1c..307184875ec 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple import torch from torch import IntTensor, Tensor From 4c63b2be29b5f32a5a74dbb46185c3fc024188ae Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:14:04 +0200 Subject: [PATCH 019/141] map test --- tests/detection/test_map.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 3d1a347eac2..80e5b0322e0 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -23,7 +23,9 @@ Input = namedtuple("Input", ["preds", "target"]) -_inputs = Input( +_inputs_masks = Input(preds=[], target=[]) + +_inputs_bboxes = Input( preds=[ [ dict( @@ -156,7 +158,7 @@ def _compare_fn(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results All classes Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.706 Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.901 @@ -233,8 +235,8 @@ def test_map(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs.preds, - target=_inputs.target, + preds=_inputs_bboxes.preds, + target=_inputs_bboxes.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From 2ceaaa765cc5f8afe8ea9003fc89424025153d7d Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:42:25 +0200 Subject: [PATCH 020/141] fixes --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 307184875ec..6c8e63a9927 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -n # Copyright The PyTorch Lightning team. +# Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 93dbf7e23744987c93b4087efa88a665b0335980 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 15:42:27 +0200 Subject: [PATCH 021/141] test added (required native RLE encoding from COCO); fixed IOU --- tests/detection/test_map.py | 138 +++++++++++++++++++++++++++++- torchmetrics/detection/mean_ap.py | 39 ++++++++- 2 files changed, 171 insertions(+), 6 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 80e5b0322e0..3fd5882332f 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -14,8 +14,10 @@ from collections import namedtuple +import numpy as np import pytest import torch +from pycocotools import mask from tests.helpers.testers import MetricTester from torchmetrics.detection.mean_ap import MeanAveragePrecision @@ -23,7 +25,87 @@ Input = namedtuple("Input", ["preds", "target"]) -_inputs_masks = Input(preds=[], target=[]) +_inputs_masks = Input( + preds=[ + [ + dict( + masks=torch.Tensor( + mask.decode( + { + "size": [478, 640], + "counts": "VQi31m>0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR dict: } +def _compare_fn_segm(preds, target) -> dict: + """Comparison function for map implementation. + + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + + """ + return { + "map": torch.Tensor([0.352]), + "map_50": torch.Tensor([0.742]), + "map_75": torch.Tensor([0.252]), + "map_small": torch.Tensor([-1]), + "map_medium": torch.Tensor([-1]), + "map_large": torch.Tensor([0.352]), + "mar_1": torch.Tensor([0.35]), + "mar_10": torch.Tensor([0.35]), + "mar_100": torch.Tensor([0.35]), + "mar_small": torch.Tensor([-1]), + "mar_medium": torch.Tensor([-1]), + "mar_large": torch.Tensor([0.35]), + "map_per_class": torch.Tensor([0.4039604, -1.0, 0.3]), + "mar_100_per_class": torch.Tensor([0.4, -1.0, 0.3]), + } + + _pytest_condition = not (_TORCHVISION_AVAILABLE and _TORCHVISION_GREATER_EQUAL_0_8) @@ -231,7 +349,8 @@ class TestMAP(MetricTester): atol = 1e-1 @pytest.mark.parametrize("ddp", [False, True]) - def test_map(self, compute_on_cpu, ddp): + def test_map_bbox(self, compute_on_cpu, ddp): + """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, @@ -244,6 +363,21 @@ def test_map(self, compute_on_cpu, ddp): metric_args={"class_metrics": True, "compute_on_cpu": compute_on_cpu}, ) + @pytest.mark.parametrize("ddp", [False]) + def test_map_segm(self, ddp): + """Test modular implementation for correctness.""" + + self.run_class_metric_test( + ddp=ddp, + preds=_inputs_masks.preds, + target=_inputs_masks.target, + metric_class=MeanAveragePrecision, + sk_metric=_compare_fn_segm, + dist_sync_on_step=False, + check_batch=False, + metric_args={"class_metrics": True, "iou_type": "segm"}, + ) + # noinspection PyTypeChecker @pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 6c8e63a9927..7e616579d62 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -29,6 +29,24 @@ log = logging.getLogger(__name__) +def mask_area(input): + n_inputs = len(input) + + return input.reshape(n_inputs, -1).sum(1) + + +def compute_area(input, type="bbox"): + if len(input) == 0: + + return torch.Tensor([]).to(input.device) + + if type == "bbox": + + return box_area(input) + else: + return mask_area(input) + + class BaseMetricResults(dict): """Base metric class, that allows fields for pre-defined metrics.""" @@ -80,13 +98,23 @@ class COCOMetricResults(BaseMetricResults): ) +def _segm_iou(mask1, mask2): + + intersection = (mask1 * mask2).sum() + if intersection == 0: + return 0.0 + union = torch.logical_or(mask1, mask2).to(torch.int).sum() + return (intersection / union).unsqueeze(0) + + def segm_iou(inputs, targets, smooth=1): n_inputs = inputs.shape[0] n_targets = targets.shape[0] # flatten label and prediction tensors - inputs = inputs.view(n_inputs, -1).repeat_interleave(n_targets, 0) - targets = targets.view(n_targets, -1).repeat(n_inputs, 1) + + inputs = inputs.reshape(n_inputs, -1).repeat_interleave(n_targets, 0) + targets = targets.reshape(n_targets, -1).repeat(n_inputs, 1) # i1 * t1 # i1 * t2 @@ -408,6 +436,7 @@ def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: C det = det[:max_det] ious = compute_iou(det, gt) + return ious def __evaluate_image_gt_no_preds( @@ -506,13 +535,14 @@ def _evaluate_image( if gt.numel() == 0 and det.numel() == 0: return None - areas = box_area(gt) + areas = compute_area(gt, self.iou_type) ignore_area = (areas < area_range[0]) | (areas > area_range[1]) # sort dt highest score first, sort gt ignore last ignore_area_sorted, gtind = torch.sort(ignore_area.to(torch.uint8)) # Convert to uint8 temporarily and back to bool, because "Sort currently does not support bool dtype on CUDA" ignore_area_sorted = ignore_area_sorted.to(torch.bool) + gt = gt[gtind] scores = self.detection_scores[idx] scores_filtered = scores[det_label_mask] @@ -542,12 +572,13 @@ def _evaluate_image( gt_matches[idx_iou, m] = 1 # set unmatched detections outside of area range to ignore - det_areas = box_area(det) + det_areas = compute_area(det, self.iou_type) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.logical_or( det_ignore, torch.logical_and(det_matches == 0, torch.repeat_interleave(ar, nb_iou_thrs, 0)) ) + return { "dtMatches": det_matches.to(self.device), "gtMatches": gt_matches.to(self.device), From 41681bcc0fc5e0c1865c3d9f4dfe037d79f1673d Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:13:34 +0200 Subject: [PATCH 022/141] rebased and fixed GPU device --- tests/detection/test_map.py | 13 +++++++------ torchmetrics/detection/mean_ap.py | 10 ++++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 3fd5882332f..c2935f22544 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -107,7 +107,7 @@ ) -_inputs_bboxes = Input( +_inputs = Input( preds=[ [ dict( @@ -354,8 +354,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs_bboxes.preds, - target=_inputs_bboxes.target, + preds=_inputs.preds, + target=_inputs.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, @@ -364,7 +364,7 @@ def test_map_bbox(self, compute_on_cpu, ddp): ) @pytest.mark.parametrize("ddp", [False]) - def test_map_segm(self, ddp): + def test_map_segm(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( @@ -375,7 +375,7 @@ def test_map_segm(self, ddp): sk_metric=_compare_fn_segm, dist_sync_on_step=False, check_batch=False, - metric_args={"class_metrics": True, "iou_type": "segm"}, + metric_args={"class_metrics": True, "compute_on_cpu": compute_on_cpu, "iou_type": "segm"}, ) @@ -507,7 +507,7 @@ def test_segm_iou_empty_mask(): metric.update( [ dict( - masks=torch.randint(0, 1, (1, 10, 10)), + masks=torch.randint(0, 1, (1, 10, 10)).bool(), scores=torch.Tensor([0.5]), labels=torch.IntTensor([4]), ), @@ -516,6 +516,7 @@ def test_segm_iou_empty_mask(): dict(masks=torch.Tensor([]), labels=torch.IntTensor([])), ], ) + metric.compute() diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 7e616579d62..fdfdfa64379 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -469,6 +469,7 @@ def __evaluate_image_preds_no_gt( """Some predictions but no GT.""" # GTs nb_gt = 0 + gt_ignore = torch.zeros(nb_gt, dtype=torch.bool, device=self.device) # Detections @@ -480,7 +481,7 @@ def __evaluate_image_preds_no_gt( if len(det) > max_det: det = det[:max_det] nb_det = len(det) - det_areas = box_area(det).to(self.device) + det_areas = compute_area(det, type=self.iou_type).to(self.device) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.repeat_interleave(ar, nb_iou_thrs, 0) @@ -488,9 +489,9 @@ def __evaluate_image_preds_no_gt( return { "dtMatches": torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device), "gtMatches": torch.zeros((nb_iou_thrs, nb_gt), dtype=torch.bool, device=self.device), - "dtScores": scores_sorted, - "gtIgnore": gt_ignore, - "dtIgnore": det_ignore, + "dtScores": scores_sorted.to(self.device), + "gtIgnore": gt_ignore.to(self.device), + "dtIgnore": det_ignore.to(self.device), } def _evaluate_image( @@ -768,6 +769,7 @@ def __calculate_recall_precision_scores( img_eval_cls_bbox = [e for e in img_eval_cls_bbox if e is not None] if not img_eval_cls_bbox: return recall, precision, scores + det_scores = torch.cat([e["dtScores"][:max_det] for e in img_eval_cls_bbox]) # different sorting method generates slightly different results. From 9e41b80077d545543f076acb2ff21e5ee35a3ffa Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:17:20 +0200 Subject: [PATCH 023/141] docstring and smooth fix --- torchmetrics/detection/mean_ap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index fdfdfa64379..e3c7022e891 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -107,7 +107,7 @@ def _segm_iou(mask1, mask2): return (intersection / union).unsqueeze(0) -def segm_iou(inputs, targets, smooth=1): +def segm_iou(inputs, targets, smooth=1e-5): n_inputs = inputs.shape[0] n_targets = targets.shape[0] @@ -209,6 +209,8 @@ class MeanAveragePrecision(Metric): Args: box_format: Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. + iou_type: + Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. From 0270285c5fc676365d0c8b933cefc6db15d4cde2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:14 +0000 Subject: [PATCH 024/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 1 - torchmetrics/detection/mean_ap.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index c2935f22544..8fdd3bfb60c 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -313,7 +313,6 @@ def _compare_fn_segm(preds, target) -> dict: Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 - """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index e3c7022e891..1968ce25d8f 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -310,10 +310,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From c1c122c078eae13c8583f41d83a3958b0c9964b6 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:27:37 +0200 Subject: [PATCH 025/141] Added pycocotools as detection requirements for test --- requirements/detection.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/detection.txt b/requirements/detection.txt index cbec7260775..da90ea549b7 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1 +1,2 @@ torchvision>=0.8 +pycocotools From 0d5da9c5503396bb27bb39e4a1517ba0e0155cc6 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:28:18 +0200 Subject: [PATCH 026/141] pycocotools test req --- requirements/detection.txt | 1 - requirements/test.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/detection.txt b/requirements/detection.txt index da90ea549b7..cbec7260775 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1,2 +1 @@ torchvision>=0.8 -pycocotools diff --git a/requirements/test.txt b/requirements/test.txt index dfffc35433d..78c17457c98 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -16,3 +16,4 @@ fire cloudpickle>=1.3 scikit-learn>=0.24 +pycocotools From 792e97e15144d0f3b14f7f93ec2ee84b4404fcff Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Tue, 26 Apr 2022 13:45:36 +0200 Subject: [PATCH 027/141] moved test inputs to json file --- .../instance_segmentation_inputs.json | 30 +++++++++++ tests/detection/test_map.py | 50 ++++--------------- torchmetrics/detection/mean_ap.py | 10 ++-- 3 files changed, 46 insertions(+), 44 deletions(-) create mode 100644 tests/detection/instance_segmentation_inputs.json diff --git a/tests/detection/instance_segmentation_inputs.json b/tests/detection/instance_segmentation_inputs.json new file mode 100644 index 00000000000..85bc854c6fc --- /dev/null +++ b/tests/detection/instance_segmentation_inputs.json @@ -0,0 +1,30 @@ +{"preds":[{ + "size": [ + 478, + 640 + ], + "counts": "VQi31m>0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR Date: Mon, 31 Jan 2022 14:31:45 +0100 Subject: [PATCH 028/141] implementaed IOU with segmentation masks and MAP for instance segmentation --- torchmetrics/detection/mean_ap.py | 76 ++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 1ef6f5ae69c..520c0a580f3 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -80,6 +80,27 @@ class COCOMetricResults(BaseMetricResults): ) +def segm_iou(inputs, targets, smooth=1): + + n_inputs = inputs.shape[0] + n_targets = targets.shape[0] + # flatten label and prediction tensors + inputs = inputs.view(n_inputs, -1).repeat_interleave(n_targets, 0) + targets = targets.view(n_targets, -1).repeat(n_inputs, 1) + + # i1 * t1 + # i1 * t2 + # i2 * t1 + # i2 * t2 + + # intersection is equivalent to True Positive count + # union is the mutually inclusive area of all labels & predictions + intersections = (inputs * targets).sum(1, keepdims=True) + unions = (inputs + targets).sum(1, keepdims=True) + + return ((intersections + smooth) / (unions + smooth)).view(n_inputs, n_targets) + + def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]]) -> None: """Ensure the correct input format of `preds` and `targets`""" if not isinstance(preds, Sequence): @@ -223,10 +244,13 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruth_boxes: List[Tensor] groundtruth_labels: List[Tensor] + groundtruth_masks: List[Tensor] + detection_masks: List[Tensor] def __init__( self, box_format: str = "xyxy", + iou_type: str = "bbox", iou_thresholds: Optional[List[float]] = None, rec_thresholds: Optional[List[float]] = None, max_detection_thresholds: Optional[List[int]] = None, @@ -243,6 +267,7 @@ def __init__( ) allowed_box_formats = ("xyxy", "xywh", "cxcywh") + allowed_iou_types = ("segm", "bbox") if box_format not in allowed_box_formats: raise ValueError(f"Expected argument `box_format` to be one of {allowed_box_formats} but got {box_format}") self.box_format = box_format @@ -250,11 +275,14 @@ def __init__( self.rec_thresholds = rec_thresholds or torch.linspace(0.0, 1.00, round(1.00 / 0.01) + 1).tolist() max_det_thr, _ = torch.sort(IntTensor(max_detection_thresholds or [1, 10, 100])) self.max_detection_thresholds = max_det_thr.tolist() + if iou_type not in allowed_iou_types: + raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") + self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): @@ -264,8 +292,10 @@ def __init__( self.add_state("detection_boxes", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -316,12 +346,16 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detection_boxes.append(boxes) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) + if "masks" in item: + self.detection_masks.append(item["masks"]) for item in target: boxes = _fix_empty_tensors(item["boxes"]) boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") self.groundtruth_boxes.append(boxes) self.groundtruth_labels.append(item["labels"]) + if "masks" in item: + self.groundtruth_masks.append(item["masks"]) def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -329,7 +363,17 @@ def _get_classes(self) -> List: return torch.cat(self.detection_labels + self.groundtruth_labels).unique().tolist() return [] - def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: + def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: + if self.iou_type == "segm": + return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) + elif self.iou_type == "bbox": + return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + else: + raise Exception(f"IOU type {self.iou_type} is not supported") + + def _compute_iou_impl( + self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable + ) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -341,10 +385,13 @@ def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: max_det: Maximum number of evaluated detection bounding boxes """ - gt = self.groundtruth_boxes[idx] - det = self.detection_boxes[idx] - gt_label_mask = self.groundtruth_labels[idx] == class_id - det_label_mask = self.detection_labels[idx] == class_id + + gt = ground_truths[id] + det = detections[id] + + gt_label_mask = self.groundtruth_labels[id] == class_id + det_label_mask = self.detection_labels[id] == class_id + if len(gt_label_mask) == 0 or len(det_label_mask) == 0: return Tensor([]) gt = gt[gt_label_mask] @@ -360,8 +407,7 @@ def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: if len(det) > max_det: det = det[:max_det] - # generalized_box_iou - ious = box_iou(det, gt) + ious = compute_iou(det, gt) return ious def __evaluate_image_gt_no_preds( @@ -764,6 +810,14 @@ def compute(self) -> dict: - map_per_class: ``torch.Tensor`` (-1 if class metrics are disabled) - mar_100_per_class: ``torch.Tensor`` (-1 if class metrics are disabled) """ + + # move everything to CPU, as we are faster here + self.detections = [box.cpu() for box in self.detection_boxes] + self.detection_labels = [label.cpu() for label in self.detection_labels] + self.detection_scores = [score.cpu() for score in self.detection_scores] + self.groundtruths = [box.cpu() for box in self.groundtruth_boxes] + self.groundtruth_labels = [label.cpu() for label in self.groundtruth_labels] + classes = self._get_classes() precisions, recalls = self._calculate(classes) map_val, mar_val = self._summarize_results(precisions, recalls) From c04efa604ed6676bf878a9023ede8da14609ff3a Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Sat, 5 Feb 2022 11:48:30 +0100 Subject: [PATCH 029/141] rebase; ddp test still failing --- torchmetrics/detection/mean_ap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 520c0a580f3..6668e92fcbd 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -844,4 +844,5 @@ def compute(self) -> dict: metrics.update(mar_val) metrics.map_per_class = map_per_class_values metrics[f"mar_{self.max_detection_thresholds[-1]}_per_class"] = mar_max_dets_per_class_values + return metrics From 14503353cb8e12a4b7fc5cf06b2164c5a386c00c Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:30:56 +0100 Subject: [PATCH 030/141] checking pr --- torchmetrics/detection/mean_ap.py | 73 ++++++++++++++++++------------- torchmetrics/metric.py | 1 + 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 6668e92fcbd..fd04c67f67c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -# Copyright The PyTorch Lightning team. +n # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -101,7 +101,9 @@ def segm_iou(inputs, targets, smooth=1): return ((intersections + smooth) / (unions + smooth)).view(n_inputs, n_targets) -def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]]) -> None: +def _input_validator( + preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]], iou_type: str = "bbox" +) -> None: """Ensure the correct input format of `preds` and `targets`""" if not isinstance(preds, Sequence): raise ValueError("Expected argument `preds` to be of type Sequence") @@ -109,37 +111,38 @@ def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[ raise ValueError("Expected argument `target` to be of type Sequence") if len(preds) != len(targets): raise ValueError("Expected argument `preds` and `target` to have the same length") + iou_attribute = "boxes" if iou_type == "bbox" else "masks" - for k in ["boxes", "scores", "labels"]: + for k in [iou_attribute, "scores", "labels"]: if any(k not in p for p in preds): raise ValueError(f"Expected all dicts in `preds` to contain the `{k}` key") - for k in ["boxes", "labels"]: + for k in [iou_attribute, "labels"]: if any(k not in p for p in targets): raise ValueError(f"Expected all dicts in `target` to contain the `{k}` key") - if any(type(pred["boxes"]) is not Tensor for pred in preds): - raise ValueError("Expected all boxes in `preds` to be of type Tensor") + if any(type(pred[iou_attribute]) is not Tensor for pred in preds): + raise ValueError(f"Expected all {iou_attribute} in `preds` to be of type Tensor") if any(type(pred["scores"]) is not Tensor for pred in preds): raise ValueError("Expected all scores in `preds` to be of type Tensor") if any(type(pred["labels"]) is not Tensor for pred in preds): raise ValueError("Expected all labels in `preds` to be of type Tensor") - if any(type(target["boxes"]) is not Tensor for target in targets): - raise ValueError("Expected all boxes in `target` to be of type Tensor") + if any(type(target[iou_attribute]) is not Tensor for target in targets): + raise ValueError(f"Expected all {iou_attribute} in `target` to be of type Tensor") if any(type(target["labels"]) is not Tensor for target in targets): raise ValueError("Expected all labels in `target` to be of type Tensor") for i, item in enumerate(targets): - if item["boxes"].size(0) != item["labels"].size(0): + if item[iou_attribute].size(0) != item["labels"].size(0): raise ValueError( - f"Input boxes and labels of sample {i} in targets have a" - f" different length (expected {item['boxes'].size(0)} labels, got {item['labels'].size(0)})" + f"Input {iou_attribute} and labels of sample {i} in targets have a" + f" different length (expected {item[iou_attribute].size(0)} labels, got {item['labels'].size(0)})" ) for i, item in enumerate(preds): - if not (item["boxes"].size(0) == item["labels"].size(0) == item["scores"].size(0)): + if not (item[iou_attribute].size(0) == item["labels"].size(0) == item["scores"].size(0)): raise ValueError( - f"Input boxes, labels and scores of sample {i} in predictions have a" - f" different length (expected {item['boxes'].size(0)} labels and scores," + f"Input {iou_attribute}, labels and scores of sample {i} in predictions have a" + f" different length (expected {item[iou_attribute].size(0)} labels and scores," f" got {item['labels'].size(0)} labels and {item['scores'].size(0)})" ) @@ -292,9 +295,9 @@ def __init__( self.add_state("detection_boxes", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore @@ -341,21 +344,29 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] _input_validator(preds, target) for item in preds: - boxes = _fix_empty_tensors(item["boxes"]) - boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + boxes, masks = self._get_safe_item_values(item) self.detection_boxes.append(boxes) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - if "masks" in item: - self.detection_masks.append(item["masks"]) + self.detection_masks.append(masks) for item in target: - boxes = _fix_empty_tensors(item["boxes"]) - boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + boxes, masks = self._get_safe_item_values(item) self.groundtruth_boxes.append(boxes) self.groundtruth_labels.append(item["labels"]) - if "masks" in item: - self.groundtruth_masks.append(item["masks"]) + self.groundtruth_masks.append(masks) + + def _get_safe_item_values(self, item): + if self.iou_type == "bbox": + boxes = _fix_empty_tensors(item["boxes"]) + boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + masks = _fix_empty_tensors(torch.Tensor()) + elif self.iou_type == "masks": + masks = _fix_empty_tensors(item["masks"]) + boxes = _fix_empty_tensors(torch.Tensor()) + else: + raise Exception(f"IOU type {self.iou_type} is not supported") + return boxes, masks def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -364,12 +375,14 @@ def _get_classes(self) -> List: return [] def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: - if self.iou_type == "segm": - return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) - elif self.iou_type == "bbox": - return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) - else: - raise Exception(f"IOU type {self.iou_type} is not supported") + return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + + # if self.iou_type == "segm": + # return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) + # elif self.iou_type == "bbox": + + # else: + # raise Exception(f"IOU type {self.iou_type} is not supported") def _compute_iou_impl( self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable @@ -407,7 +420,7 @@ def _compute_iou_impl( if len(det) > max_det: det = det[:max_det] - ious = compute_iou(det, gt) + ious = box_iou(det, gt) return ious def __evaluate_image_gt_no_preds( diff --git a/torchmetrics/metric.py b/torchmetrics/metric.py index 5e4484ef119..d0d50c862a4 100644 --- a/torchmetrics/metric.py +++ b/torchmetrics/metric.py @@ -257,6 +257,7 @@ def _sync_dist(self, dist_sync_fn: Callable = gather_all_tensors, process_group: for attr, reduction_fn in self._reductions.items(): # pre-processing ops (stack or flatten for inputs) + if isinstance(output_dict[attr][0], Tensor): output_dict[attr] = torch.stack(output_dict[attr]) elif isinstance(output_dict[attr][0], list): From 54c96747f616bc2a6879eec35261a66f54fc9b18 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:52:44 +0100 Subject: [PATCH 031/141] clean API interface; working on tests for iou_type SEGM --- tests/detection/test_map.py | 20 ++++++++++ torchmetrics/detection/mean_ap.py | 66 +++++++++++++------------------ 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index f63e3d80814..3d1a347eac2 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -363,6 +363,26 @@ def test_missing_gt(): assert result["map"] < 1, "MAP cannot be 1, as there is an image with no ground truth, but some predictions." +@pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") +def test_segm_iou_empty_mask(): + """Test empty ground truths.""" + metric = MeanAveragePrecision(iou_type="segm") + + metric.update( + [ + dict( + masks=torch.randint(0, 1, (1, 10, 10)), + scores=torch.Tensor([0.5]), + labels=torch.IntTensor([4]), + ), + ], + [ + dict(masks=torch.Tensor([]), labels=torch.IntTensor([])), + ], + ) + metric.compute() + + @pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") def test_error_on_wrong_input(): """Test class input validation.""" diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index fd04c67f67c..47ad4209f1c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -242,13 +242,11 @@ class MeanAveragePrecision(Metric): If ``class_metrics`` is not a boolean """ - detection_boxes: List[Tensor] + detections: List[Tensor] detection_scores: List[Tensor] detection_labels: List[Tensor] - groundtruth_boxes: List[Tensor] + groundtruths: List[Tensor] groundtruth_labels: List[Tensor] - groundtruth_masks: List[Tensor] - detection_masks: List[Tensor] def __init__( self, @@ -292,13 +290,11 @@ def __init__( raise ValueError("Expected argument `class_metrics` to be a boolean") self.class_metrics = class_metrics - self.add_state("detection_boxes", default=[], dist_reduce_fx=None) + self.add_state("detections", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) + self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -341,32 +337,29 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] ValueError: If any score is not type float and of length 1 """ - _input_validator(preds, target) + _input_validator(preds, target, iou_type=self.iou_type) for item in preds: - boxes, masks = self._get_safe_item_values(item) - self.detection_boxes.append(boxes) + detections = self._get_safe_item_values(item) + self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - self.detection_masks.append(masks) for item in target: - boxes, masks = self._get_safe_item_values(item) - self.groundtruth_boxes.append(boxes) + groundtruths = self._get_safe_item_values(item) + self.groundtruths.append(groundtruths) self.groundtruth_labels.append(item["labels"]) - self.groundtruth_masks.append(masks) def _get_safe_item_values(self, item): if self.iou_type == "bbox": boxes = _fix_empty_tensors(item["boxes"]) boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") - masks = _fix_empty_tensors(torch.Tensor()) - elif self.iou_type == "masks": + return boxes + elif self.iou_type == "segm": masks = _fix_empty_tensors(item["masks"]) - boxes = _fix_empty_tensors(torch.Tensor()) + return masks else: raise Exception(f"IOU type {self.iou_type} is not supported") - return boxes, masks def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -375,18 +368,11 @@ def _get_classes(self) -> List: return [] def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: - return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + iou_func = box_iou if self.iou_type == "bbox" else segm_iou - # if self.iou_type == "segm": - # return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) - # elif self.iou_type == "bbox": + return self._compute_iou_impl(id, class_id, max_det, iou_func) - # else: - # raise Exception(f"IOU type {self.iou_type} is not supported") - - def _compute_iou_impl( - self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable - ) -> Tensor: + def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: Callable) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -399,8 +385,9 @@ def _compute_iou_impl( Maximum number of evaluated detection bounding boxes """ - gt = ground_truths[id] - det = detections[id] + # if self.iou_type == "bbox": + gt = self.groundtruths[id] + det = self.detections[id] gt_label_mask = self.groundtruth_labels[id] == class_id det_label_mask = self.detection_labels[id] == class_id @@ -413,14 +400,14 @@ def _compute_iou_impl( return Tensor([]) # Sort by scores and use only max detections - scores = self.detection_scores[idx] - scores_filtered = scores[self.detection_labels[idx] == class_id] + scores = self.detection_scores[id] + scores_filtered = scores[self.detection_labels[id] == class_id] inds = torch.argsort(scores_filtered, descending=True) det = det[inds] if len(det) > max_det: det = det[:max_det] - ious = box_iou(det, gt) + ious = compute_iou(det, gt) return ious def __evaluate_image_gt_no_preds( @@ -494,8 +481,9 @@ def _evaluate_image( ious: IoU results for image and class. """ - gt = self.groundtruth_boxes[idx] - det = self.detection_boxes[idx] + + gt = self.groundtruths[idx] + det = self.detections[idx] gt_label_mask = self.groundtruth_labels[idx] == class_id det_label_mask = self.detection_labels[idx] == class_id @@ -649,7 +637,7 @@ def _calculate(self, class_ids: List) -> Tuple[MAPMetricResults, MARMetricResult class_ids: List of label class Ids. """ - img_ids = range(len(self.groundtruth_boxes)) + img_ids = range(len(self.groundtruths)) max_detections = self.max_detection_thresholds[-1] area_ranges = self.bbox_area_ranges.values() @@ -825,10 +813,10 @@ def compute(self) -> dict: """ # move everything to CPU, as we are faster here - self.detections = [box.cpu() for box in self.detection_boxes] + self.detections = [box.cpu() for box in self.detections] self.detection_labels = [label.cpu() for label in self.detection_labels] self.detection_scores = [score.cpu() for score in self.detection_scores] - self.groundtruths = [box.cpu() for box in self.groundtruth_boxes] + self.groundtruths = [box.cpu() for box in self.groundtruths] self.groundtruth_labels = [label.cpu() for label in self.groundtruth_labels] classes = self._get_classes() From 9e24e441832f57903149e42d6efb6c710df8aa0d Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Wed, 6 Apr 2022 16:25:07 +0200 Subject: [PATCH 032/141] fixing import --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 47ad4209f1c..307184875ec 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple import torch from torch import IntTensor, Tensor From 73036c8e19fc3dfb99d93eccb6f6ad5952f919f5 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:14:04 +0200 Subject: [PATCH 033/141] map test --- tests/detection/test_map.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 3d1a347eac2..80e5b0322e0 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -23,7 +23,9 @@ Input = namedtuple("Input", ["preds", "target"]) -_inputs = Input( +_inputs_masks = Input(preds=[], target=[]) + +_inputs_bboxes = Input( preds=[ [ dict( @@ -156,7 +158,7 @@ def _compare_fn(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results All classes Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.706 Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.901 @@ -233,8 +235,8 @@ def test_map(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs.preds, - target=_inputs.target, + preds=_inputs_bboxes.preds, + target=_inputs_bboxes.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From 251e8b48fd66b4fd6080943356e383714efdec3d Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:42:25 +0200 Subject: [PATCH 034/141] fixes --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 307184875ec..6c8e63a9927 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -n # Copyright The PyTorch Lightning team. +# Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 51b3b72767a13ef0a840890e4064ee460d2b049e Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 15:42:27 +0200 Subject: [PATCH 035/141] test added (required native RLE encoding from COCO); fixed IOU --- tests/detection/test_map.py | 138 +++++++++++++++++++++++++++++- torchmetrics/detection/mean_ap.py | 39 ++++++++- 2 files changed, 171 insertions(+), 6 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 80e5b0322e0..3fd5882332f 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -14,8 +14,10 @@ from collections import namedtuple +import numpy as np import pytest import torch +from pycocotools import mask from tests.helpers.testers import MetricTester from torchmetrics.detection.mean_ap import MeanAveragePrecision @@ -23,7 +25,87 @@ Input = namedtuple("Input", ["preds", "target"]) -_inputs_masks = Input(preds=[], target=[]) +_inputs_masks = Input( + preds=[ + [ + dict( + masks=torch.Tensor( + mask.decode( + { + "size": [478, 640], + "counts": "VQi31m>0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR dict: } +def _compare_fn_segm(preds, target) -> dict: + """Comparison function for map implementation. + + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + + """ + return { + "map": torch.Tensor([0.352]), + "map_50": torch.Tensor([0.742]), + "map_75": torch.Tensor([0.252]), + "map_small": torch.Tensor([-1]), + "map_medium": torch.Tensor([-1]), + "map_large": torch.Tensor([0.352]), + "mar_1": torch.Tensor([0.35]), + "mar_10": torch.Tensor([0.35]), + "mar_100": torch.Tensor([0.35]), + "mar_small": torch.Tensor([-1]), + "mar_medium": torch.Tensor([-1]), + "mar_large": torch.Tensor([0.35]), + "map_per_class": torch.Tensor([0.4039604, -1.0, 0.3]), + "mar_100_per_class": torch.Tensor([0.4, -1.0, 0.3]), + } + + _pytest_condition = not (_TORCHVISION_AVAILABLE and _TORCHVISION_GREATER_EQUAL_0_8) @@ -231,7 +349,8 @@ class TestMAP(MetricTester): atol = 1e-1 @pytest.mark.parametrize("ddp", [False, True]) - def test_map(self, compute_on_cpu, ddp): + def test_map_bbox(self, compute_on_cpu, ddp): + """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, @@ -244,6 +363,21 @@ def test_map(self, compute_on_cpu, ddp): metric_args={"class_metrics": True, "compute_on_cpu": compute_on_cpu}, ) + @pytest.mark.parametrize("ddp", [False]) + def test_map_segm(self, ddp): + """Test modular implementation for correctness.""" + + self.run_class_metric_test( + ddp=ddp, + preds=_inputs_masks.preds, + target=_inputs_masks.target, + metric_class=MeanAveragePrecision, + sk_metric=_compare_fn_segm, + dist_sync_on_step=False, + check_batch=False, + metric_args={"class_metrics": True, "iou_type": "segm"}, + ) + # noinspection PyTypeChecker @pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 6c8e63a9927..7e616579d62 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -29,6 +29,24 @@ log = logging.getLogger(__name__) +def mask_area(input): + n_inputs = len(input) + + return input.reshape(n_inputs, -1).sum(1) + + +def compute_area(input, type="bbox"): + if len(input) == 0: + + return torch.Tensor([]).to(input.device) + + if type == "bbox": + + return box_area(input) + else: + return mask_area(input) + + class BaseMetricResults(dict): """Base metric class, that allows fields for pre-defined metrics.""" @@ -80,13 +98,23 @@ class COCOMetricResults(BaseMetricResults): ) +def _segm_iou(mask1, mask2): + + intersection = (mask1 * mask2).sum() + if intersection == 0: + return 0.0 + union = torch.logical_or(mask1, mask2).to(torch.int).sum() + return (intersection / union).unsqueeze(0) + + def segm_iou(inputs, targets, smooth=1): n_inputs = inputs.shape[0] n_targets = targets.shape[0] # flatten label and prediction tensors - inputs = inputs.view(n_inputs, -1).repeat_interleave(n_targets, 0) - targets = targets.view(n_targets, -1).repeat(n_inputs, 1) + + inputs = inputs.reshape(n_inputs, -1).repeat_interleave(n_targets, 0) + targets = targets.reshape(n_targets, -1).repeat(n_inputs, 1) # i1 * t1 # i1 * t2 @@ -408,6 +436,7 @@ def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: C det = det[:max_det] ious = compute_iou(det, gt) + return ious def __evaluate_image_gt_no_preds( @@ -506,13 +535,14 @@ def _evaluate_image( if gt.numel() == 0 and det.numel() == 0: return None - areas = box_area(gt) + areas = compute_area(gt, self.iou_type) ignore_area = (areas < area_range[0]) | (areas > area_range[1]) # sort dt highest score first, sort gt ignore last ignore_area_sorted, gtind = torch.sort(ignore_area.to(torch.uint8)) # Convert to uint8 temporarily and back to bool, because "Sort currently does not support bool dtype on CUDA" ignore_area_sorted = ignore_area_sorted.to(torch.bool) + gt = gt[gtind] scores = self.detection_scores[idx] scores_filtered = scores[det_label_mask] @@ -542,12 +572,13 @@ def _evaluate_image( gt_matches[idx_iou, m] = 1 # set unmatched detections outside of area range to ignore - det_areas = box_area(det) + det_areas = compute_area(det, self.iou_type) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.logical_or( det_ignore, torch.logical_and(det_matches == 0, torch.repeat_interleave(ar, nb_iou_thrs, 0)) ) + return { "dtMatches": det_matches.to(self.device), "gtMatches": gt_matches.to(self.device), From 79ccda72b8f7dc10017e140e69e36a63dd4539d7 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:13:34 +0200 Subject: [PATCH 036/141] rebased and fixed GPU device --- tests/detection/test_map.py | 13 +++++++------ torchmetrics/detection/mean_ap.py | 10 ++++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 3fd5882332f..c2935f22544 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -107,7 +107,7 @@ ) -_inputs_bboxes = Input( +_inputs = Input( preds=[ [ dict( @@ -354,8 +354,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs_bboxes.preds, - target=_inputs_bboxes.target, + preds=_inputs.preds, + target=_inputs.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, @@ -364,7 +364,7 @@ def test_map_bbox(self, compute_on_cpu, ddp): ) @pytest.mark.parametrize("ddp", [False]) - def test_map_segm(self, ddp): + def test_map_segm(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( @@ -375,7 +375,7 @@ def test_map_segm(self, ddp): sk_metric=_compare_fn_segm, dist_sync_on_step=False, check_batch=False, - metric_args={"class_metrics": True, "iou_type": "segm"}, + metric_args={"class_metrics": True, "compute_on_cpu": compute_on_cpu, "iou_type": "segm"}, ) @@ -507,7 +507,7 @@ def test_segm_iou_empty_mask(): metric.update( [ dict( - masks=torch.randint(0, 1, (1, 10, 10)), + masks=torch.randint(0, 1, (1, 10, 10)).bool(), scores=torch.Tensor([0.5]), labels=torch.IntTensor([4]), ), @@ -516,6 +516,7 @@ def test_segm_iou_empty_mask(): dict(masks=torch.Tensor([]), labels=torch.IntTensor([])), ], ) + metric.compute() diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 7e616579d62..fdfdfa64379 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -469,6 +469,7 @@ def __evaluate_image_preds_no_gt( """Some predictions but no GT.""" # GTs nb_gt = 0 + gt_ignore = torch.zeros(nb_gt, dtype=torch.bool, device=self.device) # Detections @@ -480,7 +481,7 @@ def __evaluate_image_preds_no_gt( if len(det) > max_det: det = det[:max_det] nb_det = len(det) - det_areas = box_area(det).to(self.device) + det_areas = compute_area(det, type=self.iou_type).to(self.device) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.repeat_interleave(ar, nb_iou_thrs, 0) @@ -488,9 +489,9 @@ def __evaluate_image_preds_no_gt( return { "dtMatches": torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device), "gtMatches": torch.zeros((nb_iou_thrs, nb_gt), dtype=torch.bool, device=self.device), - "dtScores": scores_sorted, - "gtIgnore": gt_ignore, - "dtIgnore": det_ignore, + "dtScores": scores_sorted.to(self.device), + "gtIgnore": gt_ignore.to(self.device), + "dtIgnore": det_ignore.to(self.device), } def _evaluate_image( @@ -768,6 +769,7 @@ def __calculate_recall_precision_scores( img_eval_cls_bbox = [e for e in img_eval_cls_bbox if e is not None] if not img_eval_cls_bbox: return recall, precision, scores + det_scores = torch.cat([e["dtScores"][:max_det] for e in img_eval_cls_bbox]) # different sorting method generates slightly different results. From 67937153c2fd364ca71a3fe73d7dca017848fc13 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:17:20 +0200 Subject: [PATCH 037/141] docstring and smooth fix --- torchmetrics/detection/mean_ap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index fdfdfa64379..e3c7022e891 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -107,7 +107,7 @@ def _segm_iou(mask1, mask2): return (intersection / union).unsqueeze(0) -def segm_iou(inputs, targets, smooth=1): +def segm_iou(inputs, targets, smooth=1e-5): n_inputs = inputs.shape[0] n_targets = targets.shape[0] @@ -209,6 +209,8 @@ class MeanAveragePrecision(Metric): Args: box_format: Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. + iou_type: + Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. From a5f2c37642018d364ed5320858a853407e3e3def Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:14 +0000 Subject: [PATCH 038/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 1 - torchmetrics/detection/mean_ap.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index c2935f22544..8fdd3bfb60c 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -313,7 +313,6 @@ def _compare_fn_segm(preds, target) -> dict: Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 - """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index e3c7022e891..1968ce25d8f 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -310,10 +310,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 79b9e68ca77e563b06c2be798c9a60262e2e1c2d Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:27:37 +0200 Subject: [PATCH 039/141] Added pycocotools as detection requirements for test --- requirements/detection.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/detection.txt b/requirements/detection.txt index cbec7260775..da90ea549b7 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1 +1,2 @@ torchvision>=0.8 +pycocotools From 0d3e99d746aff39b4d160240f44bb6c45a25440a Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:28:18 +0200 Subject: [PATCH 040/141] pycocotools test req --- requirements/detection.txt | 1 - requirements/test.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/detection.txt b/requirements/detection.txt index da90ea549b7..cbec7260775 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1,2 +1 @@ torchvision>=0.8 -pycocotools diff --git a/requirements/test.txt b/requirements/test.txt index dfffc35433d..78c17457c98 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -16,3 +16,4 @@ fire cloudpickle>=1.3 scikit-learn>=0.24 +pycocotools From 27e682a612261b6bb9c2cd057d156367a8efb381 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Tue, 26 Apr 2022 13:45:36 +0200 Subject: [PATCH 041/141] moved test inputs to json file --- .../instance_segmentation_inputs.json | 30 +++++++++++ tests/detection/test_map.py | 50 ++++--------------- torchmetrics/detection/mean_ap.py | 10 ++-- 3 files changed, 46 insertions(+), 44 deletions(-) create mode 100644 tests/detection/instance_segmentation_inputs.json diff --git a/tests/detection/instance_segmentation_inputs.json b/tests/detection/instance_segmentation_inputs.json new file mode 100644 index 00000000000..85bc854c6fc --- /dev/null +++ b/tests/detection/instance_segmentation_inputs.json @@ -0,0 +1,30 @@ +{"preds":[{ + "size": [ + 478, + 640 + ], + "counts": "VQi31m>0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR Date: Tue, 26 Apr 2022 11:50:46 +0000 Subject: [PATCH 042/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/detection/mean_ap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index e3c7022e891..1968ce25d8f 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -310,10 +310,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 00923d61f9e5062c0a2b9916280a6edcd765df1b Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Mon, 31 Jan 2022 14:31:45 +0100 Subject: [PATCH 043/141] implementaed IOU with segmentation masks and MAP for instance segmentation --- torchmetrics/detection/mean_ap.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 61b36e7fddd..13b542ebbe7 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -98,16 +98,7 @@ class COCOMetricResults(BaseMetricResults): ) -def _segm_iou(mask1, mask2): - - intersection = (mask1 * mask2).sum() - if intersection == 0: - return 0.0 - union = torch.logical_or(mask1, mask2).to(torch.int).sum() - return (intersection / union).unsqueeze(0) - - -def segm_iou(inputs, targets, smooth=1e-5): +def segm_iou(inputs, targets, smooth=1): n_inputs = inputs.shape[0] n_targets = targets.shape[0] @@ -277,6 +268,8 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruths: List[Tensor] groundtruth_labels: List[Tensor] + groundtruth_masks: List[Tensor] + detection_masks: List[Tensor] def __init__( self, @@ -325,6 +318,7 @@ def __init__( self.add_state("detection_labels", default=[], dist_reduce_fx=None) self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -374,6 +368,8 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) + if "masks" in item: + self.detection_masks.append(item["masks"]) for item in target: groundtruths = self._get_safe_item_values(item) From 0871426e33983f044ba0956f24b1a4a88a8a1b4a Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:30:56 +0100 Subject: [PATCH 044/141] checking pr --- torchmetrics/detection/mean_ap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 13b542ebbe7..500ce9d2000 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -318,6 +318,7 @@ def __init__( self.add_state("detection_labels", default=[], dist_reduce_fx=None) self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore @@ -368,8 +369,7 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - if "masks" in item: - self.detection_masks.append(item["masks"]) + self.detection_masks.append(masks) for item in target: groundtruths = self._get_safe_item_values(item) From d740426cb459e4ee94a4b3a6781de1870624fe46 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:52:44 +0100 Subject: [PATCH 045/141] clean API interface; working on tests for iou_type SEGM --- torchmetrics/detection/mean_ap.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 500ce9d2000..474404c6fdd 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -268,8 +268,6 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruths: List[Tensor] groundtruth_labels: List[Tensor] - groundtruth_masks: List[Tensor] - detection_masks: List[Tensor] def __init__( self, @@ -318,8 +316,6 @@ def __init__( self.add_state("detection_labels", default=[], dist_reduce_fx=None) self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -369,7 +365,6 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - self.detection_masks.append(masks) for item in target: groundtruths = self._get_safe_item_values(item) From d3b6d5567c4bfe13718b817effb50a2f703f9c0e Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:14:04 +0200 Subject: [PATCH 046/141] map test --- tests/detection/test_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 6920904afc6..6d0a0ee5f41 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -325,8 +325,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs.preds, - target=_inputs.target, + preds=_inputs_bboxes.preds, + target=_inputs_bboxes.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From 29e74aa89a87988ff575418a2607ec2c3105c60d Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 15:42:27 +0200 Subject: [PATCH 047/141] test added (required native RLE encoding from COCO); fixed IOU --- tests/detection/test_map.py | 30 +++++++++++++++++------------- torchmetrics/detection/mean_ap.py | 9 +++++++++ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 6d0a0ee5f41..64c6e0c0fe0 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -272,19 +272,23 @@ def _compare_fn(preds, target) -> dict: def _compare_fn_segm(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results - Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 - Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 - Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 - Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 - Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 - Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 - Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 - Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + <<<<<<< HEAD + ======= + + >>>>>>> test added (required native RLE encoding from COCO); fixed IOU """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 474404c6fdd..14349f50849 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -98,6 +98,15 @@ class COCOMetricResults(BaseMetricResults): ) +def _segm_iou(mask1, mask2): + + intersection = (mask1 * mask2).sum() + if intersection == 0: + return 0.0 + union = torch.logical_or(mask1, mask2).to(torch.int).sum() + return (intersection / union).unsqueeze(0) + + def segm_iou(inputs, targets, smooth=1): n_inputs = inputs.shape[0] From 04be6483a6e54f40ec0cee423cf087b949c7341c Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:13:34 +0200 Subject: [PATCH 048/141] rebased and fixed GPU device --- tests/detection/test_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 64c6e0c0fe0..e096664b4f6 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -329,8 +329,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs_bboxes.preds, - target=_inputs_bboxes.target, + preds=_inputs.preds, + target=_inputs.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From 0334dd1eeaae18298d40a4ca53d2bcaee6205b08 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:17:20 +0200 Subject: [PATCH 049/141] docstring and smooth fix --- torchmetrics/detection/mean_ap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 14349f50849..e3c7022e891 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -107,7 +107,7 @@ def _segm_iou(mask1, mask2): return (intersection / union).unsqueeze(0) -def segm_iou(inputs, targets, smooth=1): +def segm_iou(inputs, targets, smooth=1e-5): n_inputs = inputs.shape[0] n_targets = targets.shape[0] @@ -211,7 +211,7 @@ class MeanAveragePrecision(Metric): Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. iou_type: Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. - iou_thresholds:`` + iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. rec_thresholds: From f9ce2ed9c3e7ce624ece304008133cd7b392a998 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:14 +0000 Subject: [PATCH 050/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 31 ++++++++++++++----------------- torchmetrics/detection/mean_ap.py | 8 ++++---- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index e096664b4f6..b2e1ee56b75 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -272,23 +272,20 @@ def _compare_fn(preds, target) -> dict: def _compare_fn_segm(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results - Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 - Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 - Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 - Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 - Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 - Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 - Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 - Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 - <<<<<<< HEAD - ======= - - >>>>>>> test added (required native RLE encoding from COCO); fixed IOU + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index e3c7022e891..1968ce25d8f 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -310,10 +310,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 4d49943bdeb9ebe8dbde40c6ec16510b40787dd2 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:27:37 +0200 Subject: [PATCH 051/141] Added pycocotools as detection requirements for test --- requirements/detection.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/detection.txt b/requirements/detection.txt index cbec7260775..da90ea549b7 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1 +1,2 @@ torchvision>=0.8 +pycocotools From 86897fca32fc3ce79157843caa3b3726f18ca3f4 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:28:18 +0200 Subject: [PATCH 052/141] pycocotools test req --- requirements/detection.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/detection.txt b/requirements/detection.txt index da90ea549b7..cbec7260775 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1,2 +1 @@ torchvision>=0.8 -pycocotools From b542ed94980ca9f5eb05b895212deb61af40dff0 Mon Sep 17 00:00:00 2001 From: Gianluca Scarpellini Date: Tue, 26 Apr 2022 13:49:25 +0000 Subject: [PATCH 053/141] Update tests/detection/test_map.py Co-authored-by: Nicki Skafte Detlefsen --- tests/detection/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 6920904afc6..86748a3cd8c 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -212,7 +212,7 @@ def _compare_fn(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results All classes Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.706 Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.901 From 3c7b4045c0ce69a8c901ad39c4f9b3918d889ceb Mon Sep 17 00:00:00 2001 From: Gianluca Scarpellini Date: Tue, 26 Apr 2022 13:49:31 +0000 Subject: [PATCH 054/141] Update tests/detection/test_map.py Co-authored-by: Nicki Skafte Detlefsen --- tests/detection/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 86748a3cd8c..3af9ad2e9b4 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -272,7 +272,7 @@ def _compare_fn(preds, target) -> dict: def _compare_fn_segm(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 From 71f836d7b8bf6573fefc3dfe662cdf9772ce855b Mon Sep 17 00:00:00 2001 From: Gianluca Scarpellini Date: Tue, 26 Apr 2022 13:49:41 +0000 Subject: [PATCH 055/141] Update tests/detection/test_map.py Co-authored-by: Nicki Skafte Detlefsen --- tests/detection/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 3af9ad2e9b4..6287a32b09a 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -270,7 +270,7 @@ def _compare_fn(preds, target) -> dict: def _compare_fn_segm(preds, target) -> dict: - """Comparison function for map implementation. + """Comparison function for map implementation for instance segmentation. Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 From a5438c3a657c6d1b3a7fdc6d9a64ea639edcf91e Mon Sep 17 00:00:00 2001 From: Gianluca Scarpellini Date: Tue, 26 Apr 2022 14:39:31 +0000 Subject: [PATCH 056/141] Update tests/detection/test_map.py Co-authored-by: Nicki Skafte Detlefsen --- tests/detection/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 6287a32b09a..7c624b8f7d4 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -334,7 +334,7 @@ def test_map_bbox(self, compute_on_cpu, ddp): metric_args={"class_metrics": True, "compute_on_cpu": compute_on_cpu}, ) - @pytest.mark.parametrize("ddp", [False]) + @pytest.mark.parametrize("ddp", [False, True]) def test_map_segm(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" From 1ce261dea9c22be06e2fa9420f038c14f8c1b641 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Thu, 28 Apr 2022 09:46:52 +0200 Subject: [PATCH 057/141] Update torchmetrics/metric.py --- torchmetrics/metric.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torchmetrics/metric.py b/torchmetrics/metric.py index d0d50c862a4..5e4484ef119 100644 --- a/torchmetrics/metric.py +++ b/torchmetrics/metric.py @@ -257,7 +257,6 @@ def _sync_dist(self, dist_sync_fn: Callable = gather_all_tensors, process_group: for attr, reduction_fn in self._reductions.items(): # pre-processing ops (stack or flatten for inputs) - if isinstance(output_dict[attr][0], Tensor): output_dict[attr] = torch.stack(output_dict[attr]) elif isinstance(output_dict[attr][0], list): From 255f443f07f61a36e1f840ace0c94ec77018f11e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 13:39:55 +0000 Subject: [PATCH 058/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 2458ffa3e2e..041f927b5d4 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -17,8 +17,8 @@ import numpy as np import pytest -from torch import Tensor from pycocotools import mask +from torch import Tensor from tests.helpers.testers import MetricTester from torchmetrics.detection.mean_ap import MeanAveragePrecision From f53257d0f87c2adc596910c42924c8698eb851bc Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Mon, 2 May 2022 15:44:11 +0200 Subject: [PATCH 059/141] missing torch --- tests/detection/test_map.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 041f927b5d4..1610d976bd9 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -18,6 +18,7 @@ import numpy as np import pytest from pycocotools import mask +import torch from torch import Tensor from tests.helpers.testers import MetricTester From eb2226656002a799755c4adabb029a757cef08a6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 13:44:47 +0000 Subject: [PATCH 060/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 1610d976bd9..fef8ddbfb6d 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -17,8 +17,8 @@ import numpy as np import pytest -from pycocotools import mask import torch +from pycocotools import mask from torch import Tensor from tests.helpers.testers import MetricTester From 225f546937268efc10ccfc9cec266863dab0b005 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Mon, 2 May 2022 15:47:27 +0200 Subject: [PATCH 061/141] flake8 --- tests/detection/test_map.py | 6 +++--- torchmetrics/detection/mean_ap.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index fef8ddbfb6d..1325f388644 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -273,9 +273,9 @@ def _compare_fn(preds, target) -> dict: def _compare_fn_segm(preds, target) -> dict: """Comparison function for map implementation for instance segmentation. - Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results - Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 - Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index b33366a80c0..7790a459e69 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -210,7 +210,8 @@ class MeanAveragePrecision(Metric): box_format: Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. iou_type: - Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. + Type of input (either masks or bounding-boxes) used for computing IOU. + Supported IOU types are ``[`bboxes`, `segm`]``. iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. From e493330a01a2ea7bf740350bc97b0e5d2bb3a385 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 13:49:11 +0000 Subject: [PATCH 062/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 7790a459e69..425c661676e 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -210,7 +210,7 @@ class MeanAveragePrecision(Metric): box_format: Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. iou_type: - Type of input (either masks or bounding-boxes) used for computing IOU. + Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` From 3245488321c4fc5a462ed4a0c690c388eb598f90 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Mon, 31 Jan 2022 14:31:45 +0100 Subject: [PATCH 063/141] implementaed IOU with segmentation masks and MAP for instance segmentation --- torchmetrics/detection/mean_ap.py | 76 ++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 4ed13944abc..dea65c8d440 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -80,6 +80,27 @@ class COCOMetricResults(BaseMetricResults): ) +def segm_iou(inputs, targets, smooth=1): + + n_inputs = inputs.shape[0] + n_targets = targets.shape[0] + # flatten label and prediction tensors + inputs = inputs.view(n_inputs, -1).repeat_interleave(n_targets, 0) + targets = targets.view(n_targets, -1).repeat(n_inputs, 1) + + # i1 * t1 + # i1 * t2 + # i2 * t1 + # i2 * t2 + + # intersection is equivalent to True Positive count + # union is the mutually inclusive area of all labels & predictions + intersections = (inputs * targets).sum(1, keepdims=True) + unions = (inputs + targets).sum(1, keepdims=True) + + return ((intersections + smooth) / (unions + smooth)).view(n_inputs, n_targets) + + def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]]) -> None: """Ensure the correct input format of `preds` and `targets`""" if not isinstance(preds, Sequence): @@ -217,10 +238,13 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruth_boxes: List[Tensor] groundtruth_labels: List[Tensor] + groundtruth_masks: List[Tensor] + detection_masks: List[Tensor] def __init__( self, box_format: str = "xyxy", + iou_type: str = "bbox", iou_thresholds: Optional[List[float]] = None, rec_thresholds: Optional[List[float]] = None, max_detection_thresholds: Optional[List[int]] = None, @@ -236,6 +260,7 @@ def __init__( ) allowed_box_formats = ("xyxy", "xywh", "cxcywh") + allowed_iou_types = ("segm", "bbox") if box_format not in allowed_box_formats: raise ValueError(f"Expected argument `box_format` to be one of {allowed_box_formats} but got {box_format}") self.box_format = box_format @@ -243,11 +268,14 @@ def __init__( self.rec_thresholds = rec_thresholds or torch.linspace(0.0, 1.00, round(1.00 / 0.01) + 1).tolist() max_det_thr, _ = torch.sort(IntTensor(max_detection_thresholds or [1, 10, 100])) self.max_detection_thresholds = max_det_thr.tolist() + if iou_type not in allowed_iou_types: + raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") + self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): @@ -257,8 +285,10 @@ def __init__( self.add_state("detection_boxes", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -309,12 +339,16 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detection_boxes.append(boxes) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) + if "masks" in item: + self.detection_masks.append(item["masks"]) for item in target: boxes = _fix_empty_tensors(item["boxes"]) boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") self.groundtruth_boxes.append(boxes) self.groundtruth_labels.append(item["labels"]) + if "masks" in item: + self.groundtruth_masks.append(item["masks"]) def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -322,7 +356,17 @@ def _get_classes(self) -> List: return torch.cat(self.detection_labels + self.groundtruth_labels).unique().tolist() return [] - def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: + def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: + if self.iou_type == "segm": + return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) + elif self.iou_type == "bbox": + return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + else: + raise Exception(f"IOU type {self.iou_type} is not supported") + + def _compute_iou_impl( + self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable + ) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -334,10 +378,13 @@ def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: max_det: Maximum number of evaluated detection bounding boxes """ - gt = self.groundtruth_boxes[idx] - det = self.detection_boxes[idx] - gt_label_mask = self.groundtruth_labels[idx] == class_id - det_label_mask = self.detection_labels[idx] == class_id + + gt = ground_truths[id] + det = detections[id] + + gt_label_mask = self.groundtruth_labels[id] == class_id + det_label_mask = self.detection_labels[id] == class_id + if len(gt_label_mask) == 0 or len(det_label_mask) == 0: return Tensor([]) gt = gt[gt_label_mask] @@ -353,8 +400,7 @@ def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: if len(det) > max_det: det = det[:max_det] - # generalized_box_iou - ious = box_iou(det, gt) + ious = compute_iou(det, gt) return ious def __evaluate_image_gt_no_preds( @@ -763,6 +809,14 @@ def compute(self) -> dict: - map_per_class: ``torch.Tensor`` (-1 if class metrics are disabled) - mar_100_per_class: ``torch.Tensor`` (-1 if class metrics are disabled) """ + + # move everything to CPU, as we are faster here + self.detections = [box.cpu() for box in self.detection_boxes] + self.detection_labels = [label.cpu() for label in self.detection_labels] + self.detection_scores = [score.cpu() for score in self.detection_scores] + self.groundtruths = [box.cpu() for box in self.groundtruth_boxes] + self.groundtruth_labels = [label.cpu() for label in self.groundtruth_labels] + classes = self._get_classes() precisions, recalls = self._calculate(classes) map_val, mar_val = self._summarize_results(precisions, recalls) From 67e50dc6344166eb806c59d0c5a522d472005604 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Sat, 5 Feb 2022 11:48:30 +0100 Subject: [PATCH 064/141] rebase; ddp test still failing --- torchmetrics/detection/mean_ap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index dea65c8d440..2bbf12efbec 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -843,4 +843,5 @@ def compute(self) -> dict: metrics.update(mar_val) metrics.map_per_class = map_per_class_values metrics[f"mar_{self.max_detection_thresholds[-1]}_per_class"] = mar_max_dets_per_class_values + return metrics From b1b6c92c9783bc960071bbfae9ccf555dac896aa Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:30:56 +0100 Subject: [PATCH 065/141] checking pr --- torchmetrics/detection/mean_ap.py | 73 ++++++++++++++++++------------- torchmetrics/metric.py | 1 + 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 2bbf12efbec..5cc7034116d 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -# Copyright The PyTorch Lightning team. +n # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -101,7 +101,9 @@ def segm_iou(inputs, targets, smooth=1): return ((intersections + smooth) / (unions + smooth)).view(n_inputs, n_targets) -def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]]) -> None: +def _input_validator( + preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[str, Tensor]], iou_type: str = "bbox" +) -> None: """Ensure the correct input format of `preds` and `targets`""" if not isinstance(preds, Sequence): raise ValueError("Expected argument `preds` to be of type Sequence") @@ -109,37 +111,38 @@ def _input_validator(preds: Sequence[Dict[str, Tensor]], targets: Sequence[Dict[ raise ValueError("Expected argument `target` to be of type Sequence") if len(preds) != len(targets): raise ValueError("Expected argument `preds` and `target` to have the same length") + iou_attribute = "boxes" if iou_type == "bbox" else "masks" - for k in ["boxes", "scores", "labels"]: + for k in [iou_attribute, "scores", "labels"]: if any(k not in p for p in preds): raise ValueError(f"Expected all dicts in `preds` to contain the `{k}` key") - for k in ["boxes", "labels"]: + for k in [iou_attribute, "labels"]: if any(k not in p for p in targets): raise ValueError(f"Expected all dicts in `target` to contain the `{k}` key") - if any(type(pred["boxes"]) is not Tensor for pred in preds): - raise ValueError("Expected all boxes in `preds` to be of type Tensor") + if any(type(pred[iou_attribute]) is not Tensor for pred in preds): + raise ValueError(f"Expected all {iou_attribute} in `preds` to be of type Tensor") if any(type(pred["scores"]) is not Tensor for pred in preds): raise ValueError("Expected all scores in `preds` to be of type Tensor") if any(type(pred["labels"]) is not Tensor for pred in preds): raise ValueError("Expected all labels in `preds` to be of type Tensor") - if any(type(target["boxes"]) is not Tensor for target in targets): - raise ValueError("Expected all boxes in `target` to be of type Tensor") + if any(type(target[iou_attribute]) is not Tensor for target in targets): + raise ValueError(f"Expected all {iou_attribute} in `target` to be of type Tensor") if any(type(target["labels"]) is not Tensor for target in targets): raise ValueError("Expected all labels in `target` to be of type Tensor") for i, item in enumerate(targets): - if item["boxes"].size(0) != item["labels"].size(0): + if item[iou_attribute].size(0) != item["labels"].size(0): raise ValueError( - f"Input boxes and labels of sample {i} in targets have a" - f" different length (expected {item['boxes'].size(0)} labels, got {item['labels'].size(0)})" + f"Input {iou_attribute} and labels of sample {i} in targets have a" + f" different length (expected {item[iou_attribute].size(0)} labels, got {item['labels'].size(0)})" ) for i, item in enumerate(preds): - if not (item["boxes"].size(0) == item["labels"].size(0) == item["scores"].size(0)): + if not (item[iou_attribute].size(0) == item["labels"].size(0) == item["scores"].size(0)): raise ValueError( - f"Input boxes, labels and scores of sample {i} in predictions have a" - f" different length (expected {item['boxes'].size(0)} labels and scores," + f"Input {iou_attribute}, labels and scores of sample {i} in predictions have a" + f" different length (expected {item[iou_attribute].size(0)} labels and scores," f" got {item['labels'].size(0)} labels and {item['scores'].size(0)})" ) @@ -285,9 +288,9 @@ def __init__( self.add_state("detection_boxes", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore @@ -334,21 +337,29 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] _input_validator(preds, target) for item in preds: - boxes = _fix_empty_tensors(item["boxes"]) - boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + boxes, masks = self._get_safe_item_values(item) self.detection_boxes.append(boxes) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - if "masks" in item: - self.detection_masks.append(item["masks"]) + self.detection_masks.append(masks) for item in target: - boxes = _fix_empty_tensors(item["boxes"]) - boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + boxes, masks = self._get_safe_item_values(item) self.groundtruth_boxes.append(boxes) self.groundtruth_labels.append(item["labels"]) - if "masks" in item: - self.groundtruth_masks.append(item["masks"]) + self.groundtruth_masks.append(masks) + + def _get_safe_item_values(self, item): + if self.iou_type == "bbox": + boxes = _fix_empty_tensors(item["boxes"]) + boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") + masks = _fix_empty_tensors(torch.Tensor()) + elif self.iou_type == "masks": + masks = _fix_empty_tensors(item["masks"]) + boxes = _fix_empty_tensors(torch.Tensor()) + else: + raise Exception(f"IOU type {self.iou_type} is not supported") + return boxes, masks def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -357,12 +368,14 @@ def _get_classes(self) -> List: return [] def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: - if self.iou_type == "segm": - return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) - elif self.iou_type == "bbox": - return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) - else: - raise Exception(f"IOU type {self.iou_type} is not supported") + return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + + # if self.iou_type == "segm": + # return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) + # elif self.iou_type == "bbox": + + # else: + # raise Exception(f"IOU type {self.iou_type} is not supported") def _compute_iou_impl( self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable @@ -400,7 +413,7 @@ def _compute_iou_impl( if len(det) > max_det: det = det[:max_det] - ious = compute_iou(det, gt) + ious = box_iou(det, gt) return ious def __evaluate_image_gt_no_preds( diff --git a/torchmetrics/metric.py b/torchmetrics/metric.py index 1e14e06e1da..190e63be886 100644 --- a/torchmetrics/metric.py +++ b/torchmetrics/metric.py @@ -259,6 +259,7 @@ def _sync_dist(self, dist_sync_fn: Callable = gather_all_tensors, process_group: for attr, reduction_fn in self._reductions.items(): # pre-processing ops (stack or flatten for inputs) + if isinstance(output_dict[attr][0], Tensor): output_dict[attr] = torch.stack(output_dict[attr]) elif isinstance(output_dict[attr][0], list): From 485061636c79b03fb9a38e736c5385bcde0d51ed Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:52:44 +0100 Subject: [PATCH 066/141] clean API interface; working on tests for iou_type SEGM --- tests/detection/test_map.py | 20 ++++++++++ torchmetrics/detection/mean_ap.py | 66 +++++++++++++------------------ 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index d3c6ea79eda..449b6cd5a18 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -377,6 +377,26 @@ def test_missing_gt(): assert result["map"] < 1, "MAP cannot be 1, as there is an image with no ground truth, but some predictions." +@pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") +def test_segm_iou_empty_mask(): + """Test empty ground truths.""" + metric = MeanAveragePrecision(iou_type="segm") + + metric.update( + [ + dict( + masks=torch.randint(0, 1, (1, 10, 10)), + scores=torch.Tensor([0.5]), + labels=torch.IntTensor([4]), + ), + ], + [ + dict(masks=torch.Tensor([]), labels=torch.IntTensor([])), + ], + ) + metric.compute() + + @pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") def test_error_on_wrong_input(): """Test class input validation.""" diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 5cc7034116d..475d6679f82 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -236,13 +236,11 @@ class MeanAveragePrecision(Metric): If ``class_metrics`` is not a boolean """ - detection_boxes: List[Tensor] + detections: List[Tensor] detection_scores: List[Tensor] detection_labels: List[Tensor] - groundtruth_boxes: List[Tensor] + groundtruths: List[Tensor] groundtruth_labels: List[Tensor] - groundtruth_masks: List[Tensor] - detection_masks: List[Tensor] def __init__( self, @@ -285,13 +283,11 @@ def __init__( raise ValueError("Expected argument `class_metrics` to be a boolean") self.class_metrics = class_metrics - self.add_state("detection_boxes", default=[], dist_reduce_fx=None) + self.add_state("detections", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) + self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -334,32 +330,29 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] ValueError: If any score is not type float and of length 1 """ - _input_validator(preds, target) + _input_validator(preds, target, iou_type=self.iou_type) for item in preds: - boxes, masks = self._get_safe_item_values(item) - self.detection_boxes.append(boxes) + detections = self._get_safe_item_values(item) + self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - self.detection_masks.append(masks) for item in target: - boxes, masks = self._get_safe_item_values(item) - self.groundtruth_boxes.append(boxes) + groundtruths = self._get_safe_item_values(item) + self.groundtruths.append(groundtruths) self.groundtruth_labels.append(item["labels"]) - self.groundtruth_masks.append(masks) def _get_safe_item_values(self, item): if self.iou_type == "bbox": boxes = _fix_empty_tensors(item["boxes"]) boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") - masks = _fix_empty_tensors(torch.Tensor()) - elif self.iou_type == "masks": + return boxes + elif self.iou_type == "segm": masks = _fix_empty_tensors(item["masks"]) - boxes = _fix_empty_tensors(torch.Tensor()) + return masks else: raise Exception(f"IOU type {self.iou_type} is not supported") - return boxes, masks def _get_classes(self) -> List: """Returns a list of unique classes found in ground truth and detection data.""" @@ -368,18 +361,11 @@ def _get_classes(self) -> List: return [] def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: - return self._compute_iou_impl(id, self.groundtruth_boxes, self.detection_boxes, class_id, max_det, box_iou) + iou_func = box_iou if self.iou_type == "bbox" else segm_iou - # if self.iou_type == "segm": - # return self._compute_iou_impl(id, self.groundtruth_masks, self.detection_masks, class_id, max_det, segm_iou) - # elif self.iou_type == "bbox": + return self._compute_iou_impl(id, class_id, max_det, iou_func) - # else: - # raise Exception(f"IOU type {self.iou_type} is not supported") - - def _compute_iou_impl( - self, id: int, ground_truths, detections, class_id: int, max_det: int, compute_iou: Callable - ) -> Tensor: + def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: Callable) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -392,8 +378,9 @@ def _compute_iou_impl( Maximum number of evaluated detection bounding boxes """ - gt = ground_truths[id] - det = detections[id] + # if self.iou_type == "bbox": + gt = self.groundtruths[id] + det = self.detections[id] gt_label_mask = self.groundtruth_labels[id] == class_id det_label_mask = self.detection_labels[id] == class_id @@ -406,14 +393,14 @@ def _compute_iou_impl( return Tensor([]) # Sort by scores and use only max detections - scores = self.detection_scores[idx] - scores_filtered = scores[self.detection_labels[idx] == class_id] + scores = self.detection_scores[id] + scores_filtered = scores[self.detection_labels[id] == class_id] inds = torch.argsort(scores_filtered, descending=True) det = det[inds] if len(det) > max_det: det = det[:max_det] - ious = box_iou(det, gt) + ious = compute_iou(det, gt) return ious def __evaluate_image_gt_no_preds( @@ -487,8 +474,9 @@ def _evaluate_image( ious: IoU results for image and class. """ - gt = self.groundtruth_boxes[idx] - det = self.detection_boxes[idx] + + gt = self.groundtruths[idx] + det = self.detections[idx] gt_label_mask = self.groundtruth_labels[idx] == class_id det_label_mask = self.detection_labels[idx] == class_id @@ -642,7 +630,7 @@ def _calculate(self, class_ids: List) -> Tuple[MAPMetricResults, MARMetricResult class_ids: List of label class Ids. """ - img_ids = range(len(self.groundtruth_boxes)) + img_ids = range(len(self.groundtruths)) max_detections = self.max_detection_thresholds[-1] area_ranges = self.bbox_area_ranges.values() @@ -824,10 +812,10 @@ def compute(self) -> dict: """ # move everything to CPU, as we are faster here - self.detections = [box.cpu() for box in self.detection_boxes] + self.detections = [box.cpu() for box in self.detections] self.detection_labels = [label.cpu() for label in self.detection_labels] self.detection_scores = [score.cpu() for score in self.detection_scores] - self.groundtruths = [box.cpu() for box in self.groundtruth_boxes] + self.groundtruths = [box.cpu() for box in self.groundtruths] self.groundtruth_labels = [label.cpu() for label in self.groundtruth_labels] classes = self._get_classes() From c2811ab4748a7b3e3ce939da7bb1df8f2bb5acd1 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Wed, 6 Apr 2022 16:25:07 +0200 Subject: [PATCH 067/141] fixing import --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 475d6679f82..5158a5492d4 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple import torch from torch import IntTensor, Tensor From ebed3b6b2ca2c88f7b96c4834a345a688d491d74 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:14:04 +0200 Subject: [PATCH 068/141] map test --- tests/detection/test_map.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 449b6cd5a18..a912bd4da27 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -24,7 +24,9 @@ Input = namedtuple("Input", ["preds", "target"]) -_inputs = Input( +_inputs_masks = Input(preds=[], target=[]) + +_inputs_bboxes = Input( preds=[ [ dict( @@ -157,7 +159,7 @@ def _compare_fn(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results All classes Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.706 Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.901 @@ -234,8 +236,8 @@ def test_map(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs.preds, - target=_inputs.target, + preds=_inputs_bboxes.preds, + target=_inputs_bboxes.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From c6ebd43cb91938cba0e204cdfdfd7d8b8dd967b0 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:42:25 +0200 Subject: [PATCH 069/141] fixes --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 5158a5492d4..134a4e65b8a 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -n # Copyright The PyTorch Lightning team. +# Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From ac770c4034420755662b761abcf2f42f2e44713b Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 15:42:27 +0200 Subject: [PATCH 070/141] test added (required native RLE encoding from COCO); fixed IOU --- tests/detection/test_map.py | 138 +++++++++++++++++++++++++++++- torchmetrics/detection/mean_ap.py | 39 ++++++++- 2 files changed, 171 insertions(+), 6 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index a912bd4da27..dd4bcb37651 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -14,8 +14,10 @@ from collections import namedtuple +import numpy as np import pytest import torch +from pycocotools import mask from torch import Tensor from tests.helpers.testers import MetricTester @@ -24,7 +26,87 @@ Input = namedtuple("Input", ["preds", "target"]) -_inputs_masks = Input(preds=[], target=[]) +_inputs_masks = Input( + preds=[ + [ + dict( + masks=torch.Tensor( + mask.decode( + { + "size": [478, 640], + "counts": "VQi31m>0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR dict: } +def _compare_fn_segm(preds, target) -> dict: + """Comparison function for map implementation. + + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + + """ + return { + "map": torch.Tensor([0.352]), + "map_50": torch.Tensor([0.742]), + "map_75": torch.Tensor([0.252]), + "map_small": torch.Tensor([-1]), + "map_medium": torch.Tensor([-1]), + "map_large": torch.Tensor([0.352]), + "mar_1": torch.Tensor([0.35]), + "mar_10": torch.Tensor([0.35]), + "mar_100": torch.Tensor([0.35]), + "mar_small": torch.Tensor([-1]), + "mar_medium": torch.Tensor([-1]), + "mar_large": torch.Tensor([0.35]), + "map_per_class": torch.Tensor([0.4039604, -1.0, 0.3]), + "mar_100_per_class": torch.Tensor([0.4, -1.0, 0.3]), + } + + _pytest_condition = not (_TORCHVISION_AVAILABLE and _TORCHVISION_GREATER_EQUAL_0_8) @@ -232,7 +350,8 @@ class TestMAP(MetricTester): atol = 1e-1 @pytest.mark.parametrize("ddp", [False, True]) - def test_map(self, compute_on_cpu, ddp): + def test_map_bbox(self, compute_on_cpu, ddp): + """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, @@ -245,6 +364,21 @@ def test_map(self, compute_on_cpu, ddp): metric_args={"class_metrics": True, "compute_on_cpu": compute_on_cpu}, ) + @pytest.mark.parametrize("ddp", [False]) + def test_map_segm(self, ddp): + """Test modular implementation for correctness.""" + + self.run_class_metric_test( + ddp=ddp, + preds=_inputs_masks.preds, + target=_inputs_masks.target, + metric_class=MeanAveragePrecision, + sk_metric=_compare_fn_segm, + dist_sync_on_step=False, + check_batch=False, + metric_args={"class_metrics": True, "iou_type": "segm"}, + ) + # noinspection PyTypeChecker @pytest.mark.skipif(_pytest_condition, reason="test requires that torchvision=>0.8.0 is installed") diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 134a4e65b8a..b05346edf1f 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -29,6 +29,24 @@ log = logging.getLogger(__name__) +def mask_area(input): + n_inputs = len(input) + + return input.reshape(n_inputs, -1).sum(1) + + +def compute_area(input, type="bbox"): + if len(input) == 0: + + return torch.Tensor([]).to(input.device) + + if type == "bbox": + + return box_area(input) + else: + return mask_area(input) + + class BaseMetricResults(dict): """Base metric class, that allows fields for pre-defined metrics.""" @@ -80,13 +98,23 @@ class COCOMetricResults(BaseMetricResults): ) +def _segm_iou(mask1, mask2): + + intersection = (mask1 * mask2).sum() + if intersection == 0: + return 0.0 + union = torch.logical_or(mask1, mask2).to(torch.int).sum() + return (intersection / union).unsqueeze(0) + + def segm_iou(inputs, targets, smooth=1): n_inputs = inputs.shape[0] n_targets = targets.shape[0] # flatten label and prediction tensors - inputs = inputs.view(n_inputs, -1).repeat_interleave(n_targets, 0) - targets = targets.view(n_targets, -1).repeat(n_inputs, 1) + + inputs = inputs.reshape(n_inputs, -1).repeat_interleave(n_targets, 0) + targets = targets.reshape(n_targets, -1).repeat(n_inputs, 1) # i1 * t1 # i1 * t2 @@ -401,6 +429,7 @@ def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: C det = det[:max_det] ious = compute_iou(det, gt) + return ious def __evaluate_image_gt_no_preds( @@ -499,13 +528,14 @@ def _evaluate_image( if gt.numel() == 0 and det.numel() == 0: return None - areas = box_area(gt) + areas = compute_area(gt, self.iou_type) ignore_area = (areas < area_range[0]) | (areas > area_range[1]) # sort dt highest score first, sort gt ignore last ignore_area_sorted, gtind = torch.sort(ignore_area.to(torch.uint8)) # Convert to uint8 temporarily and back to bool, because "Sort currently does not support bool dtype on CUDA" ignore_area_sorted = ignore_area_sorted.to(torch.bool) + gt = gt[gtind] scores = self.detection_scores[idx] scores_filtered = scores[det_label_mask] @@ -535,12 +565,13 @@ def _evaluate_image( gt_matches[idx_iou, m] = 1 # set unmatched detections outside of area range to ignore - det_areas = box_area(det) + det_areas = compute_area(det, self.iou_type) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.logical_or( det_ignore, torch.logical_and(det_matches == 0, torch.repeat_interleave(ar, nb_iou_thrs, 0)) ) + return { "dtMatches": det_matches.to(self.device), "gtMatches": gt_matches.to(self.device), From c33e5714042180dcf5c7f499fecf87a583c52480 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:13:34 +0200 Subject: [PATCH 071/141] rebased and fixed GPU device --- tests/detection/test_map.py | 13 +++++++------ torchmetrics/detection/mean_ap.py | 10 ++++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index dd4bcb37651..9ca13272204 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -108,7 +108,7 @@ ) -_inputs_bboxes = Input( +_inputs = Input( preds=[ [ dict( @@ -355,8 +355,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs_bboxes.preds, - target=_inputs_bboxes.target, + preds=_inputs.preds, + target=_inputs.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, @@ -365,7 +365,7 @@ def test_map_bbox(self, compute_on_cpu, ddp): ) @pytest.mark.parametrize("ddp", [False]) - def test_map_segm(self, ddp): + def test_map_segm(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( @@ -376,7 +376,7 @@ def test_map_segm(self, ddp): sk_metric=_compare_fn_segm, dist_sync_on_step=False, check_batch=False, - metric_args={"class_metrics": True, "iou_type": "segm"}, + metric_args={"class_metrics": True, "compute_on_cpu": compute_on_cpu, "iou_type": "segm"}, ) @@ -521,7 +521,7 @@ def test_segm_iou_empty_mask(): metric.update( [ dict( - masks=torch.randint(0, 1, (1, 10, 10)), + masks=torch.randint(0, 1, (1, 10, 10)).bool(), scores=torch.Tensor([0.5]), labels=torch.IntTensor([4]), ), @@ -530,6 +530,7 @@ def test_segm_iou_empty_mask(): dict(masks=torch.Tensor([]), labels=torch.IntTensor([])), ], ) + metric.compute() diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index b05346edf1f..bfb1ec9816f 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -462,6 +462,7 @@ def __evaluate_image_preds_no_gt( """Some predictions but no GT.""" # GTs nb_gt = 0 + gt_ignore = torch.zeros(nb_gt, dtype=torch.bool, device=self.device) # Detections @@ -473,7 +474,7 @@ def __evaluate_image_preds_no_gt( if len(det) > max_det: det = det[:max_det] nb_det = len(det) - det_areas = box_area(det).to(self.device) + det_areas = compute_area(det, type=self.iou_type).to(self.device) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.repeat_interleave(ar, nb_iou_thrs, 0) @@ -481,9 +482,9 @@ def __evaluate_image_preds_no_gt( return { "dtMatches": torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device), "gtMatches": torch.zeros((nb_iou_thrs, nb_gt), dtype=torch.bool, device=self.device), - "dtScores": scores_sorted, - "gtIgnore": gt_ignore, - "dtIgnore": det_ignore, + "dtScores": scores_sorted.to(self.device), + "gtIgnore": gt_ignore.to(self.device), + "dtIgnore": det_ignore.to(self.device), } def _evaluate_image( @@ -767,6 +768,7 @@ def __calculate_recall_precision_scores( img_eval_cls_bbox = [e for e in img_eval_cls_bbox if e is not None] if not img_eval_cls_bbox: return recall, precision, scores + det_scores = torch.cat([e["dtScores"][:max_det] for e in img_eval_cls_bbox]) # different sorting method generates slightly different results. From ec6f5c8dec19dcc0422b46061a0c7e6a58fb3800 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:17:20 +0200 Subject: [PATCH 072/141] docstring and smooth fix --- torchmetrics/detection/mean_ap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index bfb1ec9816f..910c232533d 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -107,7 +107,7 @@ def _segm_iou(mask1, mask2): return (intersection / union).unsqueeze(0) -def segm_iou(inputs, targets, smooth=1): +def segm_iou(inputs, targets, smooth=1e-5): n_inputs = inputs.shape[0] n_targets = targets.shape[0] @@ -209,6 +209,8 @@ class MeanAveragePrecision(Metric): Args: box_format: Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. + iou_type: + Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. From 7e5db6282d62c7afdacd4a5e1708be0b135dcb95 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:14 +0000 Subject: [PATCH 073/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 1 - torchmetrics/detection/mean_ap.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 9ca13272204..478713fbcb1 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -314,7 +314,6 @@ def _compare_fn_segm(preds, target) -> dict: Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 - """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 910c232533d..aa4066fb025 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -303,10 +303,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 70ae10754805830ea355997f8eaee35b49dacd16 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:27:37 +0200 Subject: [PATCH 074/141] Added pycocotools as detection requirements for test --- requirements/detection.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/detection.txt b/requirements/detection.txt index cbec7260775..da90ea549b7 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1 +1,2 @@ torchvision>=0.8 +pycocotools From 44c975e502f30ff9f76c9bbaa5b5734c28689264 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:28:18 +0200 Subject: [PATCH 075/141] pycocotools test req --- requirements/detection.txt | 1 - requirements/test.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/detection.txt b/requirements/detection.txt index da90ea549b7..cbec7260775 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1,2 +1 @@ torchvision>=0.8 -pycocotools diff --git a/requirements/test.txt b/requirements/test.txt index dfffc35433d..78c17457c98 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -16,3 +16,4 @@ fire cloudpickle>=1.3 scikit-learn>=0.24 +pycocotools From 0a0aad3fa5ba3efbe5d87c03b63487e4856fe78d Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Tue, 26 Apr 2022 13:45:36 +0200 Subject: [PATCH 076/141] moved test inputs to json file --- .../instance_segmentation_inputs.json | 30 +++++++++++ tests/detection/test_map.py | 50 ++++--------------- torchmetrics/detection/mean_ap.py | 10 ++-- 3 files changed, 46 insertions(+), 44 deletions(-) create mode 100644 tests/detection/instance_segmentation_inputs.json diff --git a/tests/detection/instance_segmentation_inputs.json b/tests/detection/instance_segmentation_inputs.json new file mode 100644 index 00000000000..85bc854c6fc --- /dev/null +++ b/tests/detection/instance_segmentation_inputs.json @@ -0,0 +1,30 @@ +{"preds":[{ + "size": [ + 478, + 640 + ], + "counts": "VQi31m>0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR Date: Mon, 31 Jan 2022 14:31:45 +0100 Subject: [PATCH 077/141] implementaed IOU with segmentation masks and MAP for instance segmentation --- torchmetrics/detection/mean_ap.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 57c30e17e9c..e9bd8116bab 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -98,16 +98,7 @@ class COCOMetricResults(BaseMetricResults): ) -def _segm_iou(mask1, mask2): - - intersection = (mask1 * mask2).sum() - if intersection == 0: - return 0.0 - union = torch.logical_or(mask1, mask2).to(torch.int).sum() - return (intersection / union).unsqueeze(0) - - -def segm_iou(inputs, targets, smooth=1e-5): +def segm_iou(inputs, targets, smooth=1): n_inputs = inputs.shape[0] n_targets = targets.shape[0] @@ -271,6 +262,8 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruths: List[Tensor] groundtruth_labels: List[Tensor] + groundtruth_masks: List[Tensor] + detection_masks: List[Tensor] def __init__( self, @@ -318,6 +311,7 @@ def __init__( self.add_state("detection_labels", default=[], dist_reduce_fx=None) self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -367,6 +361,8 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) + if "masks" in item: + self.detection_masks.append(item["masks"]) for item in target: groundtruths = self._get_safe_item_values(item) From f22c15de46c4199792d428830f0dc248dc30c27f Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:30:56 +0100 Subject: [PATCH 078/141] checking pr --- torchmetrics/detection/mean_ap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index e9bd8116bab..ea671a10c4b 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -311,6 +311,7 @@ def __init__( self.add_state("detection_labels", default=[], dist_reduce_fx=None) self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore @@ -361,8 +362,7 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - if "masks" in item: - self.detection_masks.append(item["masks"]) + self.detection_masks.append(masks) for item in target: groundtruths = self._get_safe_item_values(item) From f604370f5bec76ef7420efa53fbeba72da89e14a Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:52:44 +0100 Subject: [PATCH 079/141] clean API interface; working on tests for iou_type SEGM --- torchmetrics/detection/mean_ap.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index ea671a10c4b..8786da1b706 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -262,8 +262,6 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruths: List[Tensor] groundtruth_labels: List[Tensor] - groundtruth_masks: List[Tensor] - detection_masks: List[Tensor] def __init__( self, @@ -311,8 +309,6 @@ def __init__( self.add_state("detection_labels", default=[], dist_reduce_fx=None) self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -362,7 +358,6 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - self.detection_masks.append(masks) for item in target: groundtruths = self._get_safe_item_values(item) From 822536433dbd2f686b2cb2592bd9233508f86074 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:14:04 +0200 Subject: [PATCH 080/141] map test --- tests/detection/test_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 37015c37a77..f346ee93b7b 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -326,8 +326,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs.preds, - target=_inputs.target, + preds=_inputs_bboxes.preds, + target=_inputs_bboxes.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From 7e2a47e026a282ef0a795dcff7f473716eb620c8 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 15:42:27 +0200 Subject: [PATCH 081/141] test added (required native RLE encoding from COCO); fixed IOU --- tests/detection/test_map.py | 30 +++++++++++++++++------------- torchmetrics/detection/mean_ap.py | 9 +++++++++ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index f346ee93b7b..da23c41f8c7 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -273,19 +273,23 @@ def _compare_fn(preds, target) -> dict: def _compare_fn_segm(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results - Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 - Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 - Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 - Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 - Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 - Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 - Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 - Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + <<<<<<< HEAD + ======= + + >>>>>>> test added (required native RLE encoding from COCO); fixed IOU """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 8786da1b706..9ea5c299aa4 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -98,6 +98,15 @@ class COCOMetricResults(BaseMetricResults): ) +def _segm_iou(mask1, mask2): + + intersection = (mask1 * mask2).sum() + if intersection == 0: + return 0.0 + union = torch.logical_or(mask1, mask2).to(torch.int).sum() + return (intersection / union).unsqueeze(0) + + def segm_iou(inputs, targets, smooth=1): n_inputs = inputs.shape[0] From 9a314b5944e3c3e7524c3f3c306fd7ddbcec24cb Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:13:34 +0200 Subject: [PATCH 082/141] rebased and fixed GPU device --- tests/detection/test_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index da23c41f8c7..ef893c64466 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -330,8 +330,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs_bboxes.preds, - target=_inputs_bboxes.target, + preds=_inputs.preds, + target=_inputs.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From acccbfe45aad333d696e981adbc92d1694dc9a03 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:17:20 +0200 Subject: [PATCH 083/141] docstring and smooth fix --- torchmetrics/detection/mean_ap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 9ea5c299aa4..910c232533d 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -107,7 +107,7 @@ def _segm_iou(mask1, mask2): return (intersection / union).unsqueeze(0) -def segm_iou(inputs, targets, smooth=1): +def segm_iou(inputs, targets, smooth=1e-5): n_inputs = inputs.shape[0] n_targets = targets.shape[0] @@ -211,7 +211,7 @@ class MeanAveragePrecision(Metric): Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. iou_type: Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. - iou_thresholds:`` + iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. rec_thresholds: From e570a8d472a1f934d579e7dfc9f9d775ce13442d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:14 +0000 Subject: [PATCH 084/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 31 ++++++++++++++----------------- torchmetrics/detection/mean_ap.py | 8 ++++---- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index ef893c64466..9d5b010f969 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -273,23 +273,20 @@ def _compare_fn(preds, target) -> dict: def _compare_fn_segm(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results - Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 - Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 - Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 - Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 - Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 - Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 - Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 - Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 - Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 - <<<<<<< HEAD - ======= - - >>>>>>> test added (required native RLE encoding from COCO); fixed IOU + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 910c232533d..aa4066fb025 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -303,10 +303,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 73e350351d980b7efc18a15233a4a745ad486dc2 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:27:37 +0200 Subject: [PATCH 085/141] Added pycocotools as detection requirements for test --- requirements/detection.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/detection.txt b/requirements/detection.txt index cbec7260775..da90ea549b7 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1 +1,2 @@ torchvision>=0.8 +pycocotools From efe43ab683d7d646782d20c3302cbbf7e66896bf Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:28:18 +0200 Subject: [PATCH 086/141] pycocotools test req --- requirements/detection.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/detection.txt b/requirements/detection.txt index da90ea549b7..cbec7260775 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1,2 +1 @@ torchvision>=0.8 -pycocotools From 782b756f99bd21f3bd863e3a557196f28af032cb Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Mon, 31 Jan 2022 14:31:45 +0100 Subject: [PATCH 087/141] implementaed IOU with segmentation masks and MAP for instance segmentation --- torchmetrics/detection/mean_ap.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index aa4066fb025..a01fc5d245c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -113,9 +113,6 @@ def segm_iou(inputs, targets, smooth=1e-5): n_targets = targets.shape[0] # flatten label and prediction tensors - inputs = inputs.reshape(n_inputs, -1).repeat_interleave(n_targets, 0) - targets = targets.reshape(n_targets, -1).repeat(n_inputs, 1) - # i1 * t1 # i1 * t2 # i2 * t1 @@ -271,6 +268,8 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruths: List[Tensor] groundtruth_labels: List[Tensor] + groundtruth_masks: List[Tensor] + detection_masks: List[Tensor] def __init__( self, @@ -303,10 +302,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): @@ -316,8 +315,10 @@ def __init__( self.add_state("detections", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) + self.add_state("groundtruths", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) + self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -367,6 +368,8 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) + if "masks" in item: + self.detection_masks.append(item["masks"]) for item in target: groundtruths = self._get_safe_item_values(item) @@ -847,11 +850,6 @@ def compute(self) -> dict: """ # move everything to CPU, as we are faster here - self.detections = [box.cpu() for box in self.detections] - self.detection_labels = [label.cpu() for label in self.detection_labels] - self.detection_scores = [score.cpu() for score in self.detection_scores] - self.groundtruths = [box.cpu() for box in self.groundtruths] - self.groundtruth_labels = [label.cpu() for label in self.groundtruth_labels] classes = self._get_classes() precisions, recalls = self._calculate(classes) From cb2afe38859574e81c96f5c7470f31827b946b91 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:30:56 +0100 Subject: [PATCH 088/141] checking pr --- torchmetrics/detection/mean_ap.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index a01fc5d245c..1a99576b94e 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -# Copyright The PyTorch Lightning team. +n # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -368,8 +368,7 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - if "masks" in item: - self.detection_masks.append(item["masks"]) + self.detection_masks.append(masks) for item in target: groundtruths = self._get_safe_item_values(item) From b3f5c731cf2b5e318bb98b4de15159d1f7a33c84 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:52:44 +0100 Subject: [PATCH 089/141] clean API interface; working on tests for iou_type SEGM --- torchmetrics/detection/mean_ap.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 1a99576b94e..e06cf7c428c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -268,8 +268,6 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruths: List[Tensor] groundtruth_labels: List[Tensor] - groundtruth_masks: List[Tensor] - detection_masks: List[Tensor] def __init__( self, @@ -315,10 +313,8 @@ def __init__( self.add_state("detections", default=[], dist_reduce_fx=None) self.add_state("detection_scores", default=[], dist_reduce_fx=None) self.add_state("detection_labels", default=[], dist_reduce_fx=None) - self.add_state("groundtruths", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_boxes", default=[], dist_reduce_fx=None) + self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -368,7 +364,6 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - self.detection_masks.append(masks) for item in target: groundtruths = self._get_safe_item_values(item) From 4565eed16ca6e34508da2567a56f346c5e20201d Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:14:04 +0200 Subject: [PATCH 090/141] map test --- tests/detection/test_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 9d5b010f969..adca88ca05a 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -327,8 +327,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs.preds, - target=_inputs.target, + preds=_inputs_bboxes.preds, + target=_inputs_bboxes.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From 2ca734b488fd2994120351416b0987fff46f47e3 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:42:25 +0200 Subject: [PATCH 091/141] fixes --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index e06cf7c428c..fe8bde13b9f 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -n # Copyright The PyTorch Lightning team. +# Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 976f3b5cd5a42ea8b3c987f69f9309b86d910d12 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 15:42:27 +0200 Subject: [PATCH 092/141] test added (required native RLE encoding from COCO); fixed IOU --- tests/detection/test_map.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index adca88ca05a..b79b3d70f9f 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -306,6 +306,42 @@ def _compare_fn_segm(preds, target) -> dict: } +def _compare_fn_segm(preds, target) -> dict: + """Comparison function for map implementation. + + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + + """ + return { + "map": torch.Tensor([0.352]), + "map_50": torch.Tensor([0.742]), + "map_75": torch.Tensor([0.252]), + "map_small": torch.Tensor([-1]), + "map_medium": torch.Tensor([-1]), + "map_large": torch.Tensor([0.352]), + "mar_1": torch.Tensor([0.35]), + "mar_10": torch.Tensor([0.35]), + "mar_100": torch.Tensor([0.35]), + "mar_small": torch.Tensor([-1]), + "mar_medium": torch.Tensor([-1]), + "mar_large": torch.Tensor([0.35]), + "map_per_class": torch.Tensor([0.4039604, -1.0, 0.3]), + "mar_100_per_class": torch.Tensor([0.4, -1.0, 0.3]), + } + + _pytest_condition = not (_TORCHVISION_AVAILABLE and _TORCHVISION_GREATER_EQUAL_0_8) From a897ce302002c1ee87455ee3eb1e0070be854c34 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:13:34 +0200 Subject: [PATCH 093/141] rebased and fixed GPU device --- tests/detection/test_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index b79b3d70f9f..90e18c14fc9 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -363,8 +363,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs_bboxes.preds, - target=_inputs_bboxes.target, + preds=_inputs.preds, + target=_inputs.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From a789f3a295a6261548ecf4a847b9fc86f5b770d9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:14 +0000 Subject: [PATCH 094/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 1 - torchmetrics/detection/mean_ap.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 90e18c14fc9..89c57d096c4 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -322,7 +322,6 @@ def _compare_fn_segm(preds, target) -> dict: Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 - """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index fe8bde13b9f..e3740925ca3 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -300,10 +300,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From babac03ca78ee507f9ba196d86d5c38d44015373 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:27:37 +0200 Subject: [PATCH 095/141] Added pycocotools as detection requirements for test --- requirements/detection.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/detection.txt b/requirements/detection.txt index cbec7260775..da90ea549b7 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1 +1,2 @@ torchvision>=0.8 +pycocotools From 104948a77ad66978fc6f17d4fc44e53d645b3530 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:28:18 +0200 Subject: [PATCH 096/141] pycocotools test req --- requirements/detection.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/detection.txt b/requirements/detection.txt index da90ea549b7..cbec7260775 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1,2 +1 @@ torchvision>=0.8 -pycocotools From 768a4ca6346677e971b44aa0e2d637ec9ff9a932 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Tue, 26 Apr 2022 13:45:36 +0200 Subject: [PATCH 097/141] moved test inputs to json file --- torchmetrics/detection/mean_ap.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index e3740925ca3..56a7335cc87 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -208,7 +208,7 @@ class MeanAveragePrecision(Metric): Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. iou_type: Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. - iou_thresholds: + iou_thresholds:`` IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. rec_thresholds: @@ -300,10 +300,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): From 38362676800e77abae7908d8d0e9a8893370403f Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Mon, 31 Jan 2022 14:31:45 +0100 Subject: [PATCH 098/141] implementaed IOU with segmentation masks and MAP for instance segmentation --- torchmetrics/detection/mean_ap.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 56a7335cc87..afedb6e68a9 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -268,6 +268,8 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruths: List[Tensor] groundtruth_labels: List[Tensor] + groundtruth_masks: List[Tensor] + detection_masks: List[Tensor] def __init__( self, @@ -315,6 +317,7 @@ def __init__( self.add_state("detection_labels", default=[], dist_reduce_fx=None) self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -364,6 +367,8 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) + if "masks" in item: + self.detection_masks.append(item["masks"]) for item in target: groundtruths = self._get_safe_item_values(item) From 72aec73341b601829746e114a24104e1d047b5f4 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:30:56 +0100 Subject: [PATCH 099/141] checking pr --- torchmetrics/detection/mean_ap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index afedb6e68a9..e79b9496e2c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -# Copyright The PyTorch Lightning team. +n # Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -317,6 +317,7 @@ def __init__( self.add_state("detection_labels", default=[], dist_reduce_fx=None) self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) + self.add_state("detection_masks", default=[], dist_reduce_fx=None) self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore @@ -367,8 +368,7 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - if "masks" in item: - self.detection_masks.append(item["masks"]) + self.detection_masks.append(masks) for item in target: groundtruths = self._get_safe_item_values(item) From fab0e20da43d85a82113786480ae40d7cecfb699 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 17 Mar 2022 09:52:44 +0100 Subject: [PATCH 100/141] clean API interface; working on tests for iou_type SEGM --- torchmetrics/detection/mean_ap.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index e79b9496e2c..30316aed0cf 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -268,8 +268,6 @@ class MeanAveragePrecision(Metric): detection_labels: List[Tensor] groundtruths: List[Tensor] groundtruth_labels: List[Tensor] - groundtruth_masks: List[Tensor] - detection_masks: List[Tensor] def __init__( self, @@ -317,8 +315,6 @@ def __init__( self.add_state("detection_labels", default=[], dist_reduce_fx=None) self.add_state("groundtruths", default=[], dist_reduce_fx=None) self.add_state("groundtruth_labels", default=[], dist_reduce_fx=None) - self.add_state("detection_masks", default=[], dist_reduce_fx=None) - self.add_state("groundtruth_masks", default=[], dist_reduce_fx=None) def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]]) -> None: # type: ignore """Add detections and ground truth to the metric. @@ -368,7 +364,6 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) - self.detection_masks.append(masks) for item in target: groundtruths = self._get_safe_item_values(item) @@ -433,7 +428,6 @@ def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: C det = det[:max_det] ious = compute_iou(det, gt) - return ious def __evaluate_image_gt_no_preds( From 6591e09a0e97460a55d6045a98449b29a762f6fc Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 10:14:04 +0200 Subject: [PATCH 101/141] map test --- tests/detection/test_map.py | 4 ++-- torchmetrics/detection/mean_ap.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 89c57d096c4..7a89c2662f1 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -362,8 +362,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs.preds, - target=_inputs.target, + preds=_inputs_bboxes.preds, + target=_inputs_bboxes.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 30316aed0cf..cd72d9788d9 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -1,4 +1,4 @@ -n # Copyright The PyTorch Lightning team. +# Copyright The PyTorch Lightning team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 17b0a8198cc919a02c2b5cb7fbf30547396d4357 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 15:42:27 +0200 Subject: [PATCH 102/141] test added (required native RLE encoding from COCO); fixed IOU --- tests/detection/test_map.py | 37 ++++++++++++++++++++++++++++++- torchmetrics/detection/mean_ap.py | 4 ++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 7a89c2662f1..cc2cbf77139 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -19,7 +19,6 @@ import pytest import torch from pycocotools import mask -from torch import Tensor from tests.helpers.testers import MetricTester from torchmetrics.detection.mean_ap import MeanAveragePrecision @@ -341,6 +340,42 @@ def _compare_fn_segm(preds, target) -> dict: } +def _compare_fn_segm(preds, target) -> dict: + """Comparison function for map implementation. + + Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.352 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.350 + Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 + Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 + + """ + return { + "map": torch.Tensor([0.352]), + "map_50": torch.Tensor([0.742]), + "map_75": torch.Tensor([0.252]), + "map_small": torch.Tensor([-1]), + "map_medium": torch.Tensor([-1]), + "map_large": torch.Tensor([0.352]), + "mar_1": torch.Tensor([0.35]), + "mar_10": torch.Tensor([0.35]), + "mar_100": torch.Tensor([0.35]), + "mar_small": torch.Tensor([-1]), + "mar_medium": torch.Tensor([-1]), + "mar_large": torch.Tensor([0.35]), + "map_per_class": torch.Tensor([0.4039604, -1.0, 0.3]), + "mar_100_per_class": torch.Tensor([0.4, -1.0, 0.3]), + } + + _pytest_condition = not (_TORCHVISION_AVAILABLE and _TORCHVISION_GREATER_EQUAL_0_8) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index cd72d9788d9..45386da913b 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -113,6 +113,9 @@ def segm_iou(inputs, targets, smooth=1e-5): n_targets = targets.shape[0] # flatten label and prediction tensors + inputs = inputs.reshape(n_inputs, -1).repeat_interleave(n_targets, 0) + targets = targets.reshape(n_targets, -1).repeat(n_inputs, 1) + # i1 * t1 # i1 * t2 # i2 * t1 @@ -428,6 +431,7 @@ def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: C det = det[:max_det] ious = compute_iou(det, gt) + return ious def __evaluate_image_gt_no_preds( From 1f2bf82f5b812766b4f2d9397ec77e677254a919 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:13:34 +0200 Subject: [PATCH 103/141] rebased and fixed GPU device --- tests/detection/test_map.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index cc2cbf77139..8598ea20027 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -397,8 +397,8 @@ def test_map_bbox(self, compute_on_cpu, ddp): """Test modular implementation for correctness.""" self.run_class_metric_test( ddp=ddp, - preds=_inputs_bboxes.preds, - target=_inputs_bboxes.target, + preds=_inputs.preds, + target=_inputs.target, metric_class=MeanAveragePrecision, sk_metric=_compare_fn, dist_sync_on_step=False, From e2878a68f17482bcab398531aef9a85470989b5b Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 21 Apr 2022 16:17:20 +0200 Subject: [PATCH 104/141] docstring and smooth fix --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 45386da913b..972cfa84f88 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -211,7 +211,7 @@ class MeanAveragePrecision(Metric): Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. iou_type: Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. - iou_thresholds:`` + iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. rec_thresholds: From 378795e2c8841576576abd39e93812ced483efad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:14 +0000 Subject: [PATCH 105/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/detection/test_map.py | 1 - torchmetrics/detection/mean_ap.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 8598ea20027..692e2adf880 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -285,7 +285,6 @@ def _compare_fn_segm(preds, target) -> dict: Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000 Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 - """ return { "map": torch.Tensor([0.352]), diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 972cfa84f88..ce248472357 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -303,10 +303,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 9087fa52b6c8b62dbda0673839342315ebfd3897 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:27:37 +0200 Subject: [PATCH 106/141] Added pycocotools as detection requirements for test --- requirements/detection.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/detection.txt b/requirements/detection.txt index cbec7260775..da90ea549b7 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1 +1,2 @@ torchvision>=0.8 +pycocotools From c4eac30e284b7cbd15843879739eade627c591b9 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 22 Apr 2022 09:28:18 +0200 Subject: [PATCH 107/141] pycocotools test req --- requirements/detection.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/detection.txt b/requirements/detection.txt index da90ea549b7..cbec7260775 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1,2 +1 @@ torchvision>=0.8 -pycocotools From 76a5aa88049306889d33329738578866f10e9845 Mon Sep 17 00:00:00 2001 From: Gianluca Scarpellini Date: Tue, 26 Apr 2022 13:49:25 +0000 Subject: [PATCH 108/141] Update tests/detection/test_map.py Co-authored-by: Nicki Skafte Detlefsen --- tests/detection/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 692e2adf880..9ff054da60b 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -212,7 +212,7 @@ def _compare_fn(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results All classes Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.706 Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.901 From 34004ad795bca5f5c2903775b694a853f3e4765a Mon Sep 17 00:00:00 2001 From: Gianluca Scarpellini Date: Tue, 26 Apr 2022 13:49:31 +0000 Subject: [PATCH 109/141] Update tests/detection/test_map.py Co-authored-by: Nicki Skafte Detlefsen --- tests/detection/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 9ff054da60b..6c0dd965443 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -272,7 +272,7 @@ def _compare_fn(preds, target) -> dict: def _compare_fn_segm(preds, target) -> dict: """Comparison function for map implementation. - Official pycocotools results calculated from a subset of https://GitHub.com/cocodataset/cocoapi/tree/master/results + Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.752 Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.252 From ce58d5296b9ac8fa9bea65dd28f54ecb0f0680ba Mon Sep 17 00:00:00 2001 From: Gianluca Scarpellini Date: Tue, 26 Apr 2022 13:49:41 +0000 Subject: [PATCH 110/141] Update tests/detection/test_map.py Co-authored-by: Nicki Skafte Detlefsen --- tests/detection/test_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 6c0dd965443..11790333ca7 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -270,7 +270,7 @@ def _compare_fn(preds, target) -> dict: def _compare_fn_segm(preds, target) -> dict: - """Comparison function for map implementation. + """Comparison function for map implementation for instance segmentation. Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.352 From 08ebc415930def40b742ad7b207b559635352056 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 5 May 2022 16:03:00 +0200 Subject: [PATCH 111/141] IOU for segm and bbox working! Need to introduce fixes for handle RLE (pycocotools); a future PR could include a reimplementation of RLE encoding-decoding that does not rely on pycocotools --- tests/helpers/testers.py | 1 + torchmetrics/detection/mean_ap.py | 160 +++++++++++++++++------------- 2 files changed, 94 insertions(+), 67 deletions(-) diff --git a/tests/helpers/testers.py b/tests/helpers/testers.py index aeda9a95c98..312b8ce05c4 100644 --- a/tests/helpers/testers.py +++ b/tests/helpers/testers.py @@ -211,6 +211,7 @@ def _class_test( _assert_allclose(batch_result, sk_batch_result, atol=atol) # check that metrics are hashable + assert hash(metric) # check on all batches on all ranks diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index ce248472357..be543cd7e86 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -14,6 +14,8 @@ import logging from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +import numpy as np +import pycocotools.mask as mask_utils import torch from torch import IntTensor, Tensor @@ -38,13 +40,23 @@ def mask_area(input): def compute_area(input, type="bbox"): if len(input) == 0: - return torch.Tensor([]).to(input.device) + return torch.Tensor([]) if type == "bbox": + return box_area(torch.stack(input)) + else: + + input = [{"size": i[0], "counts": i[1]} for i in input] + area = torch.tensor(mask_utils.area(input).astype("int")) + + return area - return box_area(input) + +def compute_iou(det, gt, type="bbox") -> Tensor: + if type == "bbox": + return box_iou(torch.stack(det), torch.stack(gt)) else: - return mask_area(input) + return segm_iou(det, gt) class BaseMetricResults(dict): @@ -98,35 +110,16 @@ class COCOMetricResults(BaseMetricResults): ) -def _segm_iou(mask1, mask2): - - intersection = (mask1 * mask2).sum() - if intersection == 0: - return 0.0 - union = torch.logical_or(mask1, mask2).to(torch.int).sum() - return (intersection / union).unsqueeze(0) - - -def segm_iou(inputs, targets, smooth=1e-5): - - n_inputs = inputs.shape[0] - n_targets = targets.shape[0] - # flatten label and prediction tensors +def segm_iou(det, gt): + if isinstance(det, dict): + det = [det] + if isinstance(gt, dict): + gt = [gt] - inputs = inputs.reshape(n_inputs, -1).repeat_interleave(n_targets, 0) - targets = targets.reshape(n_targets, -1).repeat(n_inputs, 1) + det = [{"size": i[0], "counts": i[1]} for i in det] + gt = [{"size": i[0], "counts": i[1]} for i in gt] - # i1 * t1 - # i1 * t2 - # i2 * t1 - # i2 * t2 - - # intersection is equivalent to True Positive count - # union is the mutually inclusive area of all labels & predictions - intersections = (inputs * targets).sum(1, keepdims=True) - unions = (inputs + targets).sum(1, keepdims=True) - - return ((intersections + smooth) / (unions + smooth)).view(n_inputs, n_targets) + return torch.tensor(mask_utils.iou(det, gt, [False for _ in gt])) def _input_validator( @@ -177,6 +170,7 @@ def _input_validator( def _fix_empty_tensors(boxes: Tensor) -> Tensor: """Empty tensors can cause problems in DDP mode, this methods corrects them.""" + if boxes.numel() == 0 and boxes.ndim == 1: return boxes.unsqueeze(0) return boxes @@ -303,10 +297,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): @@ -363,7 +357,9 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] _input_validator(preds, target, iou_type=self.iou_type) for item in preds: + detections = self._get_safe_item_values(item) + self.detections.append(detections) self.detection_labels.append(item["labels"]) self.detection_scores.append(item["scores"]) @@ -373,14 +369,34 @@ def update(self, preds: List[Dict[str, Tensor]], target: List[Dict[str, Tensor]] self.groundtruths.append(groundtruths) self.groundtruth_labels.append(item["labels"]) + def _move_list_states_to_cpu(self) -> None: + """Move list states to cpu to save GPU memory.""" + + for key in self._defaults.keys(): + current_val = getattr(self, key) + current_to_cpu = [] + if isinstance(current_val, Sequence): + for cur_v in current_val: + # Cannot handle RLE as torch.Tensor + if not isinstance(cur_v, tuple): + cur_v = cur_v.to("cpu") + current_to_cpu.append(cur_v) + setattr(self, key, current_to_cpu) + def _get_safe_item_values(self, item): + if self.iou_type == "bbox": boxes = _fix_empty_tensors(item["boxes"]) boxes = box_convert(boxes, in_fmt=self.box_format, out_fmt="xyxy") return boxes elif self.iou_type == "segm": - masks = _fix_empty_tensors(item["masks"]) - return masks + masks = [] + + for i in item["masks"].cpu().numpy(): + rle = mask_utils.encode(np.asfortranarray(i)) + masks.append((tuple(rle["size"]), rle["counts"])) + + return tuple(masks) else: raise Exception(f"IOU type {self.iou_type} is not supported") @@ -391,11 +407,6 @@ def _get_classes(self) -> List: return [] def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: - iou_func = box_iou if self.iou_type == "bbox" else segm_iou - - return self._compute_iou_impl(id, class_id, max_det, iou_func) - - def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: Callable) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -412,13 +423,15 @@ def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: C gt = self.groundtruths[id] det = self.detections[id] - gt_label_mask = self.groundtruth_labels[id] == class_id - det_label_mask = self.detection_labels[id] == class_id + gt_label_mask = (self.groundtruth_labels[id] == class_id).nonzero().squeeze(1) + det_label_mask = (self.detection_labels[id] == class_id).nonzero().squeeze(1) if len(gt_label_mask) == 0 or len(det_label_mask) == 0: return Tensor([]) - gt = gt[gt_label_mask] - det = det[det_label_mask] + + gt = [gt[i] for i in gt_label_mask] + det = [det[i] for i in det_label_mask] + if len(gt) == 0 or len(det) == 0: return Tensor([]) @@ -426,12 +439,13 @@ def _compute_iou_impl(self, id: int, class_id: int, max_det: int, compute_iou: C scores = self.detection_scores[id] scores_filtered = scores[self.detection_labels[id] == class_id] inds = torch.argsort(scores_filtered, descending=True) - det = det[inds] + + # TODO Fix (only for masks is necessary) + det = [det[i] for i in inds] if len(det) > max_det: det = det[:max_det] - ious = compute_iou(det, gt) - + ious = compute_iou(det, gt, self.iou_type).to(self.device) return ious def __evaluate_image_gt_no_preds( @@ -468,11 +482,13 @@ def __evaluate_image_preds_no_gt( gt_ignore = torch.zeros(nb_gt, dtype=torch.bool, device=self.device) # Detections - det = det[det_label_mask] + + det = [det[i] for i in det_label_mask] scores = self.detection_scores[idx] scores_filtered = scores[det_label_mask] scores_sorted, dtind = torch.sort(scores_filtered, descending=True) - det = det[dtind] + + det = [det[i] for i in dtind] if len(det) > max_det: det = det[:max_det] nb_det = len(det) @@ -509,8 +525,8 @@ def _evaluate_image( gt = self.groundtruths[idx] det = self.detections[idx] - gt_label_mask = self.groundtruth_labels[idx] == class_id - det_label_mask = self.detection_labels[idx] == class_id + gt_label_mask = (self.groundtruth_labels[idx] == class_id).nonzero().squeeze(1) + det_label_mask = (self.detection_labels[idx] == class_id).nonzero().squeeze(1) # No Gt and No predictions --> ignore image if len(gt_label_mask) == 0 and len(det_label_mask) == 0: @@ -526,24 +542,29 @@ def _evaluate_image( if len(gt_label_mask) == 0 and len(det_label_mask) >= 0: return self.__evaluate_image_preds_no_gt(det, idx, det_label_mask, max_det, area_range, nb_iou_thrs) - gt = gt[gt_label_mask] - det = det[det_label_mask] - if gt.numel() == 0 and det.numel() == 0: + gt = [gt[i] for i in gt_label_mask] + det = [det[i] for i in det_label_mask] + if len(gt) == 0 and len(det) == 0: return None + if isinstance(det, dict): + det = [det] + if isinstance(gt, dict): + gt = [gt] - areas = compute_area(gt, self.iou_type) - ignore_area = (areas < area_range[0]) | (areas > area_range[1]) + areas = compute_area(gt, self.iou_type).to(self.device) + ignore_area = torch.tensor(areas < area_range[0]) | (areas > area_range[1]) # sort dt highest score first, sort gt ignore last ignore_area_sorted, gtind = torch.sort(ignore_area.to(torch.uint8)) # Convert to uint8 temporarily and back to bool, because "Sort currently does not support bool dtype on CUDA" - ignore_area_sorted = ignore_area_sorted.to(torch.bool) - gt = gt[gtind] + ignore_area_sorted = ignore_area_sorted.to(torch.bool).to(self.device) + + gt = [gt[i] for i in gtind] scores = self.detection_scores[idx] scores_filtered = scores[det_label_mask] scores_sorted, dtind = torch.sort(scores_filtered, descending=True) - det = det[dtind] + det = [det[i] for i in dtind] if len(det) > max_det: det = det[:max_det] # load computed ious @@ -552,10 +573,10 @@ def _evaluate_image( nb_iou_thrs = len(self.iou_thresholds) nb_gt = len(gt) nb_det = len(det) - gt_matches = torch.zeros((nb_iou_thrs, nb_gt), dtype=torch.bool, device=gt.device) - det_matches = torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=gt.device) + gt_matches = torch.zeros((nb_iou_thrs, nb_gt), dtype=torch.bool, device=self.device) + det_matches = torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device) gt_ignore = ignore_area_sorted - det_ignore = torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=gt.device) + det_ignore = torch.zeros((nb_iou_thrs, nb_det), dtype=torch.bool, device=self.device) if torch.numel(ious) > 0: for idx_iou, t in enumerate(self.iou_thresholds): @@ -568,7 +589,7 @@ def _evaluate_image( gt_matches[idx_iou, m] = 1 # set unmatched detections outside of area range to ignore - det_areas = compute_area(det, self.iou_type) + det_areas = compute_area(det, self.iou_type).to(self.device) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.logical_or( @@ -805,7 +826,8 @@ def __calculate_recall_precision_scores( diff_zero = torch.zeros((1,), device=pr.device) diff = torch.ones((1,), device=pr.device) while not torch.all(diff == 0): - diff = torch.clamp(torch.cat((pr[1:] - pr[:-1], diff_zero), 0), min=0) + + diff = torch.clamp(torch.cat(((pr[1:] - pr[:-1]), diff_zero), 0), min=0) pr += diff inds = torch.searchsorted(rc, rec_thresholds.to(rc.device), right=False) @@ -846,8 +868,6 @@ def compute(self) -> dict: - mar_100_per_class: ``torch.Tensor`` (-1 if class metrics are disabled) """ - # move everything to CPU, as we are faster here - classes = self._get_classes() precisions, recalls = self._calculate(classes) map_val, mar_val = self._summarize_results(precisions, recalls) @@ -875,4 +895,10 @@ def compute(self) -> dict: metrics.map_per_class = map_per_class_values metrics[f"mar_{self.max_detection_thresholds[-1]}_per_class"] = mar_max_dets_per_class_values + # Reset + # self.detections = [] + # self.detection_labels = [] + # self.detection_scores = [] + # self.groundtruths = [] + # self.groundtruth_labels = [] return metrics From 08847125343f19f17af9acea3bc5fd8f28f6840b Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 5 May 2022 16:49:13 +0200 Subject: [PATCH 112/141] fixes for tensor --- tests/detection/test_map.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 11790333ca7..d4389b1f879 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -19,6 +19,7 @@ import pytest import torch from pycocotools import mask +from torch import Tensor from tests.helpers.testers import MetricTester from torchmetrics.detection.mean_ap import MeanAveragePrecision From ff6f315872d13d570e2aeaecb2fadbcad861f814 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 14:52:57 +0000 Subject: [PATCH 113/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/detection/mean_ap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 44641a69c96..cb990184baa 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -297,10 +297,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From d41435aaf213de4d2623bfcbec60eda7f57b0a36 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 5 May 2022 17:12:44 +0200 Subject: [PATCH 114/141] fixed docstring --- torchmetrics/detection/mean_ap.py | 50 ++++++++++++++++++------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 44641a69c96..45ecbeb0dd9 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +from enum import Enum from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple import numpy as np @@ -31,32 +32,37 @@ log = logging.getLogger(__name__) -def mask_area(input): - n_inputs = len(input) - - return input.reshape(n_inputs, -1).sum(1) - - -def compute_area(input, type="bbox"): +def compute_area(input, iou_type="bbox"): + """ + Compute area of input depending on the specified iou_type. Default output for empty input is torch.Tensor([]) + """ if len(input) == 0: return torch.Tensor([]) - if type == "bbox": + if iou_type == "bbox": return box_area(torch.stack(input)) - else: + elif iou_type == "segm": input = [{"size": i[0], "counts": i[1]} for i in input] area = torch.tensor(mask_utils.area(input).astype("int")) return area + else: + raise Exception(f"IOU type {iou_type} is not supported") -def compute_iou(det, gt, type="bbox") -> Tensor: - if type == "bbox": +def compute_iou(det, gt, iou_type="bbox") -> Tensor: + """ + Compute IOU between detections and ground-truth using the specified iou_type + """ + + if iou_type == "bbox": return box_iou(torch.stack(det), torch.stack(gt)) + elif iou_type == "segm": + return _segm_iou(det, gt) else: - return segm_iou(det, gt) + raise Exception(f"IOU type {iou_type} is not supported") class BaseMetricResults(dict): @@ -110,11 +116,15 @@ class COCOMetricResults(BaseMetricResults): ) -def segm_iou(det, gt): - if isinstance(det, dict): - det = [det] - if isinstance(gt, dict): - gt = [gt] +def _segm_iou(det, gt): + """ + Compute IOU between detections and ground-truths using mask-IOU. Based on pycocotools toolkit for mask_utils + Args: + det: A list of detection masks as [(RLE_SIZE, RLE_COUNTS)], where RLE_SIZE is (width, height) dimension of the input and RLE_COUNTS is its RLE representation; + + gt: A list of ground-truth masks as [(RLE_SIZE, RLE_COUNTS)], where RLE_SIZE is (width, height) dimension of the input and RLE_COUNTS is its RLE representation; + + """ det = [{"size": i[0], "counts": i[1]} for i in det] gt = [{"size": i[0], "counts": i[1]} for i in gt] @@ -492,7 +502,7 @@ def __evaluate_image_preds_no_gt( if len(det) > max_det: det = det[:max_det] nb_det = len(det) - det_areas = compute_area(det, type=self.iou_type).to(self.device) + det_areas = compute_area(det, iou_type=self.iou_type).to(self.device) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.repeat_interleave(ar, nb_iou_thrs, 0) @@ -551,7 +561,7 @@ def _evaluate_image( if isinstance(gt, dict): gt = [gt] - areas = compute_area(gt, self.iou_type).to(self.device) + areas = compute_area(gt, iou_type=self.iou_type).to(self.device) ignore_area = torch.tensor(areas < area_range[0]) | (areas > area_range[1]) # sort dt highest score first, sort gt ignore last @@ -589,7 +599,7 @@ def _evaluate_image( gt_matches[idx_iou, m] = 1 # set unmatched detections outside of area range to ignore - det_areas = compute_area(det, self.iou_type).to(self.device) + det_areas = compute_area(det, iou_type=self.iou_type).to(self.device) det_ignore_area = (det_areas < area_range[0]) | (det_areas > area_range[1]) ar = det_ignore_area.reshape((1, nb_det)) det_ignore = torch.logical_or( From c2bef392ed4b947ec430fac07926e5ef30d344c9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 15:13:52 +0000 Subject: [PATCH 115/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/detection/mean_ap.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 38f37388b0d..5b267a3b439 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -33,8 +33,9 @@ def compute_area(input, iou_type="bbox"): - """ - Compute area of input depending on the specified iou_type. Default output for empty input is torch.Tensor([]) + """Compute area of input depending on the specified iou_type. + + Default output for empty input is torch.Tensor([]) """ if len(input) == 0: @@ -53,9 +54,7 @@ def compute_area(input, iou_type="bbox"): def compute_iou(det, gt, iou_type="bbox") -> Tensor: - """ - Compute IOU between detections and ground-truth using the specified iou_type - """ + """Compute IOU between detections and ground-truth using the specified iou_type.""" if iou_type == "bbox": return box_iou(torch.stack(det), torch.stack(gt)) From c0dc2429eb04a6329ba39ffc47e96d5482cf7d53 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Thu, 5 May 2022 17:16:55 +0200 Subject: [PATCH 116/141] minor fixesz --- torchmetrics/detection/mean_ap.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 38f37388b0d..84f1eadfcba 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from enum import Enum -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple import numpy as np import pycocotools.mask as mask_utils @@ -307,10 +306,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): From fbc358beaa95bb742cece0940494479937823a7d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 15:18:09 +0000 Subject: [PATCH 117/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/detection/mean_ap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 56577908c18..268e3df1209 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -305,10 +305,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 15187d6e6ff35c1a15409044f8a004470e58d2fc Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 5 May 2022 18:27:39 +0200 Subject: [PATCH 118/141] flake8 --- torchmetrics/detection/mean_ap.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 268e3df1209..51caf286101 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -118,9 +118,11 @@ def _segm_iou(det, gt): """ Compute IOU between detections and ground-truths using mask-IOU. Based on pycocotools toolkit for mask_utils Args: - det: A list of detection masks as [(RLE_SIZE, RLE_COUNTS)], where RLE_SIZE is (width, height) dimension of the input and RLE_COUNTS is its RLE representation; + det: A list of detection masks as ``[(RLE_SIZE, RLE_COUNTS)]``, where ``RLE_SIZE`` is (width, height) dimension + of the input and RLE_COUNTS is its RLE representation; - gt: A list of ground-truth masks as [(RLE_SIZE, RLE_COUNTS)], where RLE_SIZE is (width, height) dimension of the input and RLE_COUNTS is its RLE representation; + gt: A list of ground-truth masks as ``[(RLE_SIZE, RLE_COUNTS)]``, where ``RLE_SIZE`` is (width, height) dimension + of the input and RLE_COUNTS is its RLE representation; """ @@ -212,7 +214,8 @@ class MeanAveragePrecision(Metric): box_format: Input format of given boxes. Supported formats are ``[`xyxy`, `xywh`, `cxcywh`]``. iou_type: - Type of input (either masks or bounding-boxes) used for computing IOU. Supported IOU types are ``[`bboxes`, `segm`]``. + Type of input (either masks or bounding-boxes) used for computing IOU. + Supported IOU types are ``[`bboxes`, `segm`]``. iou_thresholds: IoU thresholds for evaluation. If set to ``None`` it corresponds to the stepped range ``[0.5,...,0.95]`` with step ``0.05``. Else provide a list of floats. From f5033b9ff605ac60acc1b41e64a80574b1117594 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Tue, 10 May 2022 15:12:01 +0200 Subject: [PATCH 119/141] fixing docstring --- torchmetrics/detection/mean_ap.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 56577908c18..a9e33db4216 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import numpy as np import pycocotools.mask as mask_utils @@ -31,7 +31,7 @@ log = logging.getLogger(__name__) -def compute_area(input, iou_type="bbox"): +def compute_area(input: List[Any], iou_type: str = "bbox") -> Tensor: """Compute area of input depending on the specified iou_type. Default output for empty input is torch.Tensor([]) @@ -52,7 +52,11 @@ def compute_area(input, iou_type="bbox"): raise Exception(f"IOU type {iou_type} is not supported") -def compute_iou(det, gt, iou_type="bbox") -> Tensor: +def compute_iou( + det: List[Any], + gt: List[Any], + iou_type: str = "bbox", +) -> Tensor: """Compute IOU between detections and ground-truth using the specified iou_type.""" if iou_type == "bbox": @@ -114,7 +118,7 @@ class COCOMetricResults(BaseMetricResults): ) -def _segm_iou(det, gt): +def _segm_iou(det: List[Tuple[np.ndarray, np.ndarray]], gt: List[Tuple[np.ndarray, np.ndarray]]) -> torch.Tensor: """ Compute IOU between detections and ground-truths using mask-IOU. Based on pycocotools toolkit for mask_utils Args: @@ -391,7 +395,7 @@ def _move_list_states_to_cpu(self) -> None: current_to_cpu.append(cur_v) setattr(self, key, current_to_cpu) - def _get_safe_item_values(self, item): + def _get_safe_item_values(self, item: Dict[str, Any]) -> Union[Tensor, Tuple]: if self.iou_type == "bbox": boxes = _fix_empty_tensors(item["boxes"]) From 2e7bd83a12e0938bb67e98733c6f17399e37563c Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Tue, 10 May 2022 15:16:37 +0200 Subject: [PATCH 120/141] doctest for pycocotools --- torchmetrics/detection/mean_ap.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index a9e33db4216..a68b38fdb5c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -15,12 +15,11 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import numpy as np -import pycocotools.mask as mask_utils import torch from torch import IntTensor, Tensor from torchmetrics.metric import Metric -from torchmetrics.utilities.imports import _TORCHVISION_GREATER_EQUAL_0_8 +from torchmetrics.utilities.imports import _PYCOCOTOOLS_AVAILABLE, _TORCHVISION_GREATER_EQUAL_0_8 if _TORCHVISION_GREATER_EQUAL_0_8: from torchvision.ops import box_area, box_convert, box_iou @@ -28,6 +27,13 @@ box_convert = box_iou = box_area = None __doctest_skip__ = ["MeanAveragePrecision"] +if _PYCOCOTOOLS_AVAILABLE: + import pycocotools.mask as mask_utils +else: + mask_utils = None + __doctest_skip__ = ["MeanAveragePrecision"] + + log = logging.getLogger(__name__) From 2cd7080beb83049a78d98f79d2c7ba8226a64137 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Fri, 13 May 2022 14:40:04 +0200 Subject: [PATCH 121/141] small fix for myppy --- torchmetrics/detection/mean_ap.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 3eeb2b6d34c..278ef766230 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -136,10 +136,10 @@ def _segm_iou(det: List[Tuple[np.ndarray, np.ndarray]], gt: List[Tuple[np.ndarra """ - det = [{"size": i[0], "counts": i[1]} for i in det] - gt = [{"size": i[0], "counts": i[1]} for i in gt] + det_coco_format = [{"size": i[0], "counts": i[1]} for i in det] + gt_coco_format = [{"size": i[0], "counts": i[1]} for i in gt] - return torch.tensor(mask_utils.iou(det, gt, [False for _ in gt])) + return torch.tensor(mask_utils.iou(det_coco_format, gt_coco_format, [False for _ in gt])) def _input_validator( @@ -318,10 +318,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): From 02d1c1ed265ccdf839484df0c2708f6ccfc6fc54 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 13 May 2022 12:41:06 +0000 Subject: [PATCH 122/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/detection/mean_ap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 0e924d6fe12..5eaa23b777b 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -321,10 +321,10 @@ def __init__( raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 56cf440676bc98ed35b7046cb970c903a8e41470 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 16 May 2022 09:09:46 +0200 Subject: [PATCH 123/141] simple --- tests/detection/test_map.py | 190 ++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 107 deletions(-) diff --git a/tests/detection/test_map.py b/tests/detection/test_map.py index 31b3f9dc28c..8daf40e487a 100644 --- a/tests/detection/test_map.py +++ b/tests/detection/test_map.py @@ -19,7 +19,7 @@ import pytest import torch from pycocotools import mask -from torch import Tensor +from torch import IntTensor, Tensor from tests.helpers.testers import MetricTester from torchmetrics.detection.mean_ap import MeanAveragePrecision @@ -27,54 +27,30 @@ Input = namedtuple("Input", ["preds", "target"]) -with open("tests/detection/instance_segmentation_inputs.json") as f: - inputs_json = json.load(f) +with open("tests/detection/instance_segmentation_inputs.json") as fp: + inputs_json = json.load(fp) + +_mask_unsqueeze_bool = lambda m: Tensor(mask.decode(m)).unsqueeze(0).bool() +_masks_stack_bool = lambda ms: Tensor(np.stack([mask.decode(m) for m in ms])).bool() _inputs_masks = Input( preds=[ [ + dict(masks=_mask_unsqueeze_bool(inputs_json["preds"][0]), scores=Tensor([0.236]), labels=IntTensor([4])), dict( - masks=torch.Tensor( - mask.decode( - inputs_json["preds"][0], - ) - ) - .unsqueeze(0) - .bool(), - scores=torch.Tensor([0.236]), - labels=torch.IntTensor([4]), - ), - dict( - masks=torch.Tensor( - np.stack( - [ - mask.decode(inputs_json["preds"][1]), - mask.decode(inputs_json["preds"][2]), - ] - ) - ).bool(), - scores=torch.Tensor([0.318, 0.726]), - labels=torch.IntTensor([3, 2]), + masks=_masks_stack_bool([inputs_json["preds"][1], inputs_json["preds"][2]]), + scores=Tensor([0.318, 0.726]), + labels=IntTensor([3, 2]), ), # 73 ], ], target=[ [ + dict(masks=_mask_unsqueeze_bool(inputs_json["targets"][0]), labels=IntTensor([4])), # 42 dict( - masks=torch.Tensor(mask.decode(inputs_json["targets"][0])).unsqueeze(0).bool(), - labels=torch.IntTensor([4]), - ), # 42 - dict( - labels=torch.IntTensor([2, 2]), - masks=torch.Tensor( - np.stack( - [ - mask.decode(inputs_json["targets"][1]), - mask.decode(inputs_json["targets"][2]), - ] - ) - ).bool(), # 73 - ), + masks=_masks_stack_bool([inputs_json["targets"][1], inputs_json["targets"][2]]), + labels=IntTensor([2, 2]), + ), # 73 ], ], ) @@ -86,12 +62,12 @@ dict( boxes=Tensor([[258.15, 41.29, 606.41, 285.07]]), scores=Tensor([0.236]), - labels=torch.IntTensor([4]), + labels=IntTensor([4]), ), # coco image id 42 dict( boxes=Tensor([[61.00, 22.75, 565.00, 632.42], [12.66, 3.32, 281.26, 275.23]]), scores=Tensor([0.318, 0.726]), - labels=torch.IntTensor([3, 2]), + labels=IntTensor([3, 2]), ), # coco image id 73 ], [ @@ -108,12 +84,12 @@ ] ), scores=Tensor([0.546, 0.3, 0.407, 0.611, 0.335, 0.805, 0.953]), - labels=torch.IntTensor([4, 1, 0, 0, 0, 0, 0]), + labels=IntTensor([4, 1, 0, 0, 0, 0, 0]), ), # coco image id 74 dict( boxes=Tensor([[0.00, 2.87, 601.00, 421.52]]), scores=Tensor([0.699]), - labels=torch.IntTensor([5]), + labels=IntTensor([5]), ), # coco image id 133 ], ], @@ -121,7 +97,7 @@ [ dict( boxes=Tensor([[214.1500, 41.2900, 562.4100, 285.0700]]), - labels=torch.IntTensor([4]), + labels=IntTensor([4]), ), # coco image id 42 dict( boxes=Tensor( @@ -130,7 +106,7 @@ [1.66, 3.32, 270.26, 275.23], ] ), - labels=torch.IntTensor([2, 2]), + labels=IntTensor([2, 2]), ), # coco image id 73 ], [ @@ -146,11 +122,11 @@ [277.11, 103.84, 292.44, 150.72], ] ), - labels=torch.IntTensor([4, 1, 0, 0, 0, 0, 0]), + labels=IntTensor([4, 1, 0, 0, 0, 0, 0]), ), # coco image id 74 dict( boxes=Tensor([[13.99, 2.87, 640.00, 421.52]]), - labels=torch.IntTensor([5]), + labels=IntTensor([5]), ), # coco image id 133 ], ], @@ -163,14 +139,14 @@ dict( boxes=Tensor([[258.0, 41.0, 606.0, 285.0]]), scores=Tensor([0.536]), - labels=torch.IntTensor([0]), + labels=IntTensor([0]), ), ], [ dict( boxes=Tensor([[258.0, 41.0, 606.0, 285.0]]), scores=Tensor([0.536]), - labels=torch.IntTensor([0]), + labels=IntTensor([0]), ) ], ], @@ -178,13 +154,13 @@ [ dict( boxes=Tensor([[214.0, 41.0, 562.0, 285.0]]), - labels=torch.IntTensor([0]), + labels=IntTensor([0]), ) ], [ dict( boxes=Tensor([]), - labels=torch.IntTensor([]), + labels=IntTensor([]), ) ], ], @@ -195,15 +171,15 @@ _inputs3 = Input( preds=[ [ - dict(boxes=torch.tensor([]), scores=torch.tensor([]), labels=torch.tensor([])), + dict(boxes=Tensor([]), scores=Tensor([]), labels=Tensor([])), ], ], target=[ [ dict( - boxes=torch.tensor([[1.0, 2.0, 3.0, 4.0]]), - scores=torch.tensor([0.8]), - labels=torch.tensor([1]), + boxes=Tensor([[1.0, 2.0, 3.0, 4.0]]), + scores=Tensor([0.8]), + labels=Tensor([1]), ), ], ], @@ -288,20 +264,20 @@ def _compare_fn_segm(preds, target) -> dict: Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.350 """ return { - "map": torch.Tensor([0.352]), - "map_50": torch.Tensor([0.742]), - "map_75": torch.Tensor([0.252]), - "map_small": torch.Tensor([-1]), - "map_medium": torch.Tensor([-1]), - "map_large": torch.Tensor([0.352]), - "mar_1": torch.Tensor([0.35]), - "mar_10": torch.Tensor([0.35]), - "mar_100": torch.Tensor([0.35]), - "mar_small": torch.Tensor([-1]), - "mar_medium": torch.Tensor([-1]), - "mar_large": torch.Tensor([0.35]), - "map_per_class": torch.Tensor([0.4039604, -1.0, 0.3]), - "mar_100_per_class": torch.Tensor([0.4, -1.0, 0.3]), + "map": Tensor([0.352]), + "map_50": Tensor([0.742]), + "map_75": Tensor([0.252]), + "map_small": Tensor([-1]), + "map_medium": Tensor([-1]), + "map_large": Tensor([0.352]), + "mar_1": Tensor([0.35]), + "mar_10": Tensor([0.35]), + "mar_100": Tensor([0.35]), + "mar_small": Tensor([-1]), + "mar_medium": Tensor([-1]), + "mar_large": Tensor([0.35]), + "map_per_class": Tensor([0.4039604, -1.0, 0.3]), + "mar_100_per_class": Tensor([0.4, -1.0, 0.3]), } @@ -368,10 +344,10 @@ def test_empty_preds(): metric.update( [ - dict(boxes=Tensor([]), scores=torch.Tensor([]), labels=torch.IntTensor([])), + dict(boxes=Tensor([]), scores=Tensor([]), labels=IntTensor([])), ], [ - dict(boxes=torch.Tensor([[214.1500, 41.2900, 562.4100, 285.0700]]), labels=torch.IntTensor([4])), + dict(boxes=Tensor([[214.1500, 41.2900, 562.4100, 285.0700]]), labels=IntTensor([4])), ], ) metric.compute() @@ -385,13 +361,13 @@ def test_empty_ground_truths(): metric.update( [ dict( - boxes=torch.Tensor([[214.1500, 41.2900, 562.4100, 285.0700]]), - scores=torch.Tensor([0.5]), - labels=torch.IntTensor([4]), + boxes=Tensor([[214.1500, 41.2900, 562.4100, 285.0700]]), + scores=Tensor([0.5]), + labels=IntTensor([4]), ), ], [ - dict(boxes=torch.Tensor([]), labels=torch.IntTensor([])), + dict(boxes=Tensor([]), labels=IntTensor([])), ], ) metric.compute() @@ -448,13 +424,13 @@ def test_missing_pred(): values) """ gts = [ - dict(boxes=torch.Tensor([[10, 20, 15, 25]]), labels=torch.IntTensor([0])), - dict(boxes=torch.Tensor([[10, 20, 15, 25]]), labels=torch.IntTensor([0])), + dict(boxes=Tensor([[10, 20, 15, 25]]), labels=IntTensor([0])), + dict(boxes=Tensor([[10, 20, 15, 25]]), labels=IntTensor([0])), ] preds = [ - dict(boxes=torch.Tensor([[10, 20, 15, 25]]), scores=torch.Tensor([0.9]), labels=torch.IntTensor([0])), + dict(boxes=Tensor([[10, 20, 15, 25]]), scores=Tensor([0.9]), labels=IntTensor([0])), # Empty prediction - dict(boxes=torch.Tensor([]), scores=torch.Tensor([]), labels=torch.IntTensor([])), + dict(boxes=Tensor([]), scores=Tensor([]), labels=IntTensor([])), ] metric = MeanAveragePrecision() metric.update(preds, gts) @@ -470,12 +446,12 @@ def test_missing_gt(): on where we are sampling (i.e. recall's values) """ gts = [ - dict(boxes=torch.Tensor([[10, 20, 15, 25]]), labels=torch.IntTensor([0])), - dict(boxes=torch.Tensor([]), labels=torch.IntTensor([])), + dict(boxes=Tensor([[10, 20, 15, 25]]), labels=IntTensor([0])), + dict(boxes=Tensor([]), labels=IntTensor([])), ] preds = [ - dict(boxes=torch.Tensor([[10, 20, 15, 25]]), scores=torch.Tensor([0.9]), labels=torch.IntTensor([0])), - dict(boxes=torch.Tensor([[10, 20, 15, 25]]), scores=torch.Tensor([0.95]), labels=torch.IntTensor([0])), + dict(boxes=Tensor([[10, 20, 15, 25]]), scores=Tensor([0.9]), labels=IntTensor([0])), + dict(boxes=Tensor([[10, 20, 15, 25]]), scores=Tensor([0.95]), labels=IntTensor([0])), ] metric = MeanAveragePrecision() @@ -493,12 +469,12 @@ def test_segm_iou_empty_mask(): [ dict( masks=torch.randint(0, 1, (1, 10, 10)).bool(), - scores=torch.Tensor([0.5]), - labels=torch.IntTensor([4]), + scores=Tensor([0.5]), + labels=IntTensor([4]), ), ], [ - dict(masks=torch.Tensor([]), labels=torch.IntTensor([])), + dict(masks=Tensor([]), labels=IntTensor([])), ], ) @@ -513,70 +489,70 @@ def test_error_on_wrong_input(): metric.update([], []) # no error with pytest.raises(ValueError, match="Expected argument `preds` to be of type Sequence"): - metric.update(torch.Tensor(), []) # type: ignore + metric.update(Tensor(), []) # type: ignore with pytest.raises(ValueError, match="Expected argument `target` to be of type Sequence"): - metric.update([], torch.Tensor()) # type: ignore + metric.update([], Tensor()) # type: ignore with pytest.raises(ValueError, match="Expected argument `preds` and `target` to have the same length"): metric.update([{}], [{}, {}]) with pytest.raises(ValueError, match="Expected all dicts in `preds` to contain the `boxes` key"): metric.update( - [dict(scores=torch.Tensor(), labels=torch.IntTensor)], - [dict(boxes=torch.Tensor(), labels=torch.IntTensor())], + [dict(scores=Tensor(), labels=IntTensor)], + [dict(boxes=Tensor(), labels=IntTensor())], ) with pytest.raises(ValueError, match="Expected all dicts in `preds` to contain the `scores` key"): metric.update( - [dict(boxes=torch.Tensor(), labels=torch.IntTensor)], - [dict(boxes=torch.Tensor(), labels=torch.IntTensor())], + [dict(boxes=Tensor(), labels=IntTensor)], + [dict(boxes=Tensor(), labels=IntTensor())], ) with pytest.raises(ValueError, match="Expected all dicts in `preds` to contain the `labels` key"): metric.update( - [dict(boxes=torch.Tensor(), scores=torch.IntTensor)], - [dict(boxes=torch.Tensor(), labels=torch.IntTensor())], + [dict(boxes=Tensor(), scores=IntTensor)], + [dict(boxes=Tensor(), labels=IntTensor())], ) with pytest.raises(ValueError, match="Expected all dicts in `target` to contain the `boxes` key"): metric.update( - [dict(boxes=torch.Tensor(), scores=torch.IntTensor, labels=torch.IntTensor)], - [dict(labels=torch.IntTensor())], + [dict(boxes=Tensor(), scores=IntTensor, labels=IntTensor)], + [dict(labels=IntTensor())], ) with pytest.raises(ValueError, match="Expected all dicts in `target` to contain the `labels` key"): metric.update( - [dict(boxes=torch.Tensor(), scores=torch.IntTensor, labels=torch.IntTensor)], - [dict(boxes=torch.IntTensor())], + [dict(boxes=Tensor(), scores=IntTensor, labels=IntTensor)], + [dict(boxes=IntTensor())], ) with pytest.raises(ValueError, match="Expected all boxes in `preds` to be of type Tensor"): metric.update( - [dict(boxes=[], scores=torch.Tensor(), labels=torch.IntTensor())], - [dict(boxes=torch.Tensor(), labels=torch.IntTensor())], + [dict(boxes=[], scores=Tensor(), labels=IntTensor())], + [dict(boxes=Tensor(), labels=IntTensor())], ) with pytest.raises(ValueError, match="Expected all scores in `preds` to be of type Tensor"): metric.update( - [dict(boxes=torch.Tensor(), scores=[], labels=torch.IntTensor())], - [dict(boxes=torch.Tensor(), labels=torch.IntTensor())], + [dict(boxes=Tensor(), scores=[], labels=IntTensor())], + [dict(boxes=Tensor(), labels=IntTensor())], ) with pytest.raises(ValueError, match="Expected all labels in `preds` to be of type Tensor"): metric.update( - [dict(boxes=torch.Tensor(), scores=torch.Tensor(), labels=[])], - [dict(boxes=torch.Tensor(), labels=torch.IntTensor())], + [dict(boxes=Tensor(), scores=Tensor(), labels=[])], + [dict(boxes=Tensor(), labels=IntTensor())], ) with pytest.raises(ValueError, match="Expected all boxes in `target` to be of type Tensor"): metric.update( - [dict(boxes=torch.Tensor(), scores=torch.Tensor(), labels=torch.IntTensor())], - [dict(boxes=[], labels=torch.IntTensor())], + [dict(boxes=Tensor(), scores=Tensor(), labels=IntTensor())], + [dict(boxes=[], labels=IntTensor())], ) with pytest.raises(ValueError, match="Expected all labels in `target` to be of type Tensor"): metric.update( - [dict(boxes=torch.Tensor(), scores=torch.Tensor(), labels=torch.IntTensor())], - [dict(boxes=torch.Tensor(), labels=[])], + [dict(boxes=Tensor(), scores=Tensor(), labels=IntTensor())], + [dict(boxes=Tensor(), labels=[])], ) From bd13c23d20bc913527390982b3f4c8a1f66f4f4a Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 16 May 2022 09:11:31 +0200 Subject: [PATCH 124/141] req --- requirements/detection_test.txt | 1 + requirements/devel.txt | 1 + requirements/test.txt | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 requirements/detection_test.txt diff --git a/requirements/detection_test.txt b/requirements/detection_test.txt new file mode 100644 index 00000000000..4a931d80225 --- /dev/null +++ b/requirements/detection_test.txt @@ -0,0 +1 @@ +pycocotools \ No newline at end of file diff --git a/requirements/devel.txt b/requirements/devel.txt index 2982b7de3ce..757c79a82ae 100644 --- a/requirements/devel.txt +++ b/requirements/devel.txt @@ -14,3 +14,4 @@ -r image_test.txt -r text_test.txt -r audio_test.txt +-r detection_test.txt diff --git a/requirements/test.txt b/requirements/test.txt index 78c17457c98..dfffc35433d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -16,4 +16,3 @@ fire cloudpickle>=1.3 scikit-learn>=0.24 -pycocotools From 688bb944907a336f2d69d2ff4512ff6fdd967d08 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 07:12:13 +0000 Subject: [PATCH 125/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements/detection_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/detection_test.txt b/requirements/detection_test.txt index 4a931d80225..99f22cd1258 100644 --- a/requirements/detection_test.txt +++ b/requirements/detection_test.txt @@ -1 +1 @@ -pycocotools \ No newline at end of file +pycocotools From d3d5b76d18729023cbbe6aef6ec67f2c9e1f78e3 Mon Sep 17 00:00:00 2001 From: Jirka Date: Mon, 16 May 2022 09:15:15 +0200 Subject: [PATCH 126/141] idx --- torchmetrics/detection/mean_ap.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 5eaa23b777b..adcec89c70d 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -430,7 +430,7 @@ def _get_classes(self) -> List: return torch.cat(self.detection_labels + self.groundtruth_labels).unique().tolist() return [] - def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: + def _compute_iou(self, idx: int, class_id: int, max_det: int) -> Tensor: """Computes the Intersection over Union (IoU) for ground truth and detection bounding boxes for the given image and class. @@ -444,11 +444,11 @@ def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: """ # if self.iou_type == "bbox": - gt = self.groundtruths[id] - det = self.detections[id] + gt = self.groundtruths[idx] + det = self.detections[idx] - gt_label_mask = (self.groundtruth_labels[id] == class_id).nonzero().squeeze(1) - det_label_mask = (self.detection_labels[id] == class_id).nonzero().squeeze(1) + gt_label_mask = (self.groundtruth_labels[idx] == class_id).nonzero().squeeze(1) + det_label_mask = (self.detection_labels[idx] == class_id).nonzero().squeeze(1) if len(gt_label_mask) == 0 or len(det_label_mask) == 0: return Tensor([]) @@ -460,8 +460,8 @@ def _compute_iou(self, id: int, class_id: int, max_det: int) -> Tensor: return Tensor([]) # Sort by scores and use only max detections - scores = self.detection_scores[id] - scores_filtered = scores[self.detection_labels[id] == class_id] + scores = self.detection_scores[idx] + scores_filtered = scores[self.detection_labels[idx] == class_id] inds = torch.argsort(scores_filtered, descending=True) # TODO Fix (only for masks is necessary) From 4bfc12842d5535b1cabb9d46b04a74f101166c3b Mon Sep 17 00:00:00 2001 From: Gianluca Scarpellini Date: Tue, 17 May 2022 10:06:13 +0000 Subject: [PATCH 127/141] Update torchmetrics/detection/mean_ap.py Co-authored-by: Nicki Skafte Detlefsen --- torchmetrics/detection/mean_ap.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index adcec89c70d..c908dc2a8fd 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -319,6 +319,8 @@ def __init__( self.max_detection_thresholds = max_det_thr.tolist() if iou_type not in allowed_iou_types: raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") + if iou_type == 'segm' and not _PYCOCOTOOLS_AVAILABLE: + raise ValueError("When `iou_type` is set to 'segm', pycocotools need to be installed") self.iou_type = iou_type self.bbox_area_ranges = { "all": (0**2, int(1e5**2)), From f29428517cc7b8e47d73736c22d6e50cdea0f953 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Tue, 17 May 2022 12:07:35 +0200 Subject: [PATCH 128/141] pycocotools requirements --- requirements/detection.txt | 1 + requirements/detection_test.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/detection.txt b/requirements/detection.txt index cbec7260775..da90ea549b7 100644 --- a/requirements/detection.txt +++ b/requirements/detection.txt @@ -1 +1,2 @@ torchvision>=0.8 +pycocotools diff --git a/requirements/detection_test.txt b/requirements/detection_test.txt index 99f22cd1258..8b137891791 100644 --- a/requirements/detection_test.txt +++ b/requirements/detection_test.txt @@ -1 +1 @@ -pycocotools + From 7f641f8598d8df1b8bfd56dd4c51d250b4dab0b4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 May 2022 10:07:36 +0000 Subject: [PATCH 129/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index c908dc2a8fd..11d535052e3 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -319,7 +319,7 @@ def __init__( self.max_detection_thresholds = max_det_thr.tolist() if iou_type not in allowed_iou_types: raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") - if iou_type == 'segm' and not _PYCOCOTOOLS_AVAILABLE: + if iou_type == "segm" and not _PYCOCOTOOLS_AVAILABLE: raise ValueError("When `iou_type` is set to 'segm', pycocotools need to be installed") self.iou_type = iou_type self.bbox_area_ranges = { From 5185ba1e56444a661788b9c3a9e0455388ec2103 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 May 2022 12:06:09 +0000 Subject: [PATCH 130/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements/detection_test.txt | 1 - torchmetrics/detection/mean_ap.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/requirements/detection_test.txt b/requirements/detection_test.txt index 8b137891791..e69de29bb2d 100644 --- a/requirements/detection_test.txt +++ b/requirements/detection_test.txt @@ -1 +0,0 @@ - diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 291f9bb6492..51e5c73570e 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -324,10 +324,10 @@ def __init__( raise ValueError("When `iou_type` is set to 'segm', pycocotools need to be installed") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 07e728ab09d953d58902c547c5cf83d322dce5b4 Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Tue, 17 May 2022 14:34:51 +0200 Subject: [PATCH 131/141] pycocotools both in test_detection and detection requirements? --- requirements/detection_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/detection_test.txt b/requirements/detection_test.txt index 8b137891791..99f22cd1258 100644 --- a/requirements/detection_test.txt +++ b/requirements/detection_test.txt @@ -1 +1 @@ - +pycocotools From 03a0e2c92dac20bc59c06bea5e26424c5955bf00 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 May 2022 12:36:02 +0000 Subject: [PATCH 132/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements/detection_test.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/detection_test.txt b/requirements/detection_test.txt index d4f6f86a458..99f22cd1258 100644 --- a/requirements/detection_test.txt +++ b/requirements/detection_test.txt @@ -1,2 +1 @@ pycocotools - From ac2b9e25279553f105fee04f4cd8f0937a67d29f Mon Sep 17 00:00:00 2001 From: gianscarpe Date: Tue, 17 May 2022 15:13:26 +0200 Subject: [PATCH 133/141] fix test for logical_or tensor overflown (windows only) --- torchmetrics/detection/mean_ap.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 51e5c73570e..c460df8c26c 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -324,10 +324,10 @@ def __init__( raise ValueError("When `iou_type` is set to 'segm', pycocotools need to be installed") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0**2, int(1e5**2)), - "small": (0**2, 32**2), - "medium": (32**2, 96**2), - "large": (96**2, int(1e5**2)), + "all": (0 ** 2, int(1e5 ** 2)), + "small": (0 ** 2, 32 ** 2), + "medium": (32 ** 2, 96 ** 2), + "large": (96 ** 2, int(1e5 ** 2)), } if not isinstance(class_metrics, bool): @@ -579,7 +579,8 @@ def _evaluate_image( gt = [gt] areas = compute_area(gt, iou_type=self.iou_type).to(self.device) - ignore_area = torch.tensor(float((areas < area_range[0]) | (areas > area_range[1]))) + + ignore_area = torch.logical_or(areas < area_range[0], areas > area_range[1]) # sort dt highest score first, sort gt ignore last ignore_area_sorted, gtind = torch.sort(ignore_area.to(torch.uint8)) From 9eff9068ea35df10f0c14aa0fbeeac550e2349db Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 May 2022 13:15:23 +0000 Subject: [PATCH 134/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/detection/mean_ap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index c460df8c26c..e18128872de 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -324,10 +324,10 @@ def __init__( raise ValueError("When `iou_type` is set to 'segm', pycocotools need to be installed") self.iou_type = iou_type self.bbox_area_ranges = { - "all": (0 ** 2, int(1e5 ** 2)), - "small": (0 ** 2, 32 ** 2), - "medium": (32 ** 2, 96 ** 2), - "large": (96 ** 2, int(1e5 ** 2)), + "all": (0**2, int(1e5**2)), + "small": (0**2, 32**2), + "medium": (32**2, 96**2), + "large": (96**2, int(1e5**2)), } if not isinstance(class_metrics, bool): From 9d3e5c53674986258fd01ea51aad584f3f41a660 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Thu, 19 May 2022 10:02:27 +0200 Subject: [PATCH 135/141] Update torchmetrics/detection/mean_ap.py --- torchmetrics/detection/mean_ap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index e18128872de..02281c5bbae 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -51,7 +51,7 @@ def compute_area(input: List[Any], iou_type: str = "bbox") -> Tensor: elif iou_type == "segm": input = [{"size": i[0], "counts": i[1]} for i in input] - area = torch.tensor(mask_utils.area(input).astype("int")) + area = torch.tensor(mask_utils.area(input).astype("float")) return area else: From 7c3d2b7e978057b3769a18d047e33b4213e32eac Mon Sep 17 00:00:00 2001 From: Skaftenicki Date: Thu, 19 May 2022 10:08:34 +0200 Subject: [PATCH 136/141] update docs --- torchmetrics/detection/mean_ap.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index 02281c5bbae..d87136543d2 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -216,9 +216,9 @@ class MeanAveragePrecision(Metric): a standard implementation for the mAP metric for object detection. .. note:: - This metric requires you to have `torchvision` version 0.8.0 or newer installed (with corresponding. - This metric requires `pycocotools` installed when iou_type is `segm` - version 1.7.0 of torch or newer). Please install with ``pip install torchvision`` or + This metric requires you to have `torchvision` version 0.8.0 or newer installed + (with corresponding version 1.7.0 of torch or newer). This metric requires `pycocotools` + installed when iou_type is `segm`. Please install with ``pip install torchvision`` or ``pip install torchmetrics[detection]``. Args: @@ -278,6 +278,8 @@ class MeanAveragePrecision(Metric): Raises: ModuleNotFoundError: If ``torchvision`` is not installed or version installed is lower than 0.8.0 + ModuleNotFoundError: + If ``iou_type`` is equal to ``seqm`` and ``pycocotools`` is not installed ValueError: If ``class_metrics`` is not a boolean """ @@ -321,7 +323,7 @@ def __init__( if iou_type not in allowed_iou_types: raise ValueError(f"Expected argument `iou_type` to be one of {allowed_iou_types} but got {iou_type}") if iou_type == "segm" and not _PYCOCOTOOLS_AVAILABLE: - raise ValueError("When `iou_type` is set to 'segm', pycocotools need to be installed") + raise ModuleNotFoundError("When `iou_type` is set to 'segm', pycocotools need to be installed") self.iou_type = iou_type self.bbox_area_ranges = { "all": (0**2, int(1e5**2)), From e1b71847f32d2ea98fd003047dec228490922bfa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 May 2022 08:36:21 +0000 Subject: [PATCH 137/141] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- torchmetrics/detection/mean_ap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchmetrics/detection/mean_ap.py b/torchmetrics/detection/mean_ap.py index d87136543d2..0fdf7ab5b39 100644 --- a/torchmetrics/detection/mean_ap.py +++ b/torchmetrics/detection/mean_ap.py @@ -216,8 +216,8 @@ class MeanAveragePrecision(Metric): a standard implementation for the mAP metric for object detection. .. note:: - This metric requires you to have `torchvision` version 0.8.0 or newer installed - (with corresponding version 1.7.0 of torch or newer). This metric requires `pycocotools` + This metric requires you to have `torchvision` version 0.8.0 or newer installed + (with corresponding version 1.7.0 of torch or newer). This metric requires `pycocotools` installed when iou_type is `segm`. Please install with ``pip install torchvision`` or ``pip install torchmetrics[detection]``. From e013ff136e80f3a778417966971eab392cb762a7 Mon Sep 17 00:00:00 2001 From: Jirka Date: Thu, 19 May 2022 18:40:08 +0200 Subject: [PATCH 138/141] sample --- .github/workflows/ci_test-conda.yml | 2 +- .github/workflows/ci_test-full.yml | 2 +- tests/audio/__init__.py | 6 ++-- tests/detection/__init__.py | 5 ++++ .../instance_segmentation_inputs.json | 30 ------------------- tests/detection/test_map.py | 3 +- 6 files changed, 12 insertions(+), 36 deletions(-) delete mode 100644 tests/detection/instance_segmentation_inputs.json diff --git a/.github/workflows/ci_test-conda.yml b/.github/workflows/ci_test-conda.yml index 7673d5de6bb..f1ad8e2daf2 100644 --- a/.github/workflows/ci_test-conda.yml +++ b/.github/workflows/ci_test-conda.yml @@ -89,7 +89,7 @@ jobs: # wget is simpler but does not work on Windows python -c "from urllib.request import urlretrieve ; urlretrieve('https://pl-public-data.s3.amazonaws.com/metrics/data.zip', 'data.zip')" unzip -o data.zip - ls -l data/* + ls -l _data/* - name: Testing env: diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 6106cc0449d..cae8e310a7a 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -94,7 +94,7 @@ jobs: # wget is simpler but does not work on Windows python -c "from urllib.request import urlretrieve ; urlretrieve('https://pl-public-data.s3.amazonaws.com/metrics/data.zip', 'data.zip')" unzip -o data.zip - ls -l data/* + ls -l _data/* - name: Tests env: diff --git a/tests/audio/__init__.py b/tests/audio/__init__.py index 0893c70b124..ec0b5b126de 100644 --- a/tests/audio/__init__.py +++ b/tests/audio/__init__.py @@ -2,6 +2,6 @@ from tests import _PATH_TESTS -_SAMPLE_AUDIO_SPEECH = os.path.join(_PATH_TESTS, "data", "audio", "audio_speech.wav") -_SAMPLE_AUDIO_SPEECH_BAB_DB = os.path.join(_PATH_TESTS, "data", "audio", "audio_speech_bab_0dB.wav") -_SAMPLE_NUMPY_ISSUE_895 = os.path.join(_PATH_TESTS, "data", "audio", "issue_895.npz") +_SAMPLE_AUDIO_SPEECH = os.path.join(_PATH_TESTS, "_data", "audio", "audio_speech.wav") +_SAMPLE_AUDIO_SPEECH_BAB_DB = os.path.join(_PATH_TESTS, "_data", "audio", "audio_speech_bab_0dB.wav") +_SAMPLE_NUMPY_ISSUE_895 = os.path.join(_PATH_TESTS, "_data", "audio", "issue_895.npz") diff --git a/tests/detection/__init__.py b/tests/detection/__init__.py index e69de29bb2d..eb3272c129f 100644 --- a/tests/detection/__init__.py +++ b/tests/detection/__init__.py @@ -0,0 +1,5 @@ +import os + +from tests import _PATH_TESTS + +_SAMPLE_DETECTION_SEGMENTATION = os.path.join(_PATH_TESTS, "_data", "detection", "instance_segmentation_inputs.json") diff --git a/tests/detection/instance_segmentation_inputs.json b/tests/detection/instance_segmentation_inputs.json deleted file mode 100644 index 85bc854c6fc..00000000000 --- a/tests/detection/instance_segmentation_inputs.json +++ /dev/null @@ -1,30 +0,0 @@ -{"preds":[{ - "size": [ - 478, - 640 - ], - "counts": "VQi31m>0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR0O2N100O100O2N100O10001N101O1N2O1O2M2O1O1N3N1O1N2O2N1N2O1O1N3N1O1N2O2N1N2O1O2M2O1O1M3M4K4M3M3M4L3M3M3M4L3L4M3M3M4L3M3M3M4L3O1N2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2N101N1O2O0O2N101N1O2O0O2N101N1O2O0O1O2O0O2N101N1O2O0O2N101N101O001O1O001O1N2O001O1O1O001O1O1O001O1O001O1O1N101O1O1O001O1O1O001O1O1O001O1N2O001O1O001O1O1O001O1O1O001O1O1N010000O10000O10000O10000O100O010O100O100O100O10000O100O100O10O0100O100O100O100O1O100O100O1O010O100O1O2O0O2N101N101N1O2O1N1O2O0O2O0O2N2O0O2N101N101N2N101N101N1O2O1N1O2O0O20O2O0O2O001N101N100O2O001N101N101N101O0O101N101N101N101O0O101N101N1010O010O010O00010O0O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2N1O2M2M4L3M4L3M4RNREGP;5UEGo:3XEHk:4ZEHj:2\\EJg:1_EKe:0`ELc:OcEMa:NdEN_:MgE0\\:JjE2Y:JlE2X:HnE4aP3]Nc1WOi000001O0000001O00001O00001O0000001O00001O00001O0O10001O000O2O00001N1000001N10001O0O10001O0O101O00001N1000001N10001O0O101O000O1000000O100000E;G81000O10O100000O01000000O10O1000O100000O10O100000O01000000O10O1000O1^MnCVMR Date: Thu, 19 May 2022 18:42:09 +0200 Subject: [PATCH 139/141] Apply suggestions from code review Co-authored-by: Nicki Skafte Detlefsen --- requirements/detection_test.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/detection_test.txt b/requirements/detection_test.txt index 99f22cd1258..e69de29bb2d 100644 --- a/requirements/detection_test.txt +++ b/requirements/detection_test.txt @@ -1 +0,0 @@ -pycocotools From fc8b22071241045a5a92c5518fa4754ce6535912 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Tue, 24 May 2022 10:31:11 +0200 Subject: [PATCH 140/141] Update detection_test.txt --- requirements/detection_test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/detection_test.txt b/requirements/detection_test.txt index e69de29bb2d..88d1f3a15b5 100644 --- a/requirements/detection_test.txt +++ b/requirements/detection_test.txt @@ -0,0 +1 @@ +pycocotools From 2a991a7f0596de2268567556b13246c213a594b7 Mon Sep 17 00:00:00 2001 From: Jirka Date: Tue, 24 May 2022 12:08:00 +0200 Subject: [PATCH 141/141] fix path --- tests/detection/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/detection/__init__.py b/tests/detection/__init__.py index eb3272c129f..0904ebc12bc 100644 --- a/tests/detection/__init__.py +++ b/tests/detection/__init__.py @@ -1,5 +1,5 @@ import os -from tests import _PATH_TESTS +from tests import _PATH_ROOT -_SAMPLE_DETECTION_SEGMENTATION = os.path.join(_PATH_TESTS, "_data", "detection", "instance_segmentation_inputs.json") +_SAMPLE_DETECTION_SEGMENTATION = os.path.join(_PATH_ROOT, "_data", "detection", "instance_segmentation_inputs.json")