enh: introduce new feature bg_off

CHANGELOG

@@ -1,8 +1,10 @@
 0.58.7
- - docs: document S3 environment variables
+ - enh: introduce new feature "bg_off", a float-valued feature for
+   event-wise background offset correction
  - enh: warn user about missing endpoint URLs in S3 format and raise an
    exception when push comes to shove
  - enh: add "pipeline:dcnum mapping" metadata
+ - docs: document S3 environment variables
 0.58.6
  - enh: for access to private S3 data, introduce the environment
    variables `DCLAB_S3_ENDPOINT_URL`, `DCLAB_S3_ACCESS_KEY_ID`,

dclab/definitions/feat_const.py

@@ -13,6 +13,12 @@
     ["aspect", "Aspect ratio of bounding box"],
     # The background brightness of the frame (not of the mask)
     ["bg_med", "Median frame background brightness [a.u.]"],
+    # Background image offset which should be added to "image_bg" before
+    # performing background correction. This is used in cases where the
+    # background stored in "image_bg" is not accurate enough, e.g.
+    # because the image_bg is a median image for multiple events and
+    # the imaging system exhibits flickering.
+    ["bg_off", "Background offset [a.u.]"],
     # Brightness values are computed only for pixels inside the mask
     ["bright_avg", "Brightness average [a.u.]"],
     ["bright_sd", "Brightness SD [a.u.]"],

dclab/features/bright_bc.py

@@ -5,7 +5,7 @@
 import numpy as np
 
 
-def get_bright_bc(mask, image, image_bg, ret_data="avg,sd"):
+def get_bright_bc(mask, image, image_bg, bg_off=None, ret_data="avg,sd"):
     """Compute avg and/or std of the background-corrected event brightness
 
     The background-corrected event brightness is defined by the
@@ -21,6 +21,9 @@ def get_bright_bc(mask, image, image_bg, ret_data="avg,sd"):
         of an event.
     image_bg: ndarray or list of ndarrays of shape (M,N)
         A 2D array that holds the background image for the same event.
+    bg_off: float or 1D ndarray
+        Additional offset value that is added to `image_bg` before
+        background correction
     ret_data: str
         A comma-separated list of metrices to compute
         - "avg": compute the average
@@ -46,6 +49,8 @@ def get_bright_bc(mask, image, image_bg, ret_data="avg,sd"):
         image_bg = [image_bg]
         image = [image]
         mask = [mask]
+        if bg_off is not None:
+            bg_off = np.atleast_1d(bg_off)
         ret_list = False
     else:
         ret_list = True
@@ -71,6 +76,8 @@ def get_bright_bc(mask, image, image_bg, ret_data="avg,sd"):
     results = []
     # Keep alphabetical order
     if ret_avg:
+        if bg_off is not None:
+            avg -= bg_off
         results.append(avg)
     if ret_std:
         results.append(std)

dclab/features/bright_perc.py

@@ -5,7 +5,7 @@
 import numpy as np
 
 
-def get_bright_perc(mask, image, image_bg):
+def get_bright_perc(mask, image, image_bg, bg_off=None):
     """Compute 10th and 90th percentile of the bg-corrected event brightness
 
     The background-corrected event brightness is defined by the
@@ -21,6 +21,9 @@ def get_bright_perc(mask, image, image_bg):
         of an event.
     image_bg: ndarray or list of ndarrays of shape (M,N)
         A 2D array that holds the background image for the same event.
+    bg_off: float or 1D ndarray
+        Additional offset value that is added to `image_bg` before
+        background correction
 
     Returns
     -------
@@ -50,6 +53,10 @@ def get_bright_perc(mask, image, image_bg):
         # Assign results
         p10[ii], p90[ii] = np.percentile(imgi[mski], q=[10, 90])
 
+    if bg_off:
+        p10 -= bg_off
+        p90 -= bg_off
+
     if ret_list:
         return p10, p90
     else:

dclab/rtdc_dataset/feat_anc_core/af_image_contour.py

@@ -8,27 +8,32 @@ def compute_contour(mm):
 
 
 def compute_bright(mm):
-    bavg, bsd = features.bright.get_bright(mask=mm["mask"],
-                                           image=mm["image"],
-                                           ret_data="avg,sd",
-                                           )
+    bavg, bsd = features.bright.get_bright(
+        mask=mm["mask"],
+        image=mm["image"],
+        ret_data="avg,sd",
+        )
     return {"bright_avg": bavg, "bright_sd": bsd}
 
 
 def compute_bright_bc(mm):
-    bavg, bsd = features.bright_bc.get_bright_bc(mask=mm["mask"],
-                                                 image=mm["image"],
-                                                 image_bg=mm["image_bg"],
-                                                 ret_data="avg,sd",
-                                                 )
+    bavg, bsd = features.bright_bc.get_bright_bc(
+        mask=mm["mask"],
+        image=mm["image"],
+        image_bg=mm["image_bg"],
+        bg_off=mm["bg_off"] if "bg_off" in mm else None,
+        ret_data="avg,sd",
+        )
     return {"bright_bc_avg": bavg, "bright_bc_sd": bsd}
 
 
 def compute_bright_perc(mm):
-    p10, p90 = features.bright_perc.get_bright_perc(mask=mm["mask"],
-                                                    image=mm["image"],
-                                                    image_bg=mm["image_bg"],
-                                                    )
+    p10, p90 = features.bright_perc.get_bright_perc(
+        mask=mm["mask"],
+        image=mm["image"],
+        image_bg=mm["image_bg"],
+        bg_off=mm["bg_off"] if "bg_off" in mm else None,
+        )
     return {"bright_perc_10": p10, "bright_perc_90": p90}
 
 

tests/test_feat_bright_bc.py

@@ -1,3 +1,4 @@
+import h5py
 import numpy as np
 import pytest
 
@@ -21,3 +22,25 @@ def test_af_brightness_bc():
         assert np.max(np.abs(image_corr)) < 50, "sanity check"
         assert ds["bright_bc_avg"][ii] == np.mean(image_corr[mask])
         assert ds["bright_bc_sd"][ii] == np.std(image_corr[mask])
+
+
+@pytest.mark.filterwarnings(
+    "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
+def test_af_brightness_bc_with_offset():
+    path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
+    with h5py.File(path, "a") as h5:
+        bg_off = np.linspace(0, 1, len(h5["events/deform"]))
+        h5["events/bg_off"] = bg_off
+
+    ds = dclab.new_dataset(path)
+    # sanity checks
+    assert "bright_bc_avg" not in ds.features_innate
+    assert "bright_bc_sd" not in ds.features_innate
+    # ignore first event (no image data)
+    for ii in range(1, len(ds)):
+        image_bg = ds["image_bg"][ii] + bg_off[ii]
+        image_corr = np.array(ds["image"][ii], dtype=int) - image_bg
+        mask = ds["mask"][ii]
+        assert np.max(np.abs(image_corr)) < 50, "sanity check"
+        assert ds["bright_bc_avg"][ii] == np.mean(image_corr[mask])
+        assert ds["bright_bc_sd"][ii] == np.std(image_corr[mask])

tests/test_feat_bright_perc.py

@@ -1,3 +1,4 @@
+import h5py
 import numpy as np
 import pytest
 
@@ -25,6 +26,30 @@ def test_af_brightness_bc():
         assert ds["bright_perc_90"][ii] == p90
 
 
+@pytest.mark.filterwarnings(
+    "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
+def test_af_brightness_bc_with_offset():
+    path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
+    with h5py.File(path, "a") as h5:
+        bg_off = np.linspace(0, 1, len(h5["events/deform"]))
+        h5["events/bg_off"] = bg_off
+
+    ds = dclab.new_dataset(path)
+    # sanity checks
+    assert "bright_perc_10" not in ds.features_innate
+    assert "bright_perc_90" not in ds.features_innate
+    # ignore first event (no image data)
+    for ii in range(1, len(ds)):
+        image_bg = ds["image_bg"][ii] + bg_off[ii]
+        image_corr = np.array(ds["image"][ii], dtype=int) - image_bg
+        mask = ds["mask"][ii]
+        p10 = np.percentile(image_corr[mask], 10)
+        p90 = np.percentile(image_corr[mask], 90)
+        assert np.max(np.abs(image_corr)) < 50, "sanity check"
+        assert ds["bright_perc_10"][ii] == p10
+        assert ds["bright_perc_90"][ii] == p90
+
+
 @pytest.mark.filterwarnings(
     "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
 def test_af_brightness_bc_single():
@@ -45,3 +70,31 @@ def test_af_brightness_bc_single():
             ds["image"][ii],
             ds["image_bg"][ii],
         ) == (p10, p90)
+
+
+@pytest.mark.filterwarnings(
+    "ignore::dclab.rtdc_dataset.config.WrongConfigurationTypeWarning")
+def test_af_brightness_bc_single_with_offset():
+    path = retrieve_data("fmt-hdf5_image-bg_2020.zip")
+    with h5py.File(path, "a") as h5:
+        bg_off = np.linspace(0, 1, len(h5["events/deform"]))
+        h5["events/bg_off"] = bg_off
+
+    ds = dclab.new_dataset(path)
+    # sanity checks
+    assert "bright_perc_10" not in ds.features_innate
+    assert "bright_perc_90" not in ds.features_innate
+    # ignore first event (no image data)
+    for ii in range(1, len(ds)):
+        image_bg = ds["image_bg"][ii] + bg_off[ii]
+        image_corr = np.array(ds["image"][ii], dtype=int) - image_bg
+        mask = ds["mask"][ii]
+        p10 = np.percentile(image_corr[mask], 10)
+        p90 = np.percentile(image_corr[mask], 90)
+        assert np.max(np.abs(image_corr)) < 50, "sanity check"
+        assert dclab.features.bright_perc.get_bright_perc(
+            ds["mask"][ii],
+            ds["image"][ii],
+            ds["image_bg"][ii],
+            ds["bg_off"][ii],
+        ) == (p10, p90)