diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ddf1e1ad2b..ebcce8a802 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3441,7 +3441,7 @@ def get_task_duration(self, ts: TaskState) -> double:
         """
         duration: double = ts._prefix._duration_average
         if duration >= 0:
-            return duration
+            return max(0.005, duration)
 
         s: set = self._unknown_durations.get(ts._prefix._name)  # type: ignore
         if s is None:
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 54ef0098c6..e056d905d6 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -23,7 +23,7 @@
 # submission which may include code serialization. Therefore, be very
 # conservative in the latency estimation to suppress too aggressive stealing
 # of small tasks
-LATENCY = 0.1
+LATENCY = 0.01
 
 logger = logging.getLogger(__name__)
 
@@ -199,8 +199,7 @@ def steal_time_ratio(self, ts):
 
         ws = ts.processing_on
         compute_time = ws.processing[ts]
-        if compute_time < 0.005:  # 5ms, just give up
-            return None, None
+        compute_time = max(compute_time, 0.010)
 
         nbytes = ts.get_nbytes_deps()
         transfer_time = nbytes / self.scheduler.bandwidth + LATENCY