diff --git a/prometheus/vec.go b/prometheus/vec.go
index d53848dc4..abe91638e 100644
--- a/prometheus/vec.go
+++ b/prometheus/vec.go
@@ -276,7 +276,9 @@ func (m *metricMap) deleteByHashWithLabelValues(
 	}
 
 	if len(metrics) > 1 {
+		old := metrics
 		m.metrics[h] = append(metrics[:i], metrics[i+1:]...)
+		old[len(old)-1] = metricWithLabelValues{}
 	} else {
 		delete(m.metrics, h)
 	}
@@ -302,7 +304,9 @@ func (m *metricMap) deleteByHashWithLabels(
 	}
 
 	if len(metrics) > 1 {
+		old := metrics
 		m.metrics[h] = append(metrics[:i], metrics[i+1:]...)
+		old[len(old)-1] = metricWithLabelValues{}
 	} else {
 		delete(m.metrics, h)
 	}