diff --git a/packages/gatsby-plugin-image/src/components/gatsby-image.browser.tsx b/packages/gatsby-plugin-image/src/components/gatsby-image.browser.tsx
index 82ee4a7b1b2ce..6aa6ff9cbefc1 100644
--- a/packages/gatsby-plugin-image/src/components/gatsby-image.browser.tsx
+++ b/packages/gatsby-plugin-image/src/components/gatsby-image.browser.tsx
@@ -122,12 +122,13 @@ const GatsbyImageHydrator: FC<GatsbyImageProps> = function GatsbyImageHydrator({
           ssrImage.removeAttribute(`data-gatsby-image-ssr`)
         }, 0)
       } else {
-        document.addEventListener(`load`, function onLoadListener() {
-          document.removeEventListener(`load`, onLoadListener)
+        onStartLoad?.({
+          wasCached: true,
+        })
+
+        ssrImage.addEventListener(`load`, function onLoadListener() {
+          ssrImage.removeEventListener(`load`, onLoadListener)
 
-          onStartLoad?.({
-            wasCached: true,
-          })
           onLoad?.({
             wasCached: true,
           })