diff --git a/packages/block-editor/src/hooks/color-panel.js b/packages/block-editor/src/hooks/color-panel.js
index 58d7a6916ff97..b1bf5b8206a79 100644
--- a/packages/block-editor/src/hooks/color-panel.js
+++ b/packages/block-editor/src/hooks/color-panel.js
@@ -26,9 +26,10 @@ export default function ColorPanel( {
 	const [ detectedColor, setDetectedColor ] = useState();
 	const [ detectedLinkColor, setDetectedLinkColor ] = useState();
 	const ref = useBlockRef( clientId );
+	const definedColors = settings.filter( ( setting ) => setting?.colorValue );
 
 	useEffect( () => {
-		if ( ! enableContrastChecking ) {
+		if ( ! enableContrastChecking || ! definedColors.length ) {
 			return;
 		}