diff --git a/packages/block-editor/src/components/button-block-appender/style.scss b/packages/block-editor/src/components/button-block-appender/style.scss
index 6de8718d85533..a475cb13fb360 100644
--- a/packages/block-editor/src/components/button-block-appender/style.scss
+++ b/packages/block-editor/src/components/button-block-appender/style.scss
@@ -31,3 +31,28 @@
 		color: $black;
 	}
 }
+
+// When the appender shows up in empty container blocks, such as Group and Columns, add an extra click state.
+.block-list-appender:only-child {
+	.is-layout-constrained.block-editor-block-list__block:not(.is-selected) > &,
+	.is-layout-flow.block-editor-block-list__block:not(.is-selected) > & {
+		pointer-events: none;
+
+		&::after {
+			content: "";
+			position: absolute;
+			top: 0;
+			right: 0;
+			bottom: 0;
+			left: 0;
+			border: $border-width dashed currentColor;
+			opacity: 0.4;
+			border-radius: $radius-block-ui;
+			pointer-events: none;
+		}
+
+		.block-editor-inserter {
+			visibility: hidden;
+		}
+	}
+}
diff --git a/packages/block-library/src/group/editor.scss b/packages/block-library/src/group/editor.scss
index cbc0af62acdd6..c4939d9afe3a9 100644
--- a/packages/block-library/src/group/editor.scss
+++ b/packages/block-library/src/group/editor.scss
@@ -53,28 +53,3 @@
 		pointer-events: all;
 	}
 }
-
-// Show an unselected empty group button as a dashed outline instead of the appender button.
-// This effectively adds a selectable-to-delete state.
-.is-layout-flow.block-editor-block-list__block:not(.is-selected) {
-	> .block-list-appender:only-child {
-		pointer-events: none;
-
-		&::after {
-			content: "";
-			position: absolute;
-			top: 0;
-			right: 0;
-			bottom: 0;
-			left: 0;
-			border: $border-width dashed currentColor;
-			opacity: 0.4;
-			border-radius: $radius-block-ui;
-			pointer-events: none;
-		}
-
-		.block-editor-inserter {
-			visibility: hidden;
-		}
-	}
-}
diff --git a/packages/e2e-tests/specs/editor/various/inserting-blocks.test.js b/packages/e2e-tests/specs/editor/various/inserting-blocks.test.js
index f03bd39064196..9056b5b0a62b9 100644
--- a/packages/e2e-tests/specs/editor/various/inserting-blocks.test.js
+++ b/packages/e2e-tests/specs/editor/various/inserting-blocks.test.js
@@ -279,6 +279,8 @@ describe( 'Inserting blocks', () => {
 		await insertBlock( 'Group' );
 		await insertBlock( 'Paragraph' );
 		await page.keyboard.type( 'Paragraph after group' );
+		// Click the Group first to make the appender inside it clickable.
+		await page.click( '[data-type="core/group"]' );
 		await page.click( '[data-type="core/group"] [aria-label="Add block"]' );
 		const browseAll = await page.waitForXPath(
 			'//button[text()="Browse all"]'
@@ -295,6 +297,8 @@ describe( 'Inserting blocks', () => {
 		await insertBlock( 'Group' );
 		await insertBlock( 'Paragraph' );
 		await page.keyboard.type( 'Text' );
+		// Click the Group first to make the appender inside it clickable.
+		await page.click( '[data-type="core/group"]' );
 		await page.click( '[data-type="core/group"] [aria-label="Add block"]' );
 		await page.waitForSelector( INSERTER_SEARCH_SELECTOR );
 		await page.focus( INSERTER_SEARCH_SELECTOR );