diff --git a/src/Framework/TestBuilder.php b/src/Framework/TestBuilder.php
index 4c1ae48a4d6..a4b9ab55b3a 100644
--- a/src/Framework/TestBuilder.php
+++ b/src/Framework/TestBuilder.php
@@ -9,6 +9,8 @@
  */
 namespace PHPUnit\Framework;
 
+use PHPUnit\Util\Filter;
+use PHPUnit\Util\InvalidDataSetException;
 use PHPUnit\Util\Test as TestUtil;
 
 /**
@@ -66,31 +68,31 @@ public function build(\ReflectionClass $theClass, string $methodName): Test
                 );
             } catch (IncompleteTestError $e) {
                 $message = \sprintf(
-                    'Test for %s::%s marked incomplete by data provider',
+                    "Test for %s::%s marked incomplete by data provider\n%s",
                     $className,
-                    $methodName
+                    $methodName,
+                    $this->throwableToString($e)
                 );
 
-                $message = $this->appendExceptionMessageIfAvailable($e, $message);
-                $data    = new IncompleteTestCase($className, $methodName, $message);
+                $data = new IncompleteTestCase($className, $methodName, $message);
             } catch (SkippedTestError $e) {
                 $message = \sprintf(
-                    'Test for %s::%s skipped by data provider',
+                    "Test for %s::%s skipped by data provider\n%s",
                     $className,
-                    $methodName
+                    $methodName,
+                    $this->throwableToString($e)
                 );
 
-                $message = $this->appendExceptionMessageIfAvailable($e, $message);
-                $data    = new SkippedTestCase($className, $methodName, $message);
+                $data = new SkippedTestCase($className, $methodName, $message);
             } catch (\Throwable $t) {
                 $message = \sprintf(
-                    'The data provider specified for %s::%s is invalid.',
+                    "The data provider specified for %s::%s is invalid.\n%s",
                     $className,
-                    $methodName
+                    $methodName,
+                    $this->throwableToString($t)
                 );
 
-                $message = $this->appendExceptionMessageIfAvailable($t, $message);
-                $data    = new WarningTestCase($message);
+                $data = new WarningTestCase($message);
             }
 
             // Test method with @dataProvider.
@@ -123,17 +125,6 @@ public function build(\ReflectionClass $theClass, string $methodName): Test
         return $test;
     }
 
-    private function appendExceptionMessageIfAvailable(\Throwable $e, string $message): string
-    {
-        $_message = $e->getMessage();
-
-        if (!empty($_message)) {
-            $message .= "\n" . $_message;
-        }
-
-        return $message;
-    }
-
     /** @psalm-param class-string $className */
     private function buildTestWithoutData(string $className)
     {
@@ -214,4 +205,28 @@ private function configureTestCase(
             );
         }
     }
+
+    private function throwableToString(\Throwable $t): string
+    {
+        $message = $t->getMessage();
+
+        if (empty(\trim($message))) {
+            $message = '<no message>';
+        }
+
+        if ($t instanceof InvalidDataSetException) {
+            return \sprintf(
+                "%s\n%s",
+                $message,
+                Filter::getFilteredStacktrace($t)
+            );
+        }
+
+        return \sprintf(
+            "%s: %s\n%s",
+            \get_class($t),
+            $message,
+            Filter::getFilteredStacktrace($t)
+        );
+    }
 }
diff --git a/src/Util/Annotation/DocBlock.php b/src/Util/Annotation/DocBlock.php
index 15ca1adfd6a..e1cc4847bb0 100644
--- a/src/Util/Annotation/DocBlock.php
+++ b/src/Util/Annotation/DocBlock.php
@@ -14,6 +14,7 @@
 use PHPUnit\Framework\SkippedTestError;
 use PHPUnit\Framework\Warning;
 use PHPUnit\Util\Exception;
+use PHPUnit\Util\InvalidDataSetException;
 
 /**
  * This is an abstraction around a PHPUnit-specific docBlock,
@@ -312,7 +313,7 @@ public function getProvidedData(): ?array
 
         foreach ($data as $key => $value) {
             if (!\is_array($value)) {
-                throw new Exception(
+                throw new InvalidDataSetException(
                     \sprintf(
                         'Data set %s is invalid.',
                         \is_int($key) ? '#' . $key : '"' . $key . '"'
diff --git a/src/Util/InvalidDataSetException.php b/src/Util/InvalidDataSetException.php
new file mode 100644
index 00000000000..228f066cc25
--- /dev/null
+++ b/src/Util/InvalidDataSetException.php
@@ -0,0 +1,17 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\Util;
+
+/**
+ * @internal This class is not covered by the backward compatibility promise for PHPUnit
+ */
+final class InvalidDataSetException extends \RuntimeException implements \PHPUnit\Exception
+{
+}
diff --git a/tests/end-to-end/regression/GitHub/498.phpt b/tests/end-to-end/regression/GitHub/498.phpt
index 4a96b426fd2..0d8dab73393 100644
--- a/tests/end-to-end/regression/GitHub/498.phpt
+++ b/tests/end-to-end/regression/GitHub/498.phpt
@@ -21,7 +21,8 @@ There was 1 warning:
 
 1) Warning
 The data provider specified for Issue498Test::shouldBeFalse is invalid.
-Can't create the data
+Exception: Can't create the data
+%s/Issue498Test.php:%d
 
 WARNINGS!
 Tests: 1, Assertions: 0, Warnings: 1.
diff --git a/tests/end-to-end/regression/GitHub/765.phpt b/tests/end-to-end/regression/GitHub/765.phpt
index ed8aef097ff..e53662e62b7 100644
--- a/tests/end-to-end/regression/GitHub/765.phpt
+++ b/tests/end-to-end/regression/GitHub/765.phpt
@@ -19,6 +19,8 @@ There was 1 warning:
 
 1) Warning
 The data provider specified for Issue765Test::testDependent is invalid.
+Exception: <no message>
+%s/Issue765Test.php:%d
 
 WARNINGS!
 Tests: 2, Assertions: 1, Warnings: 1.