Also exit early as soon as we can detect too many possible prefixes

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs

@@ -59,9 +59,11 @@ static bool FindPrefixesCore(RegexNode node, List<StringBuilder> results, bool i
                 // If we're too deep to analyze further, we can't trust what we've already computed, so stop iterating.
                 // Also bail if any of our results is already hitting the threshold, or if this node is RTL, which is
                 // not worth the complexity of handling.
+                // Or if we've already discovered more than the allowed number of prefixes.
                 if (!StackHelper.TryEnsureSufficientExecutionStack() ||
                     !results.TrueForAll(sb => sb.Length < MaxPrefixLength) ||
-                    (node.Options & RegexOptions.RightToLeft) != 0)
+                    (node.Options & RegexOptions.RightToLeft) != 0 ||
+                    results.Count > MaxPrefixes)
                 {
                     return false;
                 }
@@ -165,6 +167,10 @@ static bool FindPrefixesCore(RegexNode node, List<StringBuilder> results, bool i
                                     for (int rep = 0; rep < reps; rep++)
                                     {
                                         int existingCount = results.Count;
+                                        if (existingCount * charCount > MaxPrefixes)
+                                        {
+                                            return false;
+                                        }
 
                                         // Duplicate all of the existing strings for all of the new suffixes, other than the first.
                                         foreach (char suffix in setChars.Slice(1, charCount - 1))
@@ -251,6 +257,12 @@ static bool FindPrefixesCore(RegexNode node, List<StringBuilder> results, bool i
                                 {
                                     _ = FindPrefixesCore(node.Child(i), alternateBranchResults, ignoreCase);
 
+                                    // If we now have too many results, bail.
+                                    if ((allBranchResults?.Count ?? 0) + alternateBranchResults.Count > MaxPrefixes)
+                                    {
+                                        return false;
+                                    }
+
                                     Debug.Assert(alternateBranchResults.Count > 0);
                                     foreach (StringBuilder sb in alternateBranchResults)
                                     {

src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs

@@ -1172,6 +1172,7 @@ public async Task Match_VaryingLengthStrings_Huge(RegexEngine engine)
 
         public static IEnumerable<object[]> Match_DeepNesting_MemberData()
         {
+            foreach (RegexOptions options in new[] { RegexOptions.None, RegexOptions.IgnoreCase })
             foreach (RegexEngine engine in RegexHelpers.AvailableEngines)
             {
                 if (RegexHelpers.IsNonBacktracking(engine))
@@ -1180,23 +1181,23 @@ public static IEnumerable<object[]> Match_DeepNesting_MemberData()
                     continue;
                 }
 
-                yield return new object[] { engine, 1 };
-                yield return new object[] { engine, 10 };
-                yield return new object[] { engine, 100 };
+                yield return new object[] { engine, options, 1 };
+                yield return new object[] { engine, options, 10 };
+                yield return new object[] { engine, options, 100 };
             }
         }
 
         [Theory]
         [MemberData(nameof(Match_DeepNesting_MemberData))]
-        public async Task Match_DeepNesting(RegexEngine engine, int count)
+        public async Task Match_DeepNesting(RegexEngine engine, RegexOptions options, int count)
         {
             const string Start = @"((?>abc|(?:def[ghi]", End = @")))";
             const string Match = "defg";
 
             string pattern = string.Concat(Enumerable.Repeat(Start, count)) + string.Concat(Enumerable.Repeat(End, count));
             string input = string.Concat(Enumerable.Repeat(Match, count));
 
-            Regex r = await RegexHelpers.GetRegexAsync(engine, pattern);
+            Regex r = await RegexHelpers.GetRegexAsync(engine, pattern, options);
             Match m = r.Match(input);
 
             Assert.True(m.Success);