diff --git a/src/Build/Utilities/FileSpecMatchTester.cs b/src/Build/Utilities/FileSpecMatchTester.cs
index e48fca39e77..2296ad1e499 100644
--- a/src/Build/Utilities/FileSpecMatchTester.cs
+++ b/src/Build/Utilities/FileSpecMatchTester.cs
@@ -12,36 +12,57 @@ namespace Microsoft.Build.Internal
     {
         private readonly string _currentDirectory;
         private readonly string _unescapedFileSpec;
+        private readonly string _filenamePattern;
         private readonly Regex _regex;
         
-        private FileSpecMatcherTester(string currentDirectory, string unescapedFileSpec, Regex regex)
+        private FileSpecMatcherTester(string currentDirectory, string unescapedFileSpec, string filenamePattern, Regex regex)
         {
             Debug.Assert(!string.IsNullOrEmpty(unescapedFileSpec));
 
             _currentDirectory = currentDirectory;
             _unescapedFileSpec = unescapedFileSpec;
+            _filenamePattern = filenamePattern;
             _regex = regex;
         }
 
         public static FileSpecMatcherTester Parse(string currentDirectory, string fileSpec)
         {
             string unescapedFileSpec = EscapingUtilities.UnescapeAll(fileSpec);
-            Regex regex = EngineFileUtilities.FilespecHasWildcards(fileSpec) ? CreateRegex(unescapedFileSpec, currentDirectory) : null;
+            string filenamePattern = null;
+            Regex regex = null;
 
-            return new FileSpecMatcherTester(currentDirectory, unescapedFileSpec, regex);
+            if (EngineFileUtilities.FilespecHasWildcards(fileSpec))
+            {
+                CreateRegexOrFilenamePattern(unescapedFileSpec, currentDirectory, out filenamePattern, out regex);
+            }
+
+            return new FileSpecMatcherTester(currentDirectory, unescapedFileSpec, filenamePattern, regex);
         }
 
         public bool IsMatch(string fileToMatch)
         {
             Debug.Assert(!string.IsNullOrEmpty(fileToMatch));
 
-            // check if there is a regex matching the file
+            // We do the matching using one of three code paths, depending on the value of _filenamePattern and _regex.
             if (_regex != null)
             {
                 var normalizedFileToMatch = FileUtilities.GetFullPathNoThrow(Path.Combine(_currentDirectory, fileToMatch));
                 return _regex.IsMatch(normalizedFileToMatch);
             }
 
+            if (_filenamePattern != null)
+            {
+                // Check file name first as it's more likely to not match.
+                string filename = Path.GetFileName(fileToMatch);
+                if (!FileMatcher.IsMatch(filename, _filenamePattern))
+                {
+                    return false;
+                }
+
+                var normalizedFileToMatch = FileUtilities.GetFullPathNoThrow(Path.Combine(_currentDirectory, fileToMatch));
+                return normalizedFileToMatch.StartsWith(_currentDirectory, System.StringComparison.OrdinalIgnoreCase);
+            }
+
             return FileUtilities.ComparePathsNoThrow(_unescapedFileSpec, fileToMatch, _currentDirectory, alwaysIgnoreCase: true);
         }
 
@@ -49,17 +70,27 @@ public bool IsMatch(string fileToMatch)
         // without this normalization step, strings pointing outside the globbing cone would still match when they shouldn't
         // for example, we dont want "**/*.cs" to match "../Shared/Foo.cs"
         // todo: glob rooting knowledge partially duplicated with MSBuildGlob.Parse and FileMatcher.ComputeFileEnumerationCacheKey
-        private static Regex CreateRegex(string unescapedFileSpec, string currentDirectory)
+        private static void CreateRegexOrFilenamePattern(string unescapedFileSpec, string currentDirectory, out string filenamePattern, out Regex regex)
         {
             FileMatcher.Default.SplitFileSpec(
-            unescapedFileSpec,
-            out string fixedDirPart,
-            out string wildcardDirectoryPart,
-            out string filenamePart);
+                unescapedFileSpec,
+                out string fixedDirPart,
+                out string wildcardDirectoryPart,
+                out string filenamePart);
 
             if (FileUtilities.PathIsInvalid(fixedDirPart))
             {
-                return null;
+                filenamePattern = null;
+                regex = null;
+                return;
+            }
+
+            // Most file specs have "**" as their directory specification so we special case these and make matching faster.
+            if (string.IsNullOrEmpty(fixedDirPart) && FileMatcher.IsRecursiveDirectoryMatch(wildcardDirectoryPart))
+            {
+                filenamePattern = filenamePart;
+                regex = null;
+                return;
             }
 
             var absoluteFixedDirPart = Path.Combine(currentDirectory, fixedDirPart);
@@ -74,11 +105,12 @@ private static Regex CreateRegex(string unescapedFileSpec, string currentDirecto
 
             FileMatcher.Default.GetFileSpecInfoWithRegexObject(
                 recombinedFileSpec,
-                out Regex regex,
+                out Regex regexObject,
                 out bool _,
                 out bool isLegal);
 
-            return isLegal ? regex : null;
+            filenamePattern = null;
+            regex = isLegal ? regexObject : null;
         }
     }
 }
diff --git a/src/Shared/FileMatcher.cs b/src/Shared/FileMatcher.cs
index 7e4959bc618..1faf11762bc 100644
--- a/src/Shared/FileMatcher.cs
+++ b/src/Shared/FileMatcher.cs
@@ -2535,6 +2535,6 @@ private static bool DirectoryEndsWithPattern(string directoryPath, string patter
             return (index != -1 && IsMatch(directoryPath.Substring(index + 1), pattern));
         }
 
-        private static bool IsRecursiveDirectoryMatch(string path) => path.TrimTrailingSlashes() == recursiveDirectoryMatch;
+        internal static bool IsRecursiveDirectoryMatch(string path) => path.TrimTrailingSlashes() == recursiveDirectoryMatch;
     }
 }