From 99879408f2908a1178ffdcf9c976b44e89ccf4af Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Mon, 11 Jan 2021 15:21:26 +0100 Subject: [PATCH 01/19] Added Include/Exclude RegEx filtering capability to Unzip Task --- .../net/Microsoft.Build.Tasks.Core.cs | 2 + .../netstandard/Microsoft.Build.Tasks.Core.cs | 2 + src/Tasks.UnitTests/Unzip_Tests.cs | 61 +++++++++++++++++++ src/Tasks/Resources/Strings.resx | 3 + src/Tasks/Resources/xlf/Strings.cs.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.de.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.en.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.es.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.fr.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.it.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.ja.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.ko.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.pl.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.pt-BR.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.ru.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.tr.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.zh-Hans.xlf | 5 ++ src/Tasks/Resources/xlf/Strings.zh-Hant.xlf | 5 ++ src/Tasks/Unzip.cs | 39 ++++++++++++ 19 files changed, 177 insertions(+) diff --git a/ref/Microsoft.Build.Tasks.Core/net/Microsoft.Build.Tasks.Core.cs b/ref/Microsoft.Build.Tasks.Core/net/Microsoft.Build.Tasks.Core.cs index dc26ebeb9ba..6c6511693bf 100644 --- a/ref/Microsoft.Build.Tasks.Core/net/Microsoft.Build.Tasks.Core.cs +++ b/ref/Microsoft.Build.Tasks.Core/net/Microsoft.Build.Tasks.Core.cs @@ -1217,6 +1217,8 @@ public sealed partial class Unzip : Microsoft.Build.Tasks.TaskExtension, Microso public Unzip() { } [Microsoft.Build.Framework.RequiredAttribute] public Microsoft.Build.Framework.ITaskItem DestinationFolder { get { throw null; } set { } } + public string Exclude { get { throw null; } set { } } + public string Include { get { throw null; } set { } } public bool OverwriteReadOnlyFiles { get { throw null; } set { } } public bool SkipUnchangedFiles { get { throw null; } set { } } [Microsoft.Build.Framework.RequiredAttribute] diff --git a/ref/Microsoft.Build.Tasks.Core/netstandard/Microsoft.Build.Tasks.Core.cs b/ref/Microsoft.Build.Tasks.Core/netstandard/Microsoft.Build.Tasks.Core.cs index 0d85a2cc928..e788b3e532f 100644 --- a/ref/Microsoft.Build.Tasks.Core/netstandard/Microsoft.Build.Tasks.Core.cs +++ b/ref/Microsoft.Build.Tasks.Core/netstandard/Microsoft.Build.Tasks.Core.cs @@ -894,6 +894,8 @@ public sealed partial class Unzip : Microsoft.Build.Tasks.TaskExtension, Microso public Unzip() { } [Microsoft.Build.Framework.RequiredAttribute] public Microsoft.Build.Framework.ITaskItem DestinationFolder { get { throw null; } set { } } + public string Exclude { get { throw null; } set { } } + public string Include { get { throw null; } set { } } public bool OverwriteReadOnlyFiles { get { throw null; } set { } } public bool SkipUnchangedFiles { get { throw null; } set { } } [Microsoft.Build.Framework.RequiredAttribute] diff --git a/src/Tasks.UnitTests/Unzip_Tests.cs b/src/Tasks.UnitTests/Unzip_Tests.cs index 4ccb35c6a2d..006f517d351 100644 --- a/src/Tasks.UnitTests/Unzip_Tests.cs +++ b/src/Tasks.UnitTests/Unzip_Tests.cs @@ -214,5 +214,66 @@ public void LogsErrorIfSourceFileDoesNotExist() _mockEngine.Log.ShouldContain("MSB3932", () => _mockEngine.Log); } } + + [Fact] + public void CanUnzip_WithIncludeFilter() + { + using (TestEnvironment testEnvironment = TestEnvironment.Create()) + { + TransientTestFolder source = testEnvironment.CreateFolder(createFolder: true); + TransientTestFolder destination = testEnvironment.CreateFolder(createFolder: false); + testEnvironment.CreateFile(source, "BE78A17D30144B549D21F71D5C633F7D.txt", "file1"); + testEnvironment.CreateFile(source, "A04FF4B88DF14860B7C73A8E75A4FB76.txt", "file2"); + + TransientZipArchive zipArchive = TransientZipArchive.Create(source, testEnvironment.CreateFolder(createFolder: true)); + + Unzip unzip = new Unzip + { + BuildEngine = _mockEngine, + DestinationFolder = new TaskItem(destination.Path), + OverwriteReadOnlyFiles = true, + SkipUnchangedFiles = false, + SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, + Include = "BE78A17D30144B549D21F71D5C633F7D" + }; + + unzip.Execute().ShouldBeTrue(() => _mockEngine.Log); + + _mockEngine.Log.ShouldContain(Path.Combine(destination.Path, "BE78A17D30144B549D21F71D5C633F7D.txt"), () => _mockEngine.Log); + _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "A04FF4B88DF14860B7C73A8E75A4FB76.txt"), () => _mockEngine.Log); + } + } + + [Fact] + public void CanUnzip_WithIncludeAndExcludeFilter() + { + using (TestEnvironment testEnvironment = TestEnvironment.Create()) + { + TransientTestFolder source = testEnvironment.CreateFolder(createFolder: true); + TransientTestFolder destination = testEnvironment.CreateFolder(createFolder: false); + testEnvironment.CreateFile(source, "BE78A17D30144B549D21F71D5C633F7D.txt", "file1"); + testEnvironment.CreateFile(source, "A04FF4B88DF14860B7C73A8E75A4FB76.txt", "file2"); + testEnvironment.CreateFile(source, "191CD39C4DCF4749A29887E496D0F141.txt", "file3"); + + TransientZipArchive zipArchive = TransientZipArchive.Create(source, testEnvironment.CreateFolder(createFolder: true)); + + Unzip unzip = new Unzip + { + BuildEngine = _mockEngine, + DestinationFolder = new TaskItem(destination.Path), + OverwriteReadOnlyFiles = true, + SkipUnchangedFiles = false, + SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, + Include = "BE78A17D30144B549D21F71D5C633F7D", + Exclude = "A04FF4B88DF14860B7C73A8E75A4FB76" + }; + + unzip.Execute().ShouldBeTrue(() => _mockEngine.Log); + + _mockEngine.Log.ShouldContain(Path.Combine(destination.Path, "BE78A17D30144B549D21F71D5C633F7D.txt"), () => _mockEngine.Log); + _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "A04FF4B88DF14860B7C73A8E75A4FB76.txt"), () => _mockEngine.Log); + _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "191CD39C4DCF4749A29887E496D0F141.txt"), () => _mockEngine.Log); + } + } } } diff --git a/src/Tasks/Resources/Strings.resx b/src/Tasks/Resources/Strings.resx index b465dd10789..719c12b90a1 100644 --- a/src/Tasks/Resources/Strings.resx +++ b/src/Tasks/Resources/Strings.resx @@ -2792,6 +2792,9 @@ Did not unzip from file "{0}" to file "{1}" because the "{2}" parameter was set to "{3}" in the project and the files' sizes and timestamps match. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Unzipping file "{0}" to "{1}". diff --git a/src/Tasks/Resources/xlf/Strings.cs.xlf b/src/Tasks/Resources/xlf/Strings.cs.xlf index d63041a77a8..9b07a17c799 100644 --- a/src/Tasks/Resources/xlf/Strings.cs.xlf +++ b/src/Tasks/Resources/xlf/Strings.cs.xlf @@ -2480,6 +2480,11 @@ Rozzipování ze souboru {0} do souboru {1} neproběhlo, protože parametr {2} byl v projektu nastaven na hodnotu {3} a velikosti souborů a časová razítka se shodují. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: Rozzipování do adresáře {0} se nepodařilo, protože ho nebylo možné vytvořit. {1} diff --git a/src/Tasks/Resources/xlf/Strings.de.xlf b/src/Tasks/Resources/xlf/Strings.de.xlf index 6f55e8ffb7d..26fe31b49b6 100644 --- a/src/Tasks/Resources/xlf/Strings.de.xlf +++ b/src/Tasks/Resources/xlf/Strings.de.xlf @@ -2480,6 +2480,11 @@ Die Datei "{0}" wurde nicht in die Datei "{1}" entzippt, weil der Parameter "{2}" im Projekt auf "{3}" festgelegt war und die Größen und Zeitstempel der Dateien übereinstimmen. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: Fehler beim Entzippen in das Verzeichnis "{0}", weil dieses nicht erstellt werden konnte. {1} diff --git a/src/Tasks/Resources/xlf/Strings.en.xlf b/src/Tasks/Resources/xlf/Strings.en.xlf index 0c447b05ab3..2565fba156f 100644 --- a/src/Tasks/Resources/xlf/Strings.en.xlf +++ b/src/Tasks/Resources/xlf/Strings.en.xlf @@ -2530,6 +2530,11 @@ Did not unzip from file "{0}" to file "{1}" because the "{2}" parameter was set to "{3}" in the project and the files' sizes and timestamps match. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} diff --git a/src/Tasks/Resources/xlf/Strings.es.xlf b/src/Tasks/Resources/xlf/Strings.es.xlf index 620929ddfe8..68c70dad9ca 100644 --- a/src/Tasks/Resources/xlf/Strings.es.xlf +++ b/src/Tasks/Resources/xlf/Strings.es.xlf @@ -2480,6 +2480,11 @@ No se descomprimió del archivo "{0}" en el archivo "{1}" porque el parámetro "{2}" se estableció como "{3}" en el proyecto y los tamaños y las marcas de tiempo de los archivos coinciden. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: No se pudo descomprimir en el directorio "{0}" porque no se pudo crear. {1} diff --git a/src/Tasks/Resources/xlf/Strings.fr.xlf b/src/Tasks/Resources/xlf/Strings.fr.xlf index 11359a59620..0455d7998fc 100644 --- a/src/Tasks/Resources/xlf/Strings.fr.xlf +++ b/src/Tasks/Resources/xlf/Strings.fr.xlf @@ -2480,6 +2480,11 @@ Impossible de décompresser le fichier "{0}" vers le fichier "{1}", car le paramètre "{2}" a la valeur "{3}" dans le projet, et les tailles et horodatages des fichiers correspondent. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: Échec de la décompression dans le répertoire "{0}", car il n'a pas pu être créé. {1} diff --git a/src/Tasks/Resources/xlf/Strings.it.xlf b/src/Tasks/Resources/xlf/Strings.it.xlf index 39b30cfbacb..0bdc9d6cc90 100644 --- a/src/Tasks/Resources/xlf/Strings.it.xlf +++ b/src/Tasks/Resources/xlf/Strings.it.xlf @@ -2480,6 +2480,11 @@ Non è stato possibile decomprimere il file "{0}" nel file "{1}". Il parametro "{2}" è stato impostato su "{3}" nel progetto e le dimensioni e il timestamp dei file corrispondono. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: non è stato possibile decomprimere nella directory "{0}" perché non è stato possibile crearla. {1} diff --git a/src/Tasks/Resources/xlf/Strings.ja.xlf b/src/Tasks/Resources/xlf/Strings.ja.xlf index 8166fd4d2dd..ced7f3ccc0e 100644 --- a/src/Tasks/Resources/xlf/Strings.ja.xlf +++ b/src/Tasks/Resources/xlf/Strings.ja.xlf @@ -2480,6 +2480,11 @@ "{2}" パラメーターがプロジェクトで "{3}" に設定されているため、またファイルのサイズとタイムスタンプが一致するため、ファイル "{0}" からファイル "{1}" に解凍しませんでした。 + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: ディレクトリ "{0}" への解凍は、そのディレクトリを作成できなかったため、失敗しました。{1} diff --git a/src/Tasks/Resources/xlf/Strings.ko.xlf b/src/Tasks/Resources/xlf/Strings.ko.xlf index dd566fd77b2..96b55eea8b4 100644 --- a/src/Tasks/Resources/xlf/Strings.ko.xlf +++ b/src/Tasks/Resources/xlf/Strings.ko.xlf @@ -2480,6 +2480,11 @@ "{2}" 매개 변수가 프로젝트에 "{3}"(으)로 설정되었고 파일 크기와 타임스탬프가 일치하기 때문에 "{0}" 파일에서 "{1}" 파일로 압축을 풀 수 없습니다. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: "{0}" 디렉터리를 생성할 수 없기 때문에 이 디렉터리에 압축을 풀지 못했습니다. {1} diff --git a/src/Tasks/Resources/xlf/Strings.pl.xlf b/src/Tasks/Resources/xlf/Strings.pl.xlf index bf99b39d5ea..4469ee8699c 100644 --- a/src/Tasks/Resources/xlf/Strings.pl.xlf +++ b/src/Tasks/Resources/xlf/Strings.pl.xlf @@ -2480,6 +2480,11 @@ Nie wykonano rozpakowywania z pliku „{0}” do pliku „{1}”, ponieważ parametr „{2}” w projekcie został ustawiony na wartość „{3}”, a rozmiary plików i sygnatury czasowe pasują do siebie. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: Nie można rozpakować do katalogu „{0}”, ponieważ nie można go utworzyć. {1} diff --git a/src/Tasks/Resources/xlf/Strings.pt-BR.xlf b/src/Tasks/Resources/xlf/Strings.pt-BR.xlf index dc823788e5c..a3239710f13 100644 --- a/src/Tasks/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Tasks/Resources/xlf/Strings.pt-BR.xlf @@ -2480,6 +2480,11 @@ Não foi possível descompactar o arquivo "{0}" para o arquivo "{1}", pois o parâmetro "{2}" foi definido como "{3}" no projeto, e os tamanhos de arquivos e os carimbos de data/hora não correspondem. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: Falha ao descompactar no diretório "{0}" porque ele não pôde ser criado. {1} diff --git a/src/Tasks/Resources/xlf/Strings.ru.xlf b/src/Tasks/Resources/xlf/Strings.ru.xlf index 9e7588f33eb..52b1b28858a 100644 --- a/src/Tasks/Resources/xlf/Strings.ru.xlf +++ b/src/Tasks/Resources/xlf/Strings.ru.xlf @@ -2480,6 +2480,11 @@ Не удалось выполнить распаковку из файла "{0}" в файл "{1}", так как для параметра "{2}" в проекте было задано значение "{3}", а размеры файлов и отметки времени совпадают. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: не удалось выполнить распаковку в каталог "{0}", так как создать его не удалось. {1} diff --git a/src/Tasks/Resources/xlf/Strings.tr.xlf b/src/Tasks/Resources/xlf/Strings.tr.xlf index f4e09678061..96bf83212ac 100644 --- a/src/Tasks/Resources/xlf/Strings.tr.xlf +++ b/src/Tasks/Resources/xlf/Strings.tr.xlf @@ -2480,6 +2480,11 @@ Projede "{2}" parametresi "{3}" olarak ayarlandığından ve dosya boyutlarıyla zaman damgaları eşleştiğinden "{0}" dosyasını "{1}" dosyasına çıkarma işlemi gerçekleştirilmedi. + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: "{0}" dizini oluşturulamadığından bu dizine çıkarılamadı. {1} diff --git a/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf b/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf index 37aa968f49b..c9f73467afc 100644 --- a/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf @@ -2480,6 +2480,11 @@ 未从文件“{0}”解压缩到文件“{1}”，因为“{2}”参数在项目中设置为“{3}”，而两个文件的大小及时间戳一致。 + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: 未能解压缩到目录“{0}”，因为无法创建它。{1} diff --git a/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf b/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf index 2fa9517589f..29aeb8a2217 100644 --- a/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf @@ -2480,6 +2480,11 @@ 並未從檔案 "{0}" 解壓縮到檔案 "{1}"，因為在專案中的 "{2}" 參數原先設定為 "{3}"，且檔案的大小與時間戳記相符。 + + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + + MSB3931: Failed to unzip to directory "{0}" because it could not be created. {1} MSB3931: 因為無法建立目錄 "{0}"，所以無法解壓縮至該目錄。{1} diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index 401829e2f65..bd531956972 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -7,6 +7,7 @@ using System.IO; using System.IO.Compression; using System.Linq; +using System.Text.RegularExpressions; using System.Threading; using Microsoft.Build.Shared.FileSystem; @@ -49,6 +50,16 @@ public sealed class Unzip : TaskExtension, ICancelableTask [Required] public ITaskItem[] SourceFiles { get; set; } + /// + /// Gets or sets a regular expression that will be used to include files to be unzipped. + /// + public string Include { get; set; } + + /// + /// Gets or sets a regular expression that will be used to exclude files to be unzipped. + /// + public string Exclude { get; set; } + /// public void Cancel() { @@ -129,6 +140,12 @@ private void Extract(ZipArchive sourceArchive, DirectoryInfo destinationDirector { foreach (ZipArchiveEntry zipArchiveEntry in sourceArchive.Entries.TakeWhile(i => !_cancellationToken.IsCancellationRequested)) { + if (ShouldSkipEntry(zipArchiveEntry)) + { + Log.LogMessageFromResources(MessageImportance.Low, "Unzip.DidNotUnzipBecauseOfFilter", zipArchiveEntry.FullName); + continue; + } + FileInfo destinationPath = new FileInfo(Path.Combine(destinationDirectory.FullName, zipArchiveEntry.FullName)); // Zip archives can have directory entries listed explicitly. @@ -199,6 +216,28 @@ private void Extract(ZipArchive sourceArchive, DirectoryInfo destinationDirector } } + /// + /// Determines whether or not a file should be skipped when unzipping by filtering. + /// + /// The object containing information about the file in the zip archive. + /// true if the file should be skipped, otherwise false. + private bool ShouldSkipEntry(ZipArchiveEntry zipArchiveEntry) + { + bool result = false; + + if (!string.IsNullOrWhiteSpace(Include)) + { + result |= !Regex.IsMatch(zipArchiveEntry.FullName, Include); + } + + if (!string.IsNullOrWhiteSpace(Exclude)) + { + result |= Regex.IsMatch(zipArchiveEntry.FullName, Exclude); + } + + return result; + } + /// /// Determines whether or not a file should be skipped when unzipping. /// From 08cb0d135ddb548112f812b96a75dbf40db625eb Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Mon, 11 Jan 2021 15:31:10 +0100 Subject: [PATCH 02/19] Added Exclude Filter Unit Test for Unzip Task --- src/Tasks.UnitTests/Unzip_Tests.cs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/Tasks.UnitTests/Unzip_Tests.cs b/src/Tasks.UnitTests/Unzip_Tests.cs index 006f517d351..0bf7c8006b3 100644 --- a/src/Tasks.UnitTests/Unzip_Tests.cs +++ b/src/Tasks.UnitTests/Unzip_Tests.cs @@ -244,6 +244,35 @@ public void CanUnzip_WithIncludeFilter() } } + [Fact] + public void CanUnzip_WithExcludeFilter() + { + using (TestEnvironment testEnvironment = TestEnvironment.Create()) + { + TransientTestFolder source = testEnvironment.CreateFolder(createFolder: true); + TransientTestFolder destination = testEnvironment.CreateFolder(createFolder: false); + testEnvironment.CreateFile(source, "BE78A17D30144B549D21F71D5C633F7D.txt", "file1"); + testEnvironment.CreateFile(source, "A04FF4B88DF14860B7C73A8E75A4FB76.txt", "file2"); + + TransientZipArchive zipArchive = TransientZipArchive.Create(source, testEnvironment.CreateFolder(createFolder: true)); + + Unzip unzip = new Unzip + { + BuildEngine = _mockEngine, + DestinationFolder = new TaskItem(destination.Path), + OverwriteReadOnlyFiles = true, + SkipUnchangedFiles = false, + SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, + Exclude = "BE78A17D30144B549D21F71D5C633F7D" + }; + + unzip.Execute().ShouldBeTrue(() => _mockEngine.Log); + + _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "BE78A17D30144B549D21F71D5C633F7D.txt"), () => _mockEngine.Log); + _mockEngine.Log.ShouldContain(Path.Combine(destination.Path, "A04FF4B88DF14860B7C73A8E75A4FB76.txt"), () => _mockEngine.Log); + } + } + [Fact] public void CanUnzip_WithIncludeAndExcludeFilter() { From 86c25c2e90240e05734af2ba2c4e5e4c40fbeb4c Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Thu, 14 Jan 2021 19:09:44 +0100 Subject: [PATCH 03/19] Adjusted Unit Test according to feedback --- src/Tasks.UnitTests/Unzip_Tests.cs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Tasks.UnitTests/Unzip_Tests.cs b/src/Tasks.UnitTests/Unzip_Tests.cs index 0bf7c8006b3..ba7d7601ad5 100644 --- a/src/Tasks.UnitTests/Unzip_Tests.cs +++ b/src/Tasks.UnitTests/Unzip_Tests.cs @@ -280,9 +280,10 @@ public void CanUnzip_WithIncludeAndExcludeFilter() { TransientTestFolder source = testEnvironment.CreateFolder(createFolder: true); TransientTestFolder destination = testEnvironment.CreateFolder(createFolder: false); - testEnvironment.CreateFile(source, "BE78A17D30144B549D21F71D5C633F7D.txt", "file1"); - testEnvironment.CreateFile(source, "A04FF4B88DF14860B7C73A8E75A4FB76.txt", "file2"); - testEnvironment.CreateFile(source, "191CD39C4DCF4749A29887E496D0F141.txt", "file3"); + testEnvironment.CreateFile(source, "file1.js", "file1"); + testEnvironment.CreateFile(source, "file1.js.map", "file2"); + testEnvironment.CreateFile(source, "file2.js", "file3"); + testEnvironment.CreateFile(source, "readme.txt", "file4"); TransientZipArchive zipArchive = TransientZipArchive.Create(source, testEnvironment.CreateFolder(createFolder: true)); @@ -293,15 +294,16 @@ public void CanUnzip_WithIncludeAndExcludeFilter() OverwriteReadOnlyFiles = true, SkipUnchangedFiles = false, SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, - Include = "BE78A17D30144B549D21F71D5C633F7D", - Exclude = "A04FF4B88DF14860B7C73A8E75A4FB76" + Include = ".*?\\.js", + Exclude = ".*?\\.js\\.map" }; unzip.Execute().ShouldBeTrue(() => _mockEngine.Log); - _mockEngine.Log.ShouldContain(Path.Combine(destination.Path, "BE78A17D30144B549D21F71D5C633F7D.txt"), () => _mockEngine.Log); - _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "A04FF4B88DF14860B7C73A8E75A4FB76.txt"), () => _mockEngine.Log); - _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "191CD39C4DCF4749A29887E496D0F141.txt"), () => _mockEngine.Log); + _mockEngine.Log.ShouldContain(Path.Combine(destination.Path, "file1.js"), () => _mockEngine.Log); + _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "file1.js.map"), () => _mockEngine.Log); + _mockEngine.Log.ShouldContain(Path.Combine(destination.Path, "file2.js"), () => _mockEngine.Log); + _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "readme.txt"), () => _mockEngine.Log); } } } From fbc0a4f314787387c4d2880f2cd0e1da6ec48cec Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Thu, 14 Jan 2021 19:10:35 +0100 Subject: [PATCH 04/19] Applied resource string suggestion --- src/Tasks/Resources/Strings.resx | 2 +- src/Tasks/Resources/xlf/Strings.cs.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.de.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.en.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.es.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.fr.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.it.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.ja.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.ko.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.pl.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.pt-BR.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.ru.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.tr.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.zh-Hans.xlf | 4 ++-- src/Tasks/Resources/xlf/Strings.zh-Hant.xlf | 4 ++-- 15 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/Tasks/Resources/Strings.resx b/src/Tasks/Resources/Strings.resx index 719c12b90a1..ee918aea485 100644 --- a/src/Tasks/Resources/Strings.resx +++ b/src/Tasks/Resources/Strings.resx @@ -2793,7 +2793,7 @@ Did not unzip from file "{0}" to file "{1}" because the "{2}" parameter was set to "{3}" in the project and the files' sizes and timestamps match. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. Unzipping file "{0}" to "{1}". diff --git a/src/Tasks/Resources/xlf/Strings.cs.xlf b/src/Tasks/Resources/xlf/Strings.cs.xlf index 9b07a17c799..be673515e7a 100644 --- a/src/Tasks/Resources/xlf/Strings.cs.xlf +++ b/src/Tasks/Resources/xlf/Strings.cs.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.de.xlf b/src/Tasks/Resources/xlf/Strings.de.xlf index 26fe31b49b6..c6a5b30c1c7 100644 --- a/src/Tasks/Resources/xlf/Strings.de.xlf +++ b/src/Tasks/Resources/xlf/Strings.de.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.en.xlf b/src/Tasks/Resources/xlf/Strings.en.xlf index 2565fba156f..cd67017729e 100644 --- a/src/Tasks/Resources/xlf/Strings.en.xlf +++ b/src/Tasks/Resources/xlf/Strings.en.xlf @@ -2531,8 +2531,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.es.xlf b/src/Tasks/Resources/xlf/Strings.es.xlf index 68c70dad9ca..1205eddde87 100644 --- a/src/Tasks/Resources/xlf/Strings.es.xlf +++ b/src/Tasks/Resources/xlf/Strings.es.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.fr.xlf b/src/Tasks/Resources/xlf/Strings.fr.xlf index 0455d7998fc..171db90b472 100644 --- a/src/Tasks/Resources/xlf/Strings.fr.xlf +++ b/src/Tasks/Resources/xlf/Strings.fr.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.it.xlf b/src/Tasks/Resources/xlf/Strings.it.xlf index 0bdc9d6cc90..cf83cb7a7a9 100644 --- a/src/Tasks/Resources/xlf/Strings.it.xlf +++ b/src/Tasks/Resources/xlf/Strings.it.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.ja.xlf b/src/Tasks/Resources/xlf/Strings.ja.xlf index ced7f3ccc0e..9798123e095 100644 --- a/src/Tasks/Resources/xlf/Strings.ja.xlf +++ b/src/Tasks/Resources/xlf/Strings.ja.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.ko.xlf b/src/Tasks/Resources/xlf/Strings.ko.xlf index 96b55eea8b4..c3bf5d30a15 100644 --- a/src/Tasks/Resources/xlf/Strings.ko.xlf +++ b/src/Tasks/Resources/xlf/Strings.ko.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.pl.xlf b/src/Tasks/Resources/xlf/Strings.pl.xlf index 4469ee8699c..3e8b8e806e4 100644 --- a/src/Tasks/Resources/xlf/Strings.pl.xlf +++ b/src/Tasks/Resources/xlf/Strings.pl.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.pt-BR.xlf b/src/Tasks/Resources/xlf/Strings.pt-BR.xlf index a3239710f13..682eb515af6 100644 --- a/src/Tasks/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Tasks/Resources/xlf/Strings.pt-BR.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.ru.xlf b/src/Tasks/Resources/xlf/Strings.ru.xlf index 52b1b28858a..5b3a368839e 100644 --- a/src/Tasks/Resources/xlf/Strings.ru.xlf +++ b/src/Tasks/Resources/xlf/Strings.ru.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.tr.xlf b/src/Tasks/Resources/xlf/Strings.tr.xlf index 96bf83212ac..f68f8eeda0b 100644 --- a/src/Tasks/Resources/xlf/Strings.tr.xlf +++ b/src/Tasks/Resources/xlf/Strings.tr.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf b/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf index c9f73467afc..4123f129a7e 100644 --- a/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. diff --git a/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf b/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf index 29aeb8a2217..7e9e1daea0d 100644 --- a/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf @@ -2481,8 +2481,8 @@ - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. - Did not unzip file "{0}" because it didn't match the include or matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. + Did not unzip file "{0}" because it didn't match the include filter or because it matched the exclude filter. From e71e37f6a4ef30dd4a1647e3f41185c8605e9b87 Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Wed, 20 Jan 2021 14:24:40 +0100 Subject: [PATCH 05/19] Adjusted Include and Exclude to IncludePattern and ExcludePattern as per Review Suggestion --- src/Tasks/Unzip.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index bd531956972..e82340e8e17 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -53,12 +53,12 @@ public sealed class Unzip : TaskExtension, ICancelableTask /// /// Gets or sets a regular expression that will be used to include files to be unzipped. /// - public string Include { get; set; } + public string IncludePattern { get; set; } /// /// Gets or sets a regular expression that will be used to exclude files to be unzipped. /// - public string Exclude { get; set; } + public string ExcludePattern { get; set; } /// public void Cancel() From 152aadc5a43f5c8389adfeaf0d4be7971ace4d7b Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Tue, 26 Jan 2021 19:31:02 +0100 Subject: [PATCH 06/19] Adjusted property names to "Include" and "Exclude" and altered them to regular glob matching instead of RegEx --- src/Shared/FileMatcher.cs | 25 +++- src/Tasks.UnitTests/Unzip_Tests.cs | 11 +- src/Tasks/Resources/Strings.resx | 8 ++ src/Tasks/Resources/xlf/Strings.cs.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.de.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.en.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.es.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.fr.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.it.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.ja.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.ko.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.pl.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.pt-BR.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.ru.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.tr.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.zh-Hans.xlf | 10 ++ src/Tasks/Resources/xlf/Strings.zh-Hant.xlf | 10 ++ src/Tasks/Unzip.cs | 131 ++++++++++++++------ 18 files changed, 271 insertions(+), 44 deletions(-) diff --git a/src/Shared/FileMatcher.cs b/src/Shared/FileMatcher.cs index a9c8fa2b2f3..962d58a3024 100644 --- a/src/Shared/FileMatcher.cs +++ b/src/Shared/FileMatcher.cs @@ -31,6 +31,8 @@ internal class FileMatcher private static readonly char[] s_wildcardCharacters = { '*', '?' }; private static readonly char[] s_wildcardAndSemicolonCharacters = { '*', '?', ';' }; + private static readonly string[] s_propertyReferences = { "$(", "@(" }; + // on OSX both System.IO.Path separators are '/', so we have to use the literals internal static readonly char[] directorySeparatorCharacters = { '/', '\\' }; internal static readonly string[] directorySeparatorStrings = directorySeparatorCharacters.Select(c => c.ToString()).ToArray(); @@ -166,8 +168,6 @@ internal static void ClearFileEnumerationsCache() /// /// Determines whether the given path has any wild card characters. /// - /// - /// internal static bool HasWildcards(string filespec) { // Perf Note: Doing a [Last]IndexOfAny(...) is much faster than compiling a @@ -180,18 +180,33 @@ internal static bool HasWildcards(string filespec) } /// - /// Determines whether the given path has any wild card characters or any semicolons. + /// Determines whether the given path has any wild card characters or semicolons. + /// + internal static bool HasWildcardsOrSemicolon(string filespec) + { + return -1 != filespec.LastIndexOfAny(s_wildcardAndSemicolonCharacters); + } + + /// + /// Determines whether the given path has any wild card characters, any semicolons or any property references. /// internal static bool HasWildcardsSemicolonItemOrPropertyReferences(string filespec) { return (-1 != filespec.IndexOfAny(s_wildcardAndSemicolonCharacters)) || - filespec.Contains("$(") || - filespec.Contains("@(") + HasPropertyReferences(filespec) ; } + /// + /// Determines whether the given path has any property references. + /// + internal static bool HasPropertyReferences(string filespec) + { + return s_propertyReferences.Aggregate(false, (current, propertyReference) => current | filespec.Contains(propertyReference)); + } + /// /// Get the files and\or folders specified by the given path and pattern. /// diff --git a/src/Tasks.UnitTests/Unzip_Tests.cs b/src/Tasks.UnitTests/Unzip_Tests.cs index ba7d7601ad5..5f62209711e 100644 --- a/src/Tasks.UnitTests/Unzip_Tests.cs +++ b/src/Tasks.UnitTests/Unzip_Tests.cs @@ -234,7 +234,7 @@ public void CanUnzip_WithIncludeFilter() OverwriteReadOnlyFiles = true, SkipUnchangedFiles = false, SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, - Include = "BE78A17D30144B549D21F71D5C633F7D" + Include = "BE78A17D30144B549D21F71D5C633F7D.txt" }; unzip.Execute().ShouldBeTrue(() => _mockEngine.Log); @@ -263,7 +263,7 @@ public void CanUnzip_WithExcludeFilter() OverwriteReadOnlyFiles = true, SkipUnchangedFiles = false, SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, - Exclude = "BE78A17D30144B549D21F71D5C633F7D" + Exclude = "BE78A17D30144B549D21F71D5C633F7D.txt" }; unzip.Execute().ShouldBeTrue(() => _mockEngine.Log); @@ -280,10 +280,12 @@ public void CanUnzip_WithIncludeAndExcludeFilter() { TransientTestFolder source = testEnvironment.CreateFolder(createFolder: true); TransientTestFolder destination = testEnvironment.CreateFolder(createFolder: false); + TransientTestFolder sub = source.CreateDirectory("sub"); testEnvironment.CreateFile(source, "file1.js", "file1"); testEnvironment.CreateFile(source, "file1.js.map", "file2"); testEnvironment.CreateFile(source, "file2.js", "file3"); testEnvironment.CreateFile(source, "readme.txt", "file4"); + testEnvironment.CreateFile(sub, "subfile.js", "File5"); TransientZipArchive zipArchive = TransientZipArchive.Create(source, testEnvironment.CreateFolder(createFolder: true)); @@ -294,8 +296,8 @@ public void CanUnzip_WithIncludeAndExcludeFilter() OverwriteReadOnlyFiles = true, SkipUnchangedFiles = false, SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, - Include = ".*?\\.js", - Exclude = ".*?\\.js\\.map" + Include = "*.js", + Exclude = "*.js.map;sub\\*.js" }; unzip.Execute().ShouldBeTrue(() => _mockEngine.Log); @@ -304,6 +306,7 @@ public void CanUnzip_WithIncludeAndExcludeFilter() _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "file1.js.map"), () => _mockEngine.Log); _mockEngine.Log.ShouldContain(Path.Combine(destination.Path, "file2.js"), () => _mockEngine.Log); _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "readme.txt"), () => _mockEngine.Log); + _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "sub", "subfile.js"), () => _mockEngine.Log); } } } diff --git a/src/Tasks/Resources/Strings.resx b/src/Tasks/Resources/Strings.resx index ee918aea485..a054ea6c65a 100644 --- a/src/Tasks/Resources/Strings.resx +++ b/src/Tasks/Resources/Strings.resx @@ -2789,6 +2789,14 @@ MSB3936: Failed to open unzip file "{0}" to "{1}". {2} {StrBegin="MSB3936: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Did not unzip from file "{0}" to file "{1}" because the "{2}" parameter was set to "{3}" in the project and the files' sizes and timestamps match. diff --git a/src/Tasks/Resources/xlf/Strings.cs.xlf b/src/Tasks/Resources/xlf/Strings.cs.xlf index be673515e7a..c6ebbc2bf98 100644 --- a/src/Tasks/Resources/xlf/Strings.cs.xlf +++ b/src/Tasks/Resources/xlf/Strings.cs.xlf @@ -2515,6 +2515,16 @@ MSB3932: Soubor {0} se nepodařilo rozzipovat, protože neexistuje nebo není přístupný. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". Soubor {0} se rozzipovává do {1}. diff --git a/src/Tasks/Resources/xlf/Strings.de.xlf b/src/Tasks/Resources/xlf/Strings.de.xlf index c6a5b30c1c7..9d261b2b46f 100644 --- a/src/Tasks/Resources/xlf/Strings.de.xlf +++ b/src/Tasks/Resources/xlf/Strings.de.xlf @@ -2515,6 +2515,16 @@ MSB3932: Die Datei "{0}" konnte nicht entzippt werden, weil sie nicht vorhanden ist oder nicht darauf zugegriffen werden kann. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". Die Datei "{0}" wird in "{1}" entzippt. diff --git a/src/Tasks/Resources/xlf/Strings.en.xlf b/src/Tasks/Resources/xlf/Strings.en.xlf index cd67017729e..df627557c4a 100644 --- a/src/Tasks/Resources/xlf/Strings.en.xlf +++ b/src/Tasks/Resources/xlf/Strings.en.xlf @@ -2565,6 +2565,16 @@ MSB3932: Failed to unzip file "{0}" because the file does not exist or is inaccessible. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". Unzipping file "{0}" to "{1}". diff --git a/src/Tasks/Resources/xlf/Strings.es.xlf b/src/Tasks/Resources/xlf/Strings.es.xlf index 1205eddde87..8a28657818f 100644 --- a/src/Tasks/Resources/xlf/Strings.es.xlf +++ b/src/Tasks/Resources/xlf/Strings.es.xlf @@ -2515,6 +2515,16 @@ MSB3932: No se pudo descomprimir el archivo "{0}" porque no existe o no se puede tener acceso a él. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". Descomprimiendo el archivo "{0}" en "{1}". diff --git a/src/Tasks/Resources/xlf/Strings.fr.xlf b/src/Tasks/Resources/xlf/Strings.fr.xlf index 171db90b472..3854fcf0c19 100644 --- a/src/Tasks/Resources/xlf/Strings.fr.xlf +++ b/src/Tasks/Resources/xlf/Strings.fr.xlf @@ -2515,6 +2515,16 @@ MSB3932: Échec de la décompression du fichier "{0}", car le fichier n'existe pas ou est inaccessible. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". Décompression du fichier "{0}" dans "{1}". diff --git a/src/Tasks/Resources/xlf/Strings.it.xlf b/src/Tasks/Resources/xlf/Strings.it.xlf index cf83cb7a7a9..14bdb0e8baf 100644 --- a/src/Tasks/Resources/xlf/Strings.it.xlf +++ b/src/Tasks/Resources/xlf/Strings.it.xlf @@ -2515,6 +2515,16 @@ MSB3932: non è stato possibile decomprimere il file "{0}" perché non esiste oppure è inaccessibile. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". Decompressione del file "{0}" in "{1}". diff --git a/src/Tasks/Resources/xlf/Strings.ja.xlf b/src/Tasks/Resources/xlf/Strings.ja.xlf index 9798123e095..a4b07fadb16 100644 --- a/src/Tasks/Resources/xlf/Strings.ja.xlf +++ b/src/Tasks/Resources/xlf/Strings.ja.xlf @@ -2515,6 +2515,16 @@ MSB3932: ファイルが存在しないか、アクセスできないため、ファイル "{0}" を解凍できませんでした。 {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". ファイル "{0}" を "{1}" に解凍しています。 diff --git a/src/Tasks/Resources/xlf/Strings.ko.xlf b/src/Tasks/Resources/xlf/Strings.ko.xlf index c3bf5d30a15..e801bdd91a6 100644 --- a/src/Tasks/Resources/xlf/Strings.ko.xlf +++ b/src/Tasks/Resources/xlf/Strings.ko.xlf @@ -2515,6 +2515,16 @@ MSB3932: 파일이 존재하지 않거나 액세스할 수 없기 때문에 파일 "{0}"의 압축을 풀지 못했습니다. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". 파일 "{0}"의 압축을 "{1}"에 푸는 중입니다. diff --git a/src/Tasks/Resources/xlf/Strings.pl.xlf b/src/Tasks/Resources/xlf/Strings.pl.xlf index 3e8b8e806e4..4251be9ff36 100644 --- a/src/Tasks/Resources/xlf/Strings.pl.xlf +++ b/src/Tasks/Resources/xlf/Strings.pl.xlf @@ -2515,6 +2515,16 @@ MSB3932: Nie można rozpakować pliku „{0}”, ponieważ plik nie istnieje lub jest niedostępny. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". Rozpakowywanie pliku „{0}” do pliku „{1}”. diff --git a/src/Tasks/Resources/xlf/Strings.pt-BR.xlf b/src/Tasks/Resources/xlf/Strings.pt-BR.xlf index 682eb515af6..1dfff7329b4 100644 --- a/src/Tasks/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Tasks/Resources/xlf/Strings.pt-BR.xlf @@ -2515,6 +2515,16 @@ MSB3932: Falha ao descompactar o arquivo "{0}" porque ele não existe ou está inacessível. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". Descompactando o arquivo "{0}" em "{1}". diff --git a/src/Tasks/Resources/xlf/Strings.ru.xlf b/src/Tasks/Resources/xlf/Strings.ru.xlf index 5b3a368839e..9e883f29af7 100644 --- a/src/Tasks/Resources/xlf/Strings.ru.xlf +++ b/src/Tasks/Resources/xlf/Strings.ru.xlf @@ -2515,6 +2515,16 @@ MSB3932: не удалось распаковать файл "{0}", так как он не существует или недоступен. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". Распаковка файла "{0}" в"{1}". diff --git a/src/Tasks/Resources/xlf/Strings.tr.xlf b/src/Tasks/Resources/xlf/Strings.tr.xlf index f68f8eeda0b..194e8b3f3e1 100644 --- a/src/Tasks/Resources/xlf/Strings.tr.xlf +++ b/src/Tasks/Resources/xlf/Strings.tr.xlf @@ -2515,6 +2515,16 @@ MSB3932: Dosya mevcut olmadığından veya erişilebilir olmadığından "{0}" dosyasının sıkıştırması açılamadı. {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". "{0}" dosyasının sıkıştırması "{1}" hedefine açılıyor. diff --git a/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf b/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf index 4123f129a7e..44e2fce72d4 100644 --- a/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Tasks/Resources/xlf/Strings.zh-Hans.xlf @@ -2515,6 +2515,16 @@ MSB3932: 未能解压缩文件“{0}”，因为该文件不存在或无法访问。 {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". 将文件“{0}”解压缩到“{1}”。 diff --git a/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf b/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf index 7e9e1daea0d..a70ec8fb257 100644 --- a/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Tasks/Resources/xlf/Strings.zh-Hant.xlf @@ -2515,6 +2515,16 @@ MSB3932: 因為檔案不存在或無法存取，所以無法解壓縮檔案 "{0}"。 {StrBegin="MSB3932: "} + + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + MSB3937: Failed to parse pattern "{0}" because it contains an invalid path character. + {StrBegin="MSB3937: "} + + + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + MSB3938: Failed to parse pattern "{0}" because it contains a property reference which isn't supported. + {StrBegin="MSB3938: "} + Unzipping file "{0}" to "{1}". 正在將檔案 "{0}" 解壓縮到 "{1}"。 diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index e82340e8e17..f243cce66f4 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -9,6 +9,8 @@ using System.Linq; using System.Text.RegularExpressions; using System.Threading; + +using Microsoft.Build.Shared; using Microsoft.Build.Shared.FileSystem; namespace Microsoft.Build.Tasks @@ -28,6 +30,16 @@ public sealed class Unzip : TaskExtension, ICancelableTask /// private readonly CancellationTokenSource _cancellationToken = new CancellationTokenSource(); + /// + /// Stores the include patterns after parsing. + /// + private string[] _includePatterns; + + /// + /// Stores the exclude patterns after parsing. + /// + private string[] _excludePatterns; + /// /// Gets or sets a with a destination folder path to unzip the files to. /// @@ -51,14 +63,14 @@ public sealed class Unzip : TaskExtension, ICancelableTask public ITaskItem[] SourceFiles { get; set; } /// - /// Gets or sets a regular expression that will be used to include files to be unzipped. + /// Gets or sets a glob expression that will be used to determine which files to include being unzipped from the archive. /// - public string IncludePattern { get; set; } + public string Include { get; set; } /// - /// Gets or sets a regular expression that will be used to exclude files to be unzipped. + /// Gets or sets a glob expression that will be used to determine which files to exclude from being unzipped from the archive. /// - public string ExcludePattern { get; set; } + public string Exclude { get; set; } /// public void Cancel() @@ -85,42 +97,45 @@ public override bool Execute() try { - foreach (ITaskItem sourceFile in SourceFiles.TakeWhile(i => !_cancellationToken.IsCancellationRequested)) + if (ParseIncludeExclude()) { - if (!FileSystems.Default.FileExists(sourceFile.ItemSpec)) + foreach (ITaskItem sourceFile in SourceFiles.TakeWhile(i => !_cancellationToken.IsCancellationRequested)) { - Log.LogErrorWithCodeFromResources("Unzip.ErrorFileDoesNotExist", sourceFile.ItemSpec); - continue; - } + if (!FileSystems.Default.FileExists(sourceFile.ItemSpec)) + { + Log.LogErrorWithCodeFromResources("Unzip.ErrorFileDoesNotExist", sourceFile.ItemSpec); + continue; + } - try - { - using (FileStream stream = new FileStream(sourceFile.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 0x1000, useAsync: false)) + try { - using (ZipArchive zipArchive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: false)) + using (FileStream stream = new FileStream(sourceFile.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 0x1000, useAsync: false)) { - try - { - Extract(zipArchive, destinationDirectory); - } - catch (Exception e) + using (ZipArchive zipArchive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: false)) { - // Unhandled exception in Extract() is a bug! - Log.LogErrorFromException(e, showStackTrace: true); - return false; + try + { + Extract(zipArchive, destinationDirectory); + } + catch (Exception e) + { + // Unhandled exception in Extract() is a bug! + Log.LogErrorFromException(e, showStackTrace: true); + return false; + } } } } - } - catch (OperationCanceledException) - { - break; - } - catch (Exception e) - { - // Should only be thrown if the archive could not be opened (Access denied, corrupt file, etc) - Log.LogErrorWithCodeFromResources("Unzip.ErrorCouldNotOpenFile", sourceFile.ItemSpec, e.Message); - } + catch (OperationCanceledException) + { + break; + } + catch (Exception e) + { + // Should only be thrown if the archive could not be opened (Access denied, corrupt file, etc) + Log.LogErrorWithCodeFromResources("Unzip.ErrorCouldNotOpenFile", sourceFile.ItemSpec, e.Message); + } + } } } finally @@ -225,14 +240,24 @@ private bool ShouldSkipEntry(ZipArchiveEntry zipArchiveEntry) { bool result = false; - if (!string.IsNullOrWhiteSpace(Include)) + if (_includePatterns.Length > 0) { - result |= !Regex.IsMatch(zipArchiveEntry.FullName, Include); + result |= _includePatterns.Aggregate( + false, + (current, pattern) => current | !FileMatcher.IsMatch( + FileMatcher.Normalize(zipArchiveEntry.FullName), + pattern, + true)); } - if (!string.IsNullOrWhiteSpace(Exclude)) + if (_excludePatterns.Length > 0) { - result |= Regex.IsMatch(zipArchiveEntry.FullName, Exclude); + result |= _excludePatterns.Aggregate( + false, + (current, pattern) => current | FileMatcher.IsMatch( + FileMatcher.Normalize(zipArchiveEntry.FullName), + pattern, + true)); } return result; @@ -251,5 +276,41 @@ private bool ShouldSkipEntry(ZipArchiveEntry zipArchiveEntry, FileInfo fileInfo) && zipArchiveEntry.LastWriteTime == fileInfo.LastWriteTimeUtc && zipArchiveEntry.Length == fileInfo.Length; } + + private bool ParseIncludeExclude() + { + return ParsePattern(Include, out _includePatterns) && ParsePattern(Exclude, out _excludePatterns); + } + + private bool ParsePattern(string pattern, out string[] patterns) + { + bool result = false; + patterns = Array.Empty(); + if (string.IsNullOrWhiteSpace(pattern)) + { + result = true; + } + else if (FileMatcher.HasPropertyReferences(pattern)) + { + // Supporting property references would require access to Expander which is unavailable in Microsoft.Build.Tasks + Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternPropertyReferences", pattern); + } + else + { + patterns = pattern.Contains(';') + ? pattern.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(FileMatcher.Normalize).ToArray() + : new[] { pattern }; + if (patterns.Any(p => p.IndexOfAny(Path.GetInvalidPathChars()) != -1)) + { + Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternInvalidPath", pattern); + } + else + { + result = true; + } + } + + return result; + } } } From 941d7eb5620562e697f3476ee9b7b1dab539d331 Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Wed, 27 Jan 2021 07:59:00 +0100 Subject: [PATCH 07/19] Unzip task unit tests for pattern parsing --- src/Tasks.UnitTests/Unzip_Tests.cs | 112 +++++++++++++++++++++++++++++ src/Tasks/Unzip.cs | 5 +- 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/src/Tasks.UnitTests/Unzip_Tests.cs b/src/Tasks.UnitTests/Unzip_Tests.cs index 5f62209711e..5b0256bd55f 100644 --- a/src/Tasks.UnitTests/Unzip_Tests.cs +++ b/src/Tasks.UnitTests/Unzip_Tests.cs @@ -309,5 +309,117 @@ public void CanUnzip_WithIncludeAndExcludeFilter() _mockEngine.Log.ShouldNotContain(Path.Combine(destination.Path, "sub", "subfile.js"), () => _mockEngine.Log); } } + + [Fact] + public void LogsErrorIfIncludeContainsInvalidPathCharacters() + { + using (TestEnvironment testEnvironment = TestEnvironment.Create()) + { + TransientTestFolder source = testEnvironment.CreateFolder(createFolder: true); + TransientTestFolder destination = testEnvironment.CreateFolder(createFolder: false); + testEnvironment.CreateFile(source, "BE78A17D30144B549D21F71D5C633F7D.txt", "file1"); + testEnvironment.CreateFile(source, "A04FF4B88DF14860B7C73A8E75A4FB76.txt", "file2"); + + TransientZipArchive zipArchive = TransientZipArchive.Create(source, testEnvironment.CreateFolder(createFolder: true)); + + Unzip unzip = new Unzip + { + BuildEngine = _mockEngine, + DestinationFolder = new TaskItem(destination.Path), + OverwriteReadOnlyFiles = true, + SkipUnchangedFiles = false, + SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, + Include = ".txt" + }; + + unzip.Execute().ShouldBeFalse(() => _mockEngine.Log); + + _mockEngine.Log.ShouldContain("MSB3937", () => _mockEngine.Log); + } + } + + [Fact] + public void LogsErrorIfIncludeContainsPropertyReferences() + { + using (TestEnvironment testEnvironment = TestEnvironment.Create()) + { + TransientTestFolder source = testEnvironment.CreateFolder(createFolder: true); + TransientTestFolder destination = testEnvironment.CreateFolder(createFolder: false); + testEnvironment.CreateFile(source, "BE78A17D30144B549D21F71D5C633F7D.txt", "file1"); + testEnvironment.CreateFile(source, "A04FF4B88DF14860B7C73A8E75A4FB76.txt", "file2"); + + TransientZipArchive zipArchive = TransientZipArchive.Create(source, testEnvironment.CreateFolder(createFolder: true)); + + Unzip unzip = new Unzip + { + BuildEngine = _mockEngine, + DestinationFolder = new TaskItem(destination.Path), + OverwriteReadOnlyFiles = true, + SkipUnchangedFiles = false, + SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, + Include = "$(Include)" + }; + + unzip.Execute().ShouldBeFalse(() => _mockEngine.Log); + + _mockEngine.Log.ShouldContain("MSB3938", () => _mockEngine.Log); + } + } + + [Fact] + public void LogsErrorIfExcludeContainsInvalidPathCharacters() + { + using (TestEnvironment testEnvironment = TestEnvironment.Create()) + { + TransientTestFolder source = testEnvironment.CreateFolder(createFolder: true); + TransientTestFolder destination = testEnvironment.CreateFolder(createFolder: false); + testEnvironment.CreateFile(source, "BE78A17D30144B549D21F71D5C633F7D.txt", "file1"); + testEnvironment.CreateFile(source, "A04FF4B88DF14860B7C73A8E75A4FB76.txt", "file2"); + + TransientZipArchive zipArchive = TransientZipArchive.Create(source, testEnvironment.CreateFolder(createFolder: true)); + + Unzip unzip = new Unzip + { + BuildEngine = _mockEngine, + DestinationFolder = new TaskItem(destination.Path), + OverwriteReadOnlyFiles = true, + SkipUnchangedFiles = false, + SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, + Exclude = ".txt" + }; + + unzip.Execute().ShouldBeFalse(() => _mockEngine.Log); + + _mockEngine.Log.ShouldContain("MSB3937", () => _mockEngine.Log); + } + } + + [Fact] + public void LogsErrorIfExcludeContainsPropertyReferences() + { + using (TestEnvironment testEnvironment = TestEnvironment.Create()) + { + TransientTestFolder source = testEnvironment.CreateFolder(createFolder: true); + TransientTestFolder destination = testEnvironment.CreateFolder(createFolder: false); + testEnvironment.CreateFile(source, "BE78A17D30144B549D21F71D5C633F7D.txt", "file1"); + testEnvironment.CreateFile(source, "A04FF4B88DF14860B7C73A8E75A4FB76.txt", "file2"); + + TransientZipArchive zipArchive = TransientZipArchive.Create(source, testEnvironment.CreateFolder(createFolder: true)); + + Unzip unzip = new Unzip + { + BuildEngine = _mockEngine, + DestinationFolder = new TaskItem(destination.Path), + OverwriteReadOnlyFiles = true, + SkipUnchangedFiles = false, + SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, + Exclude = "$(Include)" + }; + + unzip.Execute().ShouldBeFalse(() => _mockEngine.Log); + + _mockEngine.Log.ShouldContain("MSB3938", () => _mockEngine.Log); + } + } } } diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index f243cce66f4..f44a332ff8f 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -1,17 +1,16 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using System; using System.IO; using System.IO.Compression; using System.Linq; -using System.Text.RegularExpressions; using System.Threading; +using Microsoft.Build.Framework; using Microsoft.Build.Shared; using Microsoft.Build.Shared.FileSystem; +using Microsoft.Build.Utilities; namespace Microsoft.Build.Tasks { From 658c23a91d30d78bca72197f2c6d1785962c4a88 Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Thu, 28 Jan 2021 08:53:43 +0100 Subject: [PATCH 08/19] Name adjustment to better match actual verification --- src/Shared/FileMatcher.cs | 8 ++++---- src/Tasks/Unzip.cs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Shared/FileMatcher.cs b/src/Shared/FileMatcher.cs index 962d58a3024..fa1cc09db2e 100644 --- a/src/Shared/FileMatcher.cs +++ b/src/Shared/FileMatcher.cs @@ -31,7 +31,7 @@ internal class FileMatcher private static readonly char[] s_wildcardCharacters = { '*', '?' }; private static readonly char[] s_wildcardAndSemicolonCharacters = { '*', '?', ';' }; - private static readonly string[] s_propertyReferences = { "$(", "@(" }; + private static readonly string[] s_propertyAndItemReferences = { "$(", "@(" }; // on OSX both System.IO.Path separators are '/', so we have to use the literals internal static readonly char[] directorySeparatorCharacters = { '/', '\\' }; @@ -195,16 +195,16 @@ internal static bool HasWildcardsSemicolonItemOrPropertyReferences(string filesp return (-1 != filespec.IndexOfAny(s_wildcardAndSemicolonCharacters)) || - HasPropertyReferences(filespec) + HasPropertyOrItemReferences(filespec) ; } /// /// Determines whether the given path has any property references. /// - internal static bool HasPropertyReferences(string filespec) + internal static bool HasPropertyOrItemReferences(string filespec) { - return s_propertyReferences.Aggregate(false, (current, propertyReference) => current | filespec.Contains(propertyReference)); + return s_propertyAndItemReferences.Aggregate(false, (current, propertyReference) => current | filespec.Contains(propertyReference)); } /// diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index f44a332ff8f..1ac2687628b 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -289,7 +289,7 @@ private bool ParsePattern(string pattern, out string[] patterns) { result = true; } - else if (FileMatcher.HasPropertyReferences(pattern)) + else if (FileMatcher.HasPropertyOrItemReferences(pattern)) { // Supporting property references would require access to Expander which is unavailable in Microsoft.Build.Tasks Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternPropertyReferences", pattern); From 8164bfdf08382dfb6573ecc0ae46216d1756f0c5 Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Thu, 28 Jan 2021 09:53:16 +0100 Subject: [PATCH 09/19] Attempt to fail valid character path tests on MacOS and Linux --- src/Tasks.UnitTests/Unzip_Tests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tasks.UnitTests/Unzip_Tests.cs b/src/Tasks.UnitTests/Unzip_Tests.cs index 5b0256bd55f..3bd69a4b417 100644 --- a/src/Tasks.UnitTests/Unzip_Tests.cs +++ b/src/Tasks.UnitTests/Unzip_Tests.cs @@ -329,7 +329,7 @@ public void LogsErrorIfIncludeContainsInvalidPathCharacters() OverwriteReadOnlyFiles = true, SkipUnchangedFiles = false, SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, - Include = ".txt" + Include = " _mockEngine.Log); @@ -385,7 +385,7 @@ public void LogsErrorIfExcludeContainsInvalidPathCharacters() OverwriteReadOnlyFiles = true, SkipUnchangedFiles = false, SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, - Exclude = ".txt" + Exclude = " _mockEngine.Log); From 1f1aa6d56ecca82e22537740883c37f9dfbca493 Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Thu, 28 Jan 2021 11:16:03 +0100 Subject: [PATCH 10/19] Made InvalidPath Unzip unit tests platform specific --- src/Tasks.UnitTests/Unzip_Tests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tasks.UnitTests/Unzip_Tests.cs b/src/Tasks.UnitTests/Unzip_Tests.cs index 3bd69a4b417..164dec11bb9 100644 --- a/src/Tasks.UnitTests/Unzip_Tests.cs +++ b/src/Tasks.UnitTests/Unzip_Tests.cs @@ -310,6 +310,7 @@ public void CanUnzip_WithIncludeAndExcludeFilter() } } + [PlatformSpecific(TestPlatforms.Windows)] [Fact] public void LogsErrorIfIncludeContainsInvalidPathCharacters() { @@ -366,6 +367,7 @@ public void LogsErrorIfIncludeContainsPropertyReferences() } } + [PlatformSpecific(TestPlatforms.Windows)] [Fact] public void LogsErrorIfExcludeContainsInvalidPathCharacters() { From 5e489e2cdef310963bd4c294313b07cd35aea8cf Mon Sep 17 00:00:00 2001 From: IvanLieckens Date: Fri, 29 Jan 2021 09:31:40 +0100 Subject: [PATCH 11/19] Improved property or item reference detection Co-authored-by: Forgind --- src/Shared/FileMatcher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Shared/FileMatcher.cs b/src/Shared/FileMatcher.cs index fa1cc09db2e..26fc279753f 100644 --- a/src/Shared/FileMatcher.cs +++ b/src/Shared/FileMatcher.cs @@ -204,7 +204,7 @@ internal static bool HasWildcardsSemicolonItemOrPropertyReferences(string filesp /// internal static bool HasPropertyOrItemReferences(string filespec) { - return s_propertyAndItemReferences.Aggregate(false, (current, propertyReference) => current | filespec.Contains(propertyReference)); + return s_propertyAndItemReferences.Any(ref=> filespec.Contains(ref)); } /// From 9fedb34fb31c2769aca86f5e749b06a0d20d7142 Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Fri, 29 Jan 2021 09:39:59 +0100 Subject: [PATCH 12/19] Invalid path detection before splitting improvement --- src/Tasks/Unzip.cs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index 1ac2687628b..f7e17336df2 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -134,7 +134,7 @@ public override bool Execute() // Should only be thrown if the archive could not be opened (Access denied, corrupt file, etc) Log.LogErrorWithCodeFromResources("Unzip.ErrorCouldNotOpenFile", sourceFile.ItemSpec, e.Message); } - } + } } } finally @@ -294,19 +294,17 @@ private bool ParsePattern(string pattern, out string[] patterns) // Supporting property references would require access to Expander which is unavailable in Microsoft.Build.Tasks Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternPropertyReferences", pattern); } + else if (pattern.IndexOfAny(FileUtilities.InvalidPathChars) != -1) + { + Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternInvalidPath", pattern); + } else { patterns = pattern.Contains(';') ? pattern.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(FileMatcher.Normalize).ToArray() : new[] { pattern }; - if (patterns.Any(p => p.IndexOfAny(Path.GetInvalidPathChars()) != -1)) - { - Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternInvalidPath", pattern); - } - else - { - result = true; - } + + result = true; } return result; From 62d56a9aabbd6bee7a6811f968f5dfea5f235803 Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Fri, 29 Jan 2021 09:40:22 +0100 Subject: [PATCH 13/19] Added | to invalid path testing --- src/Tasks.UnitTests/Unzip_Tests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tasks.UnitTests/Unzip_Tests.cs b/src/Tasks.UnitTests/Unzip_Tests.cs index 164dec11bb9..487960faec9 100644 --- a/src/Tasks.UnitTests/Unzip_Tests.cs +++ b/src/Tasks.UnitTests/Unzip_Tests.cs @@ -330,7 +330,7 @@ public void LogsErrorIfIncludeContainsInvalidPathCharacters() OverwriteReadOnlyFiles = true, SkipUnchangedFiles = false, SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, - Include = " _mockEngine.Log); @@ -387,7 +387,7 @@ public void LogsErrorIfExcludeContainsInvalidPathCharacters() OverwriteReadOnlyFiles = true, SkipUnchangedFiles = false, SourceFiles = new ITaskItem[] { new TaskItem(zipArchive.Path) }, - Exclude = " _mockEngine.Log); From 76a94f14c795d789455fa7bb0d6de0af709f9bed Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Fri, 29 Jan 2021 09:44:15 +0100 Subject: [PATCH 14/19] Removed platform specific attribute on Invalid Path Character tests in Unzip unit tests --- src/Tasks.UnitTests/Unzip_Tests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Tasks.UnitTests/Unzip_Tests.cs b/src/Tasks.UnitTests/Unzip_Tests.cs index 487960faec9..8b48e6ccaf8 100644 --- a/src/Tasks.UnitTests/Unzip_Tests.cs +++ b/src/Tasks.UnitTests/Unzip_Tests.cs @@ -310,7 +310,6 @@ public void CanUnzip_WithIncludeAndExcludeFilter() } } - [PlatformSpecific(TestPlatforms.Windows)] [Fact] public void LogsErrorIfIncludeContainsInvalidPathCharacters() { @@ -367,7 +366,6 @@ public void LogsErrorIfIncludeContainsPropertyReferences() } } - [PlatformSpecific(TestPlatforms.Windows)] [Fact] public void LogsErrorIfExcludeContainsInvalidPathCharacters() { From e9e0f5fed07bb43963bd9c32bba1f2de01a8352e Mon Sep 17 00:00:00 2001 From: IvanLieckens Date: Fri, 29 Jan 2021 09:46:35 +0100 Subject: [PATCH 15/19] Perf improvement detecting include pattern match Co-authored-by: Forgind --- src/Tasks/Unzip.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index f7e17336df2..59fbda75366 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -241,12 +241,7 @@ private bool ShouldSkipEntry(ZipArchiveEntry zipArchiveEntry) if (_includePatterns.Length > 0) { - result |= _includePatterns.Aggregate( - false, - (current, pattern) => current | !FileMatcher.IsMatch( - FileMatcher.Normalize(zipArchiveEntry.FullName), - pattern, - true)); + result = _includePatterns.All(pattern => !FileMatcher.IsMatch(FileMatcher.Normalize(zipArchiveEntry.FullName), pattern, true)); } if (_excludePatterns.Length > 0) From 8608320c23ddfa7b4571bee570c72a568f208023 Mon Sep 17 00:00:00 2001 From: IvanLieckens Date: Fri, 29 Jan 2021 09:46:51 +0100 Subject: [PATCH 16/19] Perf improvement detecting exclude pattern match Co-authored-by: Forgind --- src/Tasks/Unzip.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index 59fbda75366..665ab8e59a5 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -246,12 +246,7 @@ private bool ShouldSkipEntry(ZipArchiveEntry zipArchiveEntry) if (_excludePatterns.Length > 0) { - result |= _excludePatterns.Aggregate( - false, - (current, pattern) => current | FileMatcher.IsMatch( - FileMatcher.Normalize(zipArchiveEntry.FullName), - pattern, - true)); + result |= _excludePatterns.Any(pattern => FileMatcher.IsMatch(FileMatcher.Normalize(zipArchiveEntry.FullName), pattern, true)); } return result; From e4d0286a63fd89dc302453174c7c2774ade01e8d Mon Sep 17 00:00:00 2001 From: "Ivan Lieckens (Sitecore)" Date: Fri, 29 Jan 2021 10:18:54 +0100 Subject: [PATCH 17/19] Small FileMatcher correction, Removed return on parsing for patterns of Unzip Task --- src/Shared/FileMatcher.cs | 2 +- src/Tasks/Unzip.cs | 49 ++++++++++++++++++--------------------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/Shared/FileMatcher.cs b/src/Shared/FileMatcher.cs index 26fc279753f..0b187644116 100644 --- a/src/Shared/FileMatcher.cs +++ b/src/Shared/FileMatcher.cs @@ -204,7 +204,7 @@ internal static bool HasWildcardsSemicolonItemOrPropertyReferences(string filesp /// internal static bool HasPropertyOrItemReferences(string filespec) { - return s_propertyAndItemReferences.Any(ref=> filespec.Contains(ref)); + return s_propertyAndItemReferences.Any(filespec.Contains); } /// diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index 665ab8e59a5..e9b55609031 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -96,7 +96,9 @@ public override bool Execute() try { - if (ParseIncludeExclude()) + ParseIncludeExclude(); + + if (!Log.HasLoggedErrors) { foreach (ITaskItem sourceFile in SourceFiles.TakeWhile(i => !_cancellationToken.IsCancellationRequested)) { @@ -266,38 +268,33 @@ private bool ShouldSkipEntry(ZipArchiveEntry zipArchiveEntry, FileInfo fileInfo) && zipArchiveEntry.Length == fileInfo.Length; } - private bool ParseIncludeExclude() + private void ParseIncludeExclude() { - return ParsePattern(Include, out _includePatterns) && ParsePattern(Exclude, out _excludePatterns); + ParsePattern(Include, out _includePatterns); + ParsePattern(Exclude, out _excludePatterns); } - private bool ParsePattern(string pattern, out string[] patterns) + private void ParsePattern(string pattern, out string[] patterns) { - bool result = false; patterns = Array.Empty(); - if (string.IsNullOrWhiteSpace(pattern)) + if (!string.IsNullOrWhiteSpace(pattern)) { - result = true; - } - else if (FileMatcher.HasPropertyOrItemReferences(pattern)) - { - // Supporting property references would require access to Expander which is unavailable in Microsoft.Build.Tasks - Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternPropertyReferences", pattern); - } - else if (pattern.IndexOfAny(FileUtilities.InvalidPathChars) != -1) - { - Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternInvalidPath", pattern); - } - else - { - patterns = pattern.Contains(';') - ? pattern.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(FileMatcher.Normalize).ToArray() - : new[] { pattern }; - - result = true; + if (FileMatcher.HasPropertyOrItemReferences(pattern)) + { + // Supporting property references would require access to Expander which is unavailable in Microsoft.Build.Tasks + Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternPropertyReferences", pattern); + } + else if (pattern.IndexOfAny(FileUtilities.InvalidPathChars) != -1) + { + Log.LogErrorWithCodeFromResources("Unzip.ErrorParsingPatternInvalidPath", pattern); + } + else + { + patterns = pattern.Contains(';') + ? pattern.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(FileMatcher.Normalize).ToArray() + : new[] { pattern }; + } } - - return result; } } } From 164e8e7c8ed246b0b5ee45c36f8824ec5dd37974 Mon Sep 17 00:00:00 2001 From: IvanLieckens Date: Sat, 30 Jan 2021 00:17:44 +0100 Subject: [PATCH 18/19] Updated Include description Co-authored-by: Mihai Codoban --- src/Tasks/Unzip.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index e9b55609031..ba53e939b10 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -62,7 +62,7 @@ public sealed class Unzip : TaskExtension, ICancelableTask public ITaskItem[] SourceFiles { get; set; } /// - /// Gets or sets a glob expression that will be used to determine which files to include being unzipped from the archive. + /// Gets or sets an MSBuild glob expression that will be used to determine which files to include being unzipped from the archive. /// public string Include { get; set; } From 081c1796da9fd8d812e54f1d295f7143e0dc49e1 Mon Sep 17 00:00:00 2001 From: IvanLieckens Date: Sat, 30 Jan 2021 00:18:22 +0100 Subject: [PATCH 19/19] Updated Exclude description Co-authored-by: Mihai Codoban --- src/Tasks/Unzip.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tasks/Unzip.cs b/src/Tasks/Unzip.cs index ba53e939b10..4e00a677831 100644 --- a/src/Tasks/Unzip.cs +++ b/src/Tasks/Unzip.cs @@ -67,7 +67,7 @@ public sealed class Unzip : TaskExtension, ICancelableTask public string Include { get; set; } /// - /// Gets or sets a glob expression that will be used to determine which files to exclude from being unzipped from the archive. + /// Gets or sets an MSBuild glob expression that will be used to determine which files to exclude from being unzipped from the archive. /// public string Exclude { get; set; }