From feaf10a0803e62aaa2afd640c00f91c373d0cf6d Mon Sep 17 00:00:00 2001 From: Hamza El-Saawy <84944216+helsaawy@users.noreply.github.com> Date: Tue, 23 Aug 2022 16:22:24 -0400 Subject: [PATCH] Added LCOW functional tests and benchmarks for uVMs and containers. (#1351) * Added LCOW functional tests and benchmarks Split out utility functions from `test/functional` into an internal package, separate from functional tests. Updated code to use containerd instead of docker. Added new functional tests and benchmarks for LCOW uVM containers, and updated other LCOW tests as well. Not all (LCOW) functional tests were updated, and most others are now explicitly skipped. Updated `k8s.io/cri-api` to v0.22 to include `WindowsPodSandboxConfig` struct Signed-off-by: Hamza El-Saawy * Updating tests to use new test\internal package Signed-off-by: Hamza El-Saawy * PR: doc, simplified signatures Added deprecated warning to `layers.LayerFolders`, which relies on docker. Added doc comment to functional tests to clarify overlap with other tests. Removed unnecessary parameter in `WaitForError`. Updated snapshotter logic Signed-off-by: Hamza El-Saawy * PR: refactor, updated image names in cri tests Signed-off-by: Hamza El-Saawy Signed-off-by: Hamza El-Saawy --- scripts/Test-Functional.ps1 | 62 +++++ scripts/Testing.psm1 | 144 ++++++++++++ test/cri-containerd/clone_test.go | 16 +- .../container_downlevel_test.go | 4 +- .../container_layers_packing_test.go | 8 +- test/cri-containerd/container_update_test.go | 4 +- test/cri-containerd/createcontainer_test.go | 6 +- test/cri-containerd/jobcontainer_test.go | 13 +- test/cri-containerd/main_test.go | 70 ++---- test/cri-containerd/runpodsandbox_test.go | 7 +- test/functional/lcow_bench_test.go | 101 ++++++++ test/functional/lcow_container_bench_test.go | 216 ++++++++++++++++++ test/functional/lcow_container_test.go | 177 ++++++++++++++ test/functional/lcow_test.go | 110 ++++++--- test/functional/main_test.go | 175 ++++++++++++++ test/functional/manifest/manifest.go | 4 - test/functional/manifest/rsrc_amd64.syso | Bin 372470 -> 0 bytes test/functional/manifest_test.go | 3 - test/functional/test.go | 49 ---- test/functional/utilities/layerfolders.go | 53 ----- test/functional/utilities/requiresbuild.go | 19 -- test/functional/utilities/tempdir.go | 15 -- test/functional/uvm_mem_backingtype_test.go | 40 +++- test/functional/uvm_memory_test.go | 24 +- test/functional/uvm_plannine_test.go | 28 ++- test/functional/uvm_properties_test.go | 23 +- test/functional/uvm_scratch_test.go | 17 +- test/functional/uvm_scsi_test.go | 34 ++- test/functional/uvm_virtualdevice_test.go | 15 +- test/functional/uvm_vpmem_test.go | 17 +- test/functional/uvm_vsmb_test.go | 25 +- test/functional/wcow_test.go | 64 +++--- test/go.mod | 1 + test/go.sum | 1 + test/internal/cmd/cmd.go | 89 ++++++++ test/internal/cmd/io.go | 64 ++++++ test/internal/constants/constants.go | 30 +++ test/internal/constants/images.go | 86 +++++++ test/internal/container/container.go | 83 +++++++ test/internal/containerd/containerd.go | 207 +++++++++++++++++ .../defaultlinuxspec.go | 4 +- .../defaultwindowsspec.go | 4 +- test/internal/doc.go | 16 ++ test/internal/flag/flag.go | 77 +++++++ test/internal/layers/layerfolders.go | 93 ++++++++ test/internal/layers/scratch.go | 49 ++++ test/internal/manifest/manifest.go | 6 + test/internal/manifest/manifest.xml | 10 + .../manifest/resource_windows_386.syso | Bin 0 -> 998 bytes .../manifest/resource_windows_amd64.syso | Bin 0 -> 998 bytes .../manifest/resource_windows_arm.syso | Bin 0 -> 998 bytes .../manifest/resource_windows_arm64.syso | Bin 0 -> 998 bytes test/internal/manifest/versioninfo.json | 43 ++++ test/internal/oci/cri.go | 50 ++++ test/internal/oci/images.go | 19 ++ test/internal/oci/oci.go | 125 ++++++++++ test/internal/require/build.go | 23 ++ test/internal/require/requires.go | 61 +++++ test/internal/schemaversion_test.go | 3 +- .../utilities => internal}/scratch.go | 8 +- .../utilities => internal}/stringsetflag.go | 10 +- .../testdata}/defaultlinuxspec.json | 0 .../testdata}/defaultwindowsspec.json | 0 .../samples/config.justin.lcow.working.json | 0 .../samples/from-docker-linux/privileged.json | 0 .../samples/from-docker-linux/sh.json | 0 test/internal/timeout/timeout.go | 45 ++++ test/internal/uvm/lcow.go | 47 ++++ test/internal/uvm/uvm.go | 59 +++++ .../createuvm.go => internal/uvm/wcow.go} | 51 ++--- test/runhcs/e2e_matrix_test.go | 33 +-- test/runhcs/runhcs_test.go | 2 +- 72 files changed, 2526 insertions(+), 416 deletions(-) create mode 100644 scripts/Test-Functional.ps1 create mode 100644 scripts/Testing.psm1 create mode 100644 test/functional/lcow_bench_test.go create mode 100644 test/functional/lcow_container_bench_test.go create mode 100644 test/functional/lcow_container_test.go create mode 100644 test/functional/main_test.go delete mode 100644 test/functional/manifest/manifest.go delete mode 100644 test/functional/manifest/rsrc_amd64.syso delete mode 100644 test/functional/manifest_test.go delete mode 100644 test/functional/test.go delete mode 100644 test/functional/utilities/layerfolders.go delete mode 100644 test/functional/utilities/requiresbuild.go delete mode 100644 test/functional/utilities/tempdir.go create mode 100644 test/internal/cmd/cmd.go create mode 100644 test/internal/cmd/io.go create mode 100644 test/internal/constants/constants.go create mode 100644 test/internal/constants/images.go create mode 100644 test/internal/container/container.go create mode 100644 test/internal/containerd/containerd.go rename test/{functional/utilities => internal}/defaultlinuxspec.go (92%) rename test/{functional/utilities => internal}/defaultwindowsspec.go (93%) create mode 100644 test/internal/doc.go create mode 100644 test/internal/flag/flag.go create mode 100644 test/internal/layers/layerfolders.go create mode 100644 test/internal/layers/scratch.go create mode 100644 test/internal/manifest/manifest.go create mode 100644 test/internal/manifest/manifest.xml create mode 100644 test/internal/manifest/resource_windows_386.syso create mode 100644 test/internal/manifest/resource_windows_amd64.syso create mode 100644 test/internal/manifest/resource_windows_arm.syso create mode 100644 test/internal/manifest/resource_windows_arm64.syso create mode 100644 test/internal/manifest/versioninfo.json create mode 100644 test/internal/oci/cri.go create mode 100644 test/internal/oci/images.go create mode 100644 test/internal/oci/oci.go create mode 100644 test/internal/require/build.go create mode 100644 test/internal/require/requires.go rename test/{functional/utilities => internal}/scratch.go (91%) rename test/{functional/utilities => internal}/stringsetflag.go (82%) rename test/{functional/assets => internal/testdata}/defaultlinuxspec.json (100%) rename test/{functional/assets => internal/testdata}/defaultwindowsspec.json (100%) rename test/{functional/assets => internal/testdata}/samples/config.justin.lcow.working.json (100%) rename test/{functional/assets => internal/testdata}/samples/from-docker-linux/privileged.json (100%) rename test/{functional/assets => internal/testdata}/samples/from-docker-linux/sh.json (100%) create mode 100644 test/internal/timeout/timeout.go create mode 100644 test/internal/uvm/lcow.go create mode 100644 test/internal/uvm/uvm.go rename test/{functional/utilities/createuvm.go => internal/uvm/wcow.go} (51%) diff --git a/scripts/Test-Functional.ps1 b/scripts/Test-Functional.ps1 new file mode 100644 index 0000000000..5884cd6d82 --- /dev/null +++ b/scripts/Test-Functional.ps1 @@ -0,0 +1,62 @@ +# ex: .\scripts\Test-Functional.ps1 -Action Bench -Count 2 -BenchTime "2x" + +[CmdletBinding()] +param ( + [ValidateSet('Test', 'Bench', 'List')] + [alias('a')] + [string] + $Action = 'Bench', + + [string] + $Note = '', + + [string] + $OutDirectory = '.\test\results', + + # test parameters + [int] + $Count = 1, + + [string] + $BenchTime = '5s', + + [string] + $Timeout = '10m', + + [alias('tv')] + [switch] + $TestVerbose, + + [string] + $Run = '', + + [string] + $Feature = '' +) + +Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force + +$date = Get-Date +$testcmd, $out = New-TestCommand ` + -Action $Action ` + -Path .\bin\test\functional.exe ` + -Name functional ` + -OutDirectory $OutDirectory ` + -Date $date ` + -Note $Note ` + -TestVerbose:$TestVerbose ` + -Count $Count ` + -BenchTime $BenchTime ` + -Timeout $Timeout ` + -Run $Run ` + -Feature $Feature ` + -Verbose:$Verbose + +Invoke-TestCommand ` + -TestCmd $testcmd ` + -OutputFile $out ` + -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) ` + -Preamble ` + -Date $Date ` + -Note $Note ` + -Verbose:$Verbose diff --git a/scripts/Testing.psm1 b/scripts/Testing.psm1 new file mode 100644 index 0000000000..3b79edc710 --- /dev/null +++ b/scripts/Testing.psm1 @@ -0,0 +1,144 @@ +function New-TestCommand { + [CmdletBinding()] + param ( + [ValidateSet('Test', 'Bench', 'List')] + [alias('a')] + [string] + $Action = 'Bench', + + [Parameter(Mandatory)] + [string] + $Path, + + [Parameter(Mandatory)] + [string] + $Name, + + [Parameter(Mandatory)] + [string] + $OutDirectory , + + [DateTime] + $Date = (Get-Date), + + [string] + $Note = '', + + # test parameters + [alias('tv')] + [switch] + $TestVerbose = $false, + + [int] + $Count = 1, + + [string] + $BenchTime = '5s', + + [string] + $Timeout = '10m', + + [string] + $Run = '', + + [string] + $Feature = '' + ) + + $OutDirectory = Resolve-Path $OutDirectory + Write-Verbose "creating $OutDirectory" + + New-Item -ItemType 'directory' -Path $OutDirectory -Force > $null + + $testcmd = "$Path `'-test.timeout=$Timeout`' `'-test.shuffle=on`' `'-test.count=$Count`' " + + if ( $TestVerbose ) { + $testcmd += ' ''-test.v'' ' + } + + switch ( $Action ) { + 'List' { + if ( $Run -eq '' ) { + $Run = '.' + } + $testcmd += " `'-test.list=$Run`' " + } + 'Test' { + if ( $Run -ne '' ) { + $testcmd += " `'-test.run=$Run`' " + } + } + 'Bench' { + if ( $Run -eq '' ) { + $Run = '.' + } + $testcmd += ' ''-test.run=^#'' ''-test.benchmem'' ' + ` + " `'-test.bench=$Run`' `'-test.benchtime=$BenchTime`' " + } + } + + if ( $Feature -ne '' ) { + $testcmd += " `'-feature=$Feature`' " + } + + $f = $Name + '-' + $Action + if ($Note -ne '' ) { + $f += '-' + $Note + } + $out = Join-Path $OutDirectory "$f-$(Get-Date -Date $date -Format FileDateTime).txt" + + return $testcmd, $out +} + +function Invoke-TestCommand { + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [string] + $TestCmd, + + [string] + $TestCmdPreamble = $TestCmd, + + [string] + $OutputFile = 'nul', + + [string] + $OutputCmd, + + [switch] + $Preamble, + + [DateTime] + $Date = (Get-Date), + + [string] + $Note + ) + + if ($OutputFile -eq '' ) { + $OutputFile = 'nul' + } + + Write-Verbose "Saving output to: $OutputFile" + if ( $Preamble ) { + & { + Write-Output "test.date: $(Get-Date -Date $Date -UFormat '%FT%R%Z' -AsUTC)" + if ( $Note -ne '' ) { + Write-Output "note: $Note" + } + Write-Output "test.command: $TestCmdPreamble" + Write-Output "pkg.commit: $(git rev-parse HEAD)" + } | Tee-Object -Append -FilePath $OutputFile + } + + Write-Verbose "Running command: $TestCmd" + Invoke-Expression $TestCmd | Tee-Object -Append -FilePath $OutputFile + + if ( $OutputCmd -ne '' -and $OutputFile -ne 'nul' ) { + $oc = "$OutputCmd $OutputFile" + Write-Verbose "Running command: $oc" + Invoke-Expression $oc + } + +} \ No newline at end of file diff --git a/test/cri-containerd/clone_test.go b/test/cri-containerd/clone_test.go index 1da12e4903..44c0e292ec 100644 --- a/test/cri-containerd/clone_test.go +++ b/test/cri-containerd/clone_test.go @@ -14,7 +14,7 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -240,7 +240,7 @@ func cleanupContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx con // cloned container from that template. func Test_CloneContainer_WCOW(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) - testutilities.RequiresBuild(t, osversion.V20H2) + require.Build(t, osversion.V20H2) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -265,7 +265,7 @@ func Test_CloneContainer_WCOW(t *testing.T) { // A test for creating multiple clones(3 clones) from one template container. func Test_MultiplClonedContainers_WCOW(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) - testutilities.RequiresBuild(t, osversion.V20H2) + require.Build(t, osversion.V20H2) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -302,7 +302,7 @@ func Test_MultiplClonedContainers_WCOW(t *testing.T) { // container. func Test_NormalContainerInClonedPod_WCOW(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) - testutilities.RequiresBuild(t, osversion.V20H2) + require.Build(t, osversion.V20H2) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -342,7 +342,7 @@ func Test_NormalContainerInClonedPod_WCOW(t *testing.T) { // of those pods. func Test_CloneContainersWithClonedPodPool_WCOW(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) - testutilities.RequiresBuild(t, osversion.V20H2) + require.Build(t, osversion.V20H2) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -390,7 +390,7 @@ func Test_CloneContainersWithClonedPodPool_WCOW(t *testing.T) { func Test_ClonedContainerRunningAfterDeletingTemplate(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) - testutilities.RequiresBuild(t, osversion.V20H2) + require.Build(t, osversion.V20H2) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -424,7 +424,7 @@ func Test_ClonedContainerRunningAfterDeletingTemplate(t *testing.T) { // can be made from each of them simultaneously. func Test_MultipleTemplateAndClones_WCOW(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) - testutilities.RequiresBuild(t, osversion.V20H2) + require.Build(t, osversion.V20H2) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -467,7 +467,7 @@ func Test_MultipleTemplateAndClones_WCOW(t *testing.T) { // and verifies that the request correctly fails with an error. func Test_VerifyCloneAndTemplateConfig(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) - testutilities.RequiresBuild(t, osversion.V20H2) + require.Build(t, osversion.V20H2) ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/test/cri-containerd/container_downlevel_test.go b/test/cri-containerd/container_downlevel_test.go index 6bee88adf6..d9f5e748bb 100644 --- a/test/cri-containerd/container_downlevel_test.go +++ b/test/cri-containerd/container_downlevel_test.go @@ -7,13 +7,13 @@ import ( "testing" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) func Test_CreateContainer_DownLevel_WCOW_Hypervisor(t *testing.T) { requireFeatures(t, featureWCOWHypervisor) - testutilities.RequiresBuild(t, osversion.V19H1) + require.Build(t, osversion.V19H1) pullRequiredImages(t, []string{imageWindowsNanoserver17763}) diff --git a/test/cri-containerd/container_layers_packing_test.go b/test/cri-containerd/container_layers_packing_test.go index 89a64bbb02..01043311bd 100644 --- a/test/cri-containerd/container_layers_packing_test.go +++ b/test/cri-containerd/container_layers_packing_test.go @@ -11,7 +11,7 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" ) const ( @@ -38,7 +38,7 @@ func validateTargets(ctx context.Context, t *testing.T, deviceNumber int, podID } func Test_Container_Layer_Packing_On_VPMem(t *testing.T) { - testutilities.RequiresBuild(t, osversion.V19H1) + require.Build(t, osversion.V19H1) client := newTestRuntimeClient(t) ctx, cancel := context.WithCancel(context.Background()) @@ -93,7 +93,7 @@ func Test_Container_Layer_Packing_On_VPMem(t *testing.T) { } func Test_Many_Container_Layers_Supported_On_VPMem(t *testing.T) { - testutilities.RequiresBuild(t, osversion.V19H1) + require.Build(t, osversion.V19H1) client := newTestRuntimeClient(t) ctx, cancel := context.WithCancel(context.Background()) @@ -124,7 +124,7 @@ func Test_Many_Container_Layers_Supported_On_VPMem(t *testing.T) { } func Test_Annotation_Disable_Multi_Mapping(t *testing.T) { - testutilities.RequiresBuild(t, osversion.V19H1) + require.Build(t, osversion.V19H1) client := newTestRuntimeClient(t) ctx, cancel := context.WithCancel(context.Background()) diff --git a/test/cri-containerd/container_update_test.go b/test/cri-containerd/container_update_test.go index 1218183163..6337cae20f 100644 --- a/test/cri-containerd/container_update_test.go +++ b/test/cri-containerd/container_update_test.go @@ -11,7 +11,7 @@ import ( "github.com/Microsoft/hcsshim/internal/memory" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -33,7 +33,7 @@ func calculateJobCPURate(hostProcs uint32, processorCount uint32) uint32 { } func Test_Container_UpdateResources_CPUShare(t *testing.T) { - testutilities.RequiresBuild(t, osversion.V20H2) + require.Build(t, osversion.V20H2) type config struct { name string requiredFeatures []string diff --git a/test/cri-containerd/createcontainer_test.go b/test/cri-containerd/createcontainer_test.go index b05b6a1b50..81f7d0194a 100644 --- a/test/cri-containerd/createcontainer_test.go +++ b/test/cri-containerd/createcontainer_test.go @@ -14,7 +14,7 @@ import ( "github.com/Microsoft/hcsshim/internal/memory" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -844,7 +844,7 @@ func Test_CreateContainer_CPUShares_LCOW(t *testing.T) { func Test_CreateContainer_Mount_File_LCOW(t *testing.T) { requireFeatures(t, featureLCOW) - testutilities.RequiresBuild(t, osversion.V19H1) + require.Build(t, osversion.V19H1) pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) @@ -889,7 +889,7 @@ func Test_CreateContainer_Mount_File_LCOW(t *testing.T) { func Test_CreateContainer_Mount_ReadOnlyFile_LCOW(t *testing.T) { requireFeatures(t, featureLCOW) - testutilities.RequiresBuild(t, osversion.V19H1) + require.Build(t, osversion.V19H1) pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) diff --git a/test/cri-containerd/jobcontainer_test.go b/test/cri-containerd/jobcontainer_test.go index bfbe3ea2f7..17405dcf4d 100644 --- a/test/cri-containerd/jobcontainer_test.go +++ b/test/cri-containerd/jobcontainer_test.go @@ -15,12 +15,13 @@ import ( "time" "github.com/Microsoft/go-winio/vhd" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/internal/winapi" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + "github.com/Microsoft/hcsshim/test/internal/require" ) func getJobContainerPodRequestWCOW(t *testing.T) *runtime.RunPodSandboxRequest { @@ -417,7 +418,7 @@ func Test_RunContainer_HostVolumes_JobContainer_WCOW(t *testing.T) { func Test_RunContainer_JobContainer_VolumeMount(t *testing.T) { client := newTestRuntimeClient(t) - testutilities.RequiresExactBuild(t, osversion.RS5) + require.ExactBuild(t, osversion.RS5) dir := t.TempDir() @@ -600,7 +601,7 @@ func Test_RunContainer_JobContainer_Environment(t *testing.T) { func Test_RunContainer_WorkingDirectory_JobContainer_WCOW(t *testing.T) { client := newTestRuntimeClient(t) - testutilities.RequiresExactBuild(t, osversion.RS5) + require.ExactBuild(t, osversion.RS5) type config struct { name string @@ -736,7 +737,7 @@ func Test_DoubleQuoting_JobContainer_WCOW(t *testing.T) { // Test that mounts show up at the expected destination if the host supports file binding. func Test_BindSupport_JobContainer_WCOW(t *testing.T) { requireFeatures(t, featureWCOWProcess, featureHostProcess) - testutilities.RequiresBuild(t, osversion.V20H1) + require.Build(t, osversion.V20H1) pullRequiredImages(t, []string{imageWindowsNanoserver}) client := newTestRuntimeClient(t) @@ -792,7 +793,7 @@ func Test_BindSupport_JobContainer_WCOW(t *testing.T) { // Test that mounts are unique per container even if the same container path is used. func Test_BindSupport_MultipleContainers_JobContainer_WCOW(t *testing.T) { requireFeatures(t, featureWCOWProcess, featureHostProcess) - testutilities.RequiresBuild(t, osversion.V20H1) + require.Build(t, osversion.V20H1) pullRequiredImages(t, []string{imageWindowsNanoserver}) client := newTestRuntimeClient(t) diff --git a/test/cri-containerd/main_test.go b/test/cri-containerd/main_test.go index e1b445f25f..90fed1f476 100644 --- a/test/cri-containerd/main_test.go +++ b/test/cri-containerd/main_test.go @@ -14,8 +14,7 @@ import ( "time" "github.com/Microsoft/hcsshim/osversion" - _ "github.com/Microsoft/hcsshim/test/functional/manifest" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + testutilities "github.com/Microsoft/hcsshim/test/internal" "github.com/containerd/containerd" eventtypes "github.com/containerd/containerd/api/events" eventsapi "github.com/containerd/containerd/api/services/events/v1" @@ -25,6 +24,9 @@ import ( "github.com/gogo/protobuf/types" "google.golang.org/grpc" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + + "github.com/Microsoft/hcsshim/test/internal/constants" + _ "github.com/Microsoft/hcsshim/test/internal/manifest" ) const ( @@ -72,12 +74,12 @@ const ( var ( imageWindowsNanoserver = getWindowsNanoserverImage(osversion.Build()) imageWindowsServercore = getWindowsServerCoreImage(osversion.Build()) - imageWindowsNanoserver17763 = getWindowsNanoserverImage(osversion.RS5) - imageWindowsNanoserver18362 = getWindowsNanoserverImage(osversion.V19H1) - imageWindowsNanoserver19041 = getWindowsNanoserverImage(osversion.V20H1) - imageWindowsServercore17763 = getWindowsServerCoreImage(osversion.RS5) - imageWindowsServercore18362 = getWindowsServerCoreImage(osversion.V19H1) - imageWindowsServercore19041 = getWindowsServerCoreImage(osversion.V20H1) + imageWindowsNanoserver17763 = constants.ImageWindowsNanoserver1809 + imageWindowsNanoserver18362 = constants.ImageWindowsNanoserver1903 + imageWindowsNanoserver19041 = constants.ImageWindowsNanoserver2004 + imageWindowsServercore17763 = constants.ImageWindowsServercore1809 + imageWindowsServercore18362 = constants.ImageWindowsServercore1903 + imageWindowsServercore19041 = constants.ImageWindowsServercore2004 ) // Flags @@ -161,55 +163,19 @@ func requireBinary(t *testing.T, binary string) string { } func getWindowsNanoserverImage(build uint16) string { - switch build { - case osversion.RS5: - return "mcr.microsoft.com/windows/nanoserver:1809" - case osversion.V19H1: - return "mcr.microsoft.com/windows/nanoserver:1903" - case osversion.V19H2: - return "mcr.microsoft.com/windows/nanoserver:1909" - case osversion.V20H1: - return "mcr.microsoft.com/windows/nanoserver:2004" - case osversion.V20H2: - return "mcr.microsoft.com/windows/nanoserver:2009" - case osversion.V21H2Server: - return "mcr.microsoft.com/windows/nanoserver:ltsc2022" - default: - // Due to some efforts in improving down-level compatibility for Windows containers (see - // https://techcommunity.microsoft.com/t5/containers/windows-server-2022-and-beyond-for-containers/ba-p/2712487) - // the ltsc2022 image should continue to work on builds ws2022 and onwards. With this in mind, - // if there's no mapping for the host build, just use the Windows Server 2022 image. - if build > osversion.V21H2Server { - return "mcr.microsoft.com/windows/nanoserver:ltsc2022" - } - panic("unsupported build") + tag, err := constants.ImageFromBuild(build) + if err != nil { + panic(err) } + return constants.NanoserverImage(tag) } func getWindowsServerCoreImage(build uint16) string { - switch build { - case osversion.RS5: - return "mcr.microsoft.com/windows/servercore:1809" - case osversion.V19H1: - return "mcr.microsoft.com/windows/servercore:1903" - case osversion.V19H2: - return "mcr.microsoft.com/windows/servercore:1909" - case osversion.V20H1: - return "mcr.microsoft.com/windows/servercore:2004" - case osversion.V20H2: - return "mcr.microsoft.com/windows/servercore:2009" - case osversion.V21H2Server: - return "mcr.microsoft.com/windows/servercore:ltsc2022" - default: - // Due to some efforts in improving down-level compatibility for Windows containers (see - // https://techcommunity.microsoft.com/t5/containers/windows-server-2022-and-beyond-for-containers/ba-p/2712487) - // the ltsc2022 image should continue to work on builds ws2022 and onwards. With this in mind, - // if there's no mapping for the host build, just use the Windows Server 2022 image. - if build > osversion.V21H2Server { - return "mcr.microsoft.com/windows/servercore:ltsc2022" - } - panic("unsupported build") + tag, err := constants.ImageFromBuild(build) + if err != nil { + panic(err) } + return constants.ServercoreImage(tag) } func createGRPCConn(ctx context.Context) (*grpc.ClientConn, error) { diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index 515ab9765e..fa795926ab 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -22,7 +22,8 @@ import ( "github.com/Microsoft/hcsshim/internal/shimdiag" "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/annotations" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" + testuvm "github.com/Microsoft/hcsshim/test/internal/uvm" "github.com/containerd/containerd/log" "golang.org/x/sys/windows" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" @@ -982,7 +983,7 @@ func Test_RunPodSandbox_Mount_SandboxDir_NoShare_WCOW(t *testing.T) { } func Test_RunPodSandbox_CPUGroup(t *testing.T) { - testutilities.RequiresBuild(t, osversion.V21H1) + require.Build(t, osversion.V21H1) ctx := context.Background() presentID := "FA22A12C-36B3-486D-A3E9-BC526C2B450B" @@ -1062,7 +1063,7 @@ func createExt4VHD(ctx context.Context, t *testing.T, path string) { log.L.Logger.SetOutput(io.Discard) defer log.L.Logger.SetOutput(origLogOut) } - uvm := testutilities.CreateLCOWUVM(ctx, t, t.Name()+"-createExt4VHD") + uvm := testuvm.CreateAndStartLCOW(ctx, t, t.Name()+"-createExt4VHD") defer uvm.Close() if err := lcow.CreateScratch(ctx, uvm, path, 2, ""); err != nil { diff --git a/test/functional/lcow_bench_test.go b/test/functional/lcow_bench_test.go new file mode 100644 index 0000000000..263fa2d870 --- /dev/null +++ b/test/functional/lcow_bench_test.go @@ -0,0 +1,101 @@ +//go:build windows && functional +// +build windows,functional + +package functional + +import ( + "context" + "testing" + + "github.com/Microsoft/hcsshim/osversion" + + "github.com/Microsoft/hcsshim/test/internal/require" + "github.com/Microsoft/hcsshim/test/internal/uvm" +) + +func BenchmarkLCOW_UVM_Create(b *testing.B) { + requireFeatures(b, featureLCOW) + require.Build(b, osversion.RS5) + + ctx := context.Background() + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + opts := defaultLCOWOptions(b) + + b.StartTimer() + vm := uvm.CreateLCOW(ctx, b, opts) + b.StopTimer() + + // vm.Close() hangs unless the vm was started + cleanup := uvm.Start(ctx, b, vm) + cleanup() + } +} + +func BenchmarkLCOW_UVM_Start(b *testing.B) { + requireFeatures(b, featureLCOW) + require.Build(b, osversion.RS5) + + ctx := context.Background() + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + vm := uvm.CreateLCOW(ctx, b, defaultLCOWOptions(b)) + + b.StartTimer() + if err := vm.Start(ctx); err != nil { + b.Fatalf("could not start UVM: %v", err) + } + b.StopTimer() + + vm.Close() + } +} + +func BenchmarkLCOW_UVM_Kill(b *testing.B) { + requireFeatures(b, featureLCOW) + require.Build(b, osversion.RS5) + + ctx := context.Background() + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + vm := uvm.CreateLCOW(ctx, b, defaultLCOWOptions(b)) + cleanup := uvm.Start(ctx, b, vm) + + b.StartTimer() + uvm.Kill(ctx, b, vm) + if err := vm.Wait(); err != nil { + b.Fatalf("could not kill uvm %q: %v", vm.ID(), err) + } + b.StopTimer() + + cleanup() + } +} + +func BenchmarkLCOW_UVM_Close(b *testing.B) { + requireFeatures(b, featureLCOW) + require.Build(b, osversion.RS5) + + ctx := context.Background() + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + vm := uvm.CreateLCOW(ctx, b, defaultLCOWOptions(b)) + cleanup := uvm.Start(ctx, b, vm) + + b.StartTimer() + if err := vm.Close(); err != nil { + b.Fatalf("could not kill uvm %q: %v", vm.ID(), err) + } + b.StopTimer() + + cleanup() + } +} diff --git a/test/functional/lcow_container_bench_test.go b/test/functional/lcow_container_bench_test.go new file mode 100644 index 0000000000..7c6e3ec8ad --- /dev/null +++ b/test/functional/lcow_container_bench_test.go @@ -0,0 +1,216 @@ +//go:build windows && functional +// +build windows,functional + +package functional + +import ( + "context" + "testing" + + ctrdoci "github.com/containerd/containerd/oci" + cri_util "github.com/containerd/containerd/pkg/cri/util" + + "github.com/Microsoft/hcsshim/internal/hcsoci" + "github.com/Microsoft/hcsshim/internal/resources" + "github.com/Microsoft/hcsshim/osversion" + + "github.com/Microsoft/hcsshim/test/internal/cmd" + "github.com/Microsoft/hcsshim/test/internal/constants" + "github.com/Microsoft/hcsshim/test/internal/container" + "github.com/Microsoft/hcsshim/test/internal/layers" + "github.com/Microsoft/hcsshim/test/internal/oci" + "github.com/Microsoft/hcsshim/test/internal/require" + "github.com/Microsoft/hcsshim/test/internal/uvm" +) + +func BenchmarkLCOW_Container(b *testing.B) { + requireFeatures(b, featureLCOW, featureContainer) + require.Build(b, osversion.RS5) + + ctx, _, client := newContainerdClient(context.Background(), b) + ls := layers.FromImage(ctx, b, client, constants.ImageLinuxAlpineLatest, + constants.PlatformLinux, constants.SnapshotterLinux) + + // Create a new uvm per benchmark in case any left over state lingers + + b.Run("Create", func(b *testing.B) { + vm := uvm.CreateAndStartLCOWFromOpts(ctx, b, defaultLCOWOptions(b)) + uvm.SetSecurityPolicy(ctx, b, vm, "") + cache := layers.CacheFile(ctx, b, "") + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, _ := layers.ScratchSpace(ctx, b, vm, "", "", cache) + spec := oci.CreateLinuxSpec(ctx, b, id, + oci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", "true"), + oci.WithWindowsLayerFolders(append(ls, scratch)))...) + + co := &hcsoci.CreateOptions{ + ID: id, + HostingSystem: vm, + Owner: hcsOwner, + Spec: spec, + // dont create a network namespace on the host side + NetworkNamespace: "", + } + + b.StartTimer() + c, r, err := hcsoci.CreateContainer(ctx, co) + if err != nil { + b.Fatalf("could not create container %q: %v", co.ID, err) + } + b.StopTimer() + + // container creations launches gorountines on the guest that do + // not finish until the init process has terminated. + // so start the container, then clean everything up + init := container.Start(ctx, b, c, nil) + cmd.WaitExitCode(ctx, b, init, 0) + + container.Kill(ctx, b, c) + container.Wait(ctx, b, c) + if err := resources.ReleaseResources(ctx, r, vm, true); err != nil { + b.Errorf("failed to release container resources: %v", err) + } + if err := c.Close(); err != nil { + b.Errorf("could not close container %q: %v", c.ID(), err) + } + } + }) + + b.Run("Start", func(b *testing.B) { + vm := uvm.CreateAndStartLCOWFromOpts(ctx, b, defaultLCOWOptions(b)) + uvm.SetSecurityPolicy(ctx, b, vm, "") + cache := layers.CacheFile(ctx, b, "") + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, _ := layers.ScratchSpace(ctx, b, vm, "", "", cache) + spec := oci.CreateLinuxSpec(ctx, b, id, + oci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", "true"), + oci.WithWindowsLayerFolders(append(ls, scratch)))...) + + c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) + + b.StartTimer() + if err := c.Start(ctx); err != nil { + b.Fatalf("could not start %q: %v", c.ID(), err) + } + b.StopTimer() + + init := cmd.Create(ctx, b, c, nil, nil) + cmd.Start(ctx, b, init) + cmd.WaitExitCode(ctx, b, init, 0) + + container.Kill(ctx, b, c) + container.Wait(ctx, b, c) + cleanup() + } + }) + + b.Run("InitExec", func(b *testing.B) { + vm := uvm.CreateAndStartLCOWFromOpts(ctx, b, defaultLCOWOptions(b)) + uvm.SetSecurityPolicy(ctx, b, vm, "") + cache := layers.CacheFile(ctx, b, "") + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, _ := layers.ScratchSpace(ctx, b, vm, "", "", cache) + spec := oci.CreateLinuxSpec(ctx, b, id, + oci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), + oci.WithWindowsLayerFolders(append(ls, scratch)))...) + + c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) + if err := c.Start(ctx); err != nil { + b.Fatalf("could not start %q: %v", c.ID(), err) + } + init := cmd.Create(ctx, b, c, nil, nil) + + b.StartTimer() + cmd.Start(ctx, b, init) + b.StopTimer() + + cmd.Kill(ctx, b, init) + cmd.WaitExitCode(ctx, b, init, 137) + + container.Kill(ctx, b, c) + container.Wait(ctx, b, c) + cleanup() + } + }) + + b.Run("InitExecKill", func(b *testing.B) { + vm := uvm.CreateAndStartLCOWFromOpts(ctx, b, defaultLCOWOptions(b)) + uvm.SetSecurityPolicy(ctx, b, vm, "") + cache := layers.CacheFile(ctx, b, "") + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, _ := layers.ScratchSpace(ctx, b, vm, "", "", cache) + spec := oci.CreateLinuxSpec(ctx, b, id, + oci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), + oci.WithWindowsLayerFolders(append(ls, scratch)))...) + + c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) + init := container.Start(ctx, b, c, nil) + + b.StartTimer() + cmd.Kill(ctx, b, init) + cmd.WaitExitCode(ctx, b, init, 137) + b.StopTimer() + + container.Kill(ctx, b, c) + container.Wait(ctx, b, c) + cleanup() + } + }) + + b.Run("ContainerKill", func(b *testing.B) { + vm := uvm.CreateAndStartLCOWFromOpts(ctx, b, defaultLCOWOptions(b)) + uvm.SetSecurityPolicy(ctx, b, vm, "") + cache := layers.CacheFile(ctx, b, "") + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, _ := layers.ScratchSpace(ctx, b, vm, "", "", cache) + spec := oci.CreateLinuxSpec(ctx, b, id, + oci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", "true"), + oci.WithWindowsLayerFolders(append(ls, scratch)))...) + + c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) + + // (c container).Wait() waits until the gc receives a notification message from + // the guest (via the bridge) that the container exited. + // The Linux guest starts a goroutine to send that notification (bridge_v2.go:createContainerV2) + // That goroutine, in turn, waits on the init process, which does not unblock until it has + // been waited on (usually via a WaitForProcess request) and had its exit code read + // (hcsv2/process.go:(*containerProcess).Wait). + // + // So ... to test container kill and wait times, we need to first start and wait on the init process + init := container.Start(ctx, b, c, nil) + cmd.WaitExitCode(ctx, b, init, 0) + + b.StartTimer() + container.Kill(ctx, b, c) + container.Wait(ctx, b, c) + b.StopTimer() + + cleanup() + } + }) +} diff --git a/test/functional/lcow_container_test.go b/test/functional/lcow_container_test.go new file mode 100644 index 0000000000..c4b993b287 --- /dev/null +++ b/test/functional/lcow_container_test.go @@ -0,0 +1,177 @@ +//go:build windows && functional +// +build windows,functional + +package functional + +import ( + "context" + "strings" + "testing" + + ctrdoci "github.com/containerd/containerd/oci" + + "github.com/Microsoft/hcsshim/osversion" + + "github.com/Microsoft/hcsshim/test/internal/cmd" + "github.com/Microsoft/hcsshim/test/internal/constants" + "github.com/Microsoft/hcsshim/test/internal/container" + "github.com/Microsoft/hcsshim/test/internal/layers" + "github.com/Microsoft/hcsshim/test/internal/oci" + "github.com/Microsoft/hcsshim/test/internal/require" + "github.com/Microsoft/hcsshim/test/internal/uvm" +) + +func TestLCOW_ContainerLifecycle(t *testing.T) { + requireFeatures(t, featureLCOW, featureContainer) + require.Build(t, osversion.RS5) + + ctx, _, client := newContainerdClient(context.Background(), t) + ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest, + constants.PlatformLinux, constants.SnapshotterLinux) + + opts := defaultLCOWOptions(t) + vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) + uvm.SetSecurityPolicy(ctx, t, vm, "") + + scratch, _ := layers.ScratchSpace(ctx, t, vm, "", "", "") + + spec := oci.CreateLinuxSpec(ctx, t, t.Name(), + oci.DefaultLinuxSpecOpts("", + ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), + oci.WithWindowsLayerFolders(append(ls, scratch)))...) + + c, _, cleanup := container.Create(ctx, t, vm, spec, t.Name(), hcsOwner) + t.Cleanup(cleanup) + + init := container.Start(ctx, t, c, nil) + t.Cleanup(func() { + container.Kill(ctx, t, c) + container.Wait(ctx, t, c) + }) + cmd.Kill(ctx, t, init) + if e := cmd.Wait(ctx, t, init); e != 137 { + t.Errorf("got exit code %d, wanted %d", e, 137) + } +} + +var ioTests = []struct { + name string + args []string + in string + want string +}{ + { + name: "true", + args: []string{"/bin/sh", "-c", "true"}, + want: "", + }, + { + name: "echo", + args: []string{"/bin/sh", "-c", `echo -n "hi y'all"`}, + want: "hi y'all", + }, + { + name: "tee", + args: []string{"/bin/sh", "-c", "tee"}, + in: "are you copying me?", + want: "are you copying me?", + }, +} + +func TestLCOW_ContainerIO(t *testing.T) { + requireFeatures(t, featureLCOW, featureContainer) + require.Build(t, osversion.RS5) + + ctx, _, client := newContainerdClient(context.Background(), t) + ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest, + constants.PlatformLinux, constants.SnapshotterLinux) + + opts := defaultLCOWOptions(t) + cache := layers.CacheFile(ctx, t, "") + vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) + uvm.SetSecurityPolicy(ctx, t, vm, "") + + for _, tt := range ioTests { + t.Run(tt.name, func(t *testing.T) { + id := strings.ReplaceAll(t.Name(), "/", "") + scratch, _ := layers.ScratchSpace(ctx, t, vm, "", "", cache) + spec := oci.CreateLinuxSpec(ctx, t, id, + oci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs(tt.args...), + oci.WithWindowsLayerFolders(append(ls, scratch)))...) + + c, _, cleanup := container.Create(ctx, t, vm, spec, id, hcsOwner) + t.Cleanup(cleanup) + + io := cmd.NewBufferedIO() + if tt.in != "" { + io = cmd.NewBufferedIOFromString(tt.in) + } + init := container.Start(ctx, t, c, io) + + t.Cleanup(func() { + container.Kill(ctx, t, c) + container.Wait(ctx, t, c) + }) + + if e := cmd.Wait(ctx, t, init); e != 0 { + t.Fatalf("got exit code %d, wanted %d", e, 0) + } + + io.TestOutput(t, tt.want, nil) + }) + } +} + +func TestLCOW_ContainerExec(t *testing.T) { + requireFeatures(t, featureLCOW, featureContainer) + require.Build(t, osversion.RS5) + + ctx, _, client := newContainerdClient(context.Background(), t) + ls := layers.FromImage(ctx, t, client, constants.ImageLinuxAlpineLatest, + constants.PlatformLinux, constants.SnapshotterLinux) + + opts := defaultLCOWOptions(t) + vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) + uvm.SetSecurityPolicy(ctx, t, vm, "") + + id := strings.ReplaceAll(t.Name(), "/", "") + scratch, _ := layers.ScratchSpace(ctx, t, vm, "", "", "") + spec := oci.CreateLinuxSpec(ctx, t, id, + oci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), + oci.WithWindowsLayerFolders(append(ls, scratch)))...) + + c, _, cleanup := container.Create(ctx, t, vm, spec, id, hcsOwner) + t.Cleanup(cleanup) + init := container.Start(ctx, t, c, nil) + t.Cleanup(func() { + cmd.Kill(ctx, t, init) + cmd.Wait(ctx, t, init) + container.Kill(ctx, t, c) + container.Wait(ctx, t, c) + }) + + for _, tt := range ioTests { + t.Run(tt.name, func(t *testing.T) { + ps := oci.CreateLinuxSpec(ctx, t, id, + oci.DefaultLinuxSpecOpts(id, + // oci.WithTTY, + ctrdoci.WithDefaultPathEnv, + ctrdoci.WithProcessArgs(tt.args...))..., + ).Process + io := cmd.NewBufferedIO() + if tt.in != "" { + io = cmd.NewBufferedIOFromString(tt.in) + } + p := cmd.Create(ctx, t, c, ps, io) + cmd.Start(ctx, t, p) + + if e := cmd.Wait(ctx, t, p); e != 0 { + t.Fatalf("got exit code %d, wanted %d", e, 0) + } + + io.TestOutput(t, tt.want, nil) + }) + } +} diff --git a/test/functional/lcow_test.go b/test/functional/lcow_test.go index d788e4e0fb..fadc462e15 100644 --- a/test/functional/lcow_test.go +++ b/test/functional/lcow_test.go @@ -1,5 +1,5 @@ -//go:build functional || lcow -// +build functional lcow +//go:build windows && functional +// +build windows,functional package functional @@ -7,12 +7,13 @@ import ( "bytes" "context" "fmt" - "os/exec" "path/filepath" "strings" "testing" "time" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/cow" "github.com/Microsoft/hcsshim/internal/hcsoci" @@ -20,50 +21,87 @@ import ( "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + + testutilities "github.com/Microsoft/hcsshim/test/internal" + testcmd "github.com/Microsoft/hcsshim/test/internal/cmd" + "github.com/Microsoft/hcsshim/test/internal/layers" + "github.com/Microsoft/hcsshim/test/internal/require" + testuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) -// TestLCOWUVMNoSCSINoVPMemInitrd starts an LCOW utility VM without a SCSI controller and +// test if waiting after creating (but not starting) an LCOW uVM returns +func TestLCOW_UVMCreateWait(t *testing.T) { + t.Skip("closing a created-but-not-started uVM hangs indefinitely") + requireFeatures(t, featureLCOW) + require.Build(t, osversion.RS5) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + + vm := testuvm.CreateLCOW(ctx, t, defaultLCOWOptions(t)) + testuvm.Close(ctx, t, vm) +} + +// TestLCOW_UVMNoSCSINoVPMemInitrd starts an LCOW utility VM without a SCSI controller and // no VPMem device. Uses initrd. -func TestLCOWUVMNoSCSINoVPMemInitrd(t *testing.T) { - opts := uvm.NewDefaultOptionsLCOW(t.Name(), "") +func TestLCOW_UVMNoSCSINoVPMemInitrd(t *testing.T) { + requireFeatures(t, featureLCOW) + + opts := defaultLCOWOptions(t) opts.SCSIControllerCount = 0 opts.VPMemDeviceCount = 0 opts.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd opts.RootFSFile = uvm.InitrdFile + opts.KernelDirect = false testLCOWUVMNoSCSISingleVPMem(t, opts, fmt.Sprintf("Command line: initrd=/%s", opts.RootFSFile)) } -// TestLCOWUVMNoSCSISingleVPMemVHD starts an LCOW utility VM without a SCSI controller and +// TestLCOW_UVMNoSCSISingleVPMemVHD starts an LCOW utility VM without a SCSI controller and // only a single VPMem device. Uses VPMEM VHD -func TestLCOWUVMNoSCSISingleVPMemVHD(t *testing.T) { - opts := uvm.NewDefaultOptionsLCOW(t.Name(), "") +func TestLCOW_UVMNoSCSISingleVPMemVHD(t *testing.T) { + requireFeatures(t, featureLCOW) + + opts := defaultLCOWOptions(t) opts.SCSIControllerCount = 0 opts.VPMemDeviceCount = 1 opts.PreferredRootFSType = uvm.PreferredRootFSTypeVHD opts.RootFSFile = uvm.VhdFile - testLCOWUVMNoSCSISingleVPMem(t, opts, `Command line: root=/dev/pmem0 init=/init`) + testLCOWUVMNoSCSISingleVPMem(t, opts, `Command line: root=/dev/pmem0`, `init=/init`) } -func testLCOWUVMNoSCSISingleVPMem(t *testing.T, opts *uvm.OptionsLCOW, expected string) { - testutilities.RequiresBuild(t, osversion.RS5) - lcowUVM := testutilities.CreateLCOWUVMFromOpts(context.Background(), t, opts) +func testLCOWUVMNoSCSISingleVPMem(t *testing.T, opts *uvm.OptionsLCOW, expected ...string) { + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW) + ctx := context.Background() + + lcowUVM := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) defer lcowUVM.Close() - out, err := exec.Command(`hcsdiag`, `exec`, `-uvm`, lcowUVM.ID(), `dmesg`).Output() // TODO: Move the CreateProcess. + + io := testcmd.NewBufferedIO() + // c := cmd.Command(lcowUVM, "dmesg") + c := testcmd.Create(ctx, t, lcowUVM, &specs.Process{Args: []string{"dmesg"}}, io) + testcmd.Run(ctx, t, c) + + out, err := io.Output() + if err != nil { - t.Fatal(string(err.(*exec.ExitError).Stderr)) + t.Fatalf("uvm exec failed with: %s", err) } - if !strings.Contains(string(out), expected) { - t.Fatalf("Expected dmesg output to have %q: %s", expected, string(out)) + + for _, s := range expected { + if !strings.Contains(out, s) { + t.Fatalf("Expected dmesg output to have %q: %s", s, out) + } } } -// TestLCOWTimeUVMStartVHD starts/terminates a utility VM booting from VPMem- +// TestLCOW_TimeUVMStartVHD starts/terminates a utility VM booting from VPMem- // attached root filesystem a number of times. -func TestLCOWTimeUVMStartVHD(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) +func TestLCOW_TimeUVMStartVHD(t *testing.T) { + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW) testLCOWTimeUVMStart(t, false, uvm.PreferredRootFSTypeVHD) } @@ -71,16 +109,18 @@ func TestLCOWTimeUVMStartVHD(t *testing.T) { // TestLCOWUVMStart_KernelDirect_VHD starts/terminates a utility VM booting from // VPMem- attached root filesystem a number of times starting from the Linux // Kernel directly and skipping EFI. -func TestLCOWUVMStart_KernelDirect_VHD(t *testing.T) { - testutilities.RequiresBuild(t, 18286) +func TestLCOW_UVMStart_KernelDirect_VHD(t *testing.T) { + require.Build(t, 18286) + requireFeatures(t, featureLCOW) testLCOWTimeUVMStart(t, true, uvm.PreferredRootFSTypeVHD) } // TestLCOWTimeUVMStartInitRD starts/terminates a utility VM booting from initrd- // attached root file system a number of times. -func TestLCOWTimeUVMStartInitRD(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) +func TestLCOW_TimeUVMStartInitRD(t *testing.T) { + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW) testLCOWTimeUVMStart(t, false, uvm.PreferredRootFSTypeInitRd) } @@ -88,15 +128,18 @@ func TestLCOWTimeUVMStartInitRD(t *testing.T) { // TestLCOWUVMStart_KernelDirect_InitRd starts/terminates a utility VM booting // from initrd- attached root file system a number of times starting from the // Linux Kernel directly and skipping EFI. -func TestLCOWUVMStart_KernelDirect_InitRd(t *testing.T) { - testutilities.RequiresBuild(t, 18286) +func TestLCOW_UVMStart_KernelDirect_InitRd(t *testing.T) { + require.Build(t, 18286) + requireFeatures(t, featureLCOW) testLCOWTimeUVMStart(t, true, uvm.PreferredRootFSTypeInitRd) } func testLCOWTimeUVMStart(t *testing.T, kernelDirect bool, rfsType uvm.PreferredRootFSType) { + requireFeatures(t, featureLCOW) + for i := 0; i < 3; i++ { - opts := uvm.NewDefaultOptionsLCOW(t.Name(), "") + opts := defaultLCOWOptions(t) opts.KernelDirect = kernelDirect opts.VPMemDeviceCount = 32 opts.PreferredRootFSType = rfsType @@ -107,15 +150,18 @@ func testLCOWTimeUVMStart(t *testing.T, kernelDirect bool, rfsType uvm.Preferred opts.RootFSFile = uvm.VhdFile } - lcowUVM := testutilities.CreateLCOWUVMFromOpts(context.Background(), t, opts) + lcowUVM := testuvm.CreateAndStartLCOWFromOpts(context.Background(), t, opts) lcowUVM.Close() } } func TestLCOWSimplePodScenario(t *testing.T) { t.Skip("Doesn't work quite yet") - testutilities.RequiresBuild(t, osversion.RS5) - alpineLayers := testutilities.LayerFolders(t, "alpine") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW, featureContainer) + + alpineLayers := layers.LayerFolders(t, "alpine") cacheDir := t.TempDir() cacheFile := filepath.Join(cacheDir, "cache.vhdx") @@ -132,7 +178,7 @@ func TestLCOWSimplePodScenario(t *testing.T) { c2ScratchDir := t.TempDir() c2ScratchFile := filepath.Join(c2ScratchDir, "sandbox.vhdx") - lcowUVM := testutilities.CreateLCOWUVM(context.Background(), t, "uvm") + lcowUVM := testuvm.CreateAndStartLCOW(context.Background(), t, "uvm") defer lcowUVM.Close() // Populate the cache and generate the scratch file for /tmp/scratch diff --git a/test/functional/main_test.go b/test/functional/main_test.go new file mode 100644 index 0000000000..b3ddf9af92 --- /dev/null +++ b/test/functional/main_test.go @@ -0,0 +1,175 @@ +//go:build windows && functional + +// This package tests the internals of hcsshim, independent of the OCI interfaces it exposes +// and the container runtime (or CRI API) that normally would be communicating with the shim. +// +// While these tests may overlap with CRI/containerd or shim tests, they exercise `internal/*` +// code paths and primitives directly. +package functional + +import ( + "context" + "flag" + "log" + "os" + "os/exec" + "regexp" + "strconv" + "testing" + "time" + + "github.com/containerd/containerd" + "github.com/sirupsen/logrus" + + "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/hcsoci" + "github.com/Microsoft/hcsshim/internal/resources" + "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/internal/winapi" + + testctrd "github.com/Microsoft/hcsshim/test/internal/containerd" + testflag "github.com/Microsoft/hcsshim/test/internal/flag" + "github.com/Microsoft/hcsshim/test/internal/require" +) + +// owner field for uVMs +const hcsOwner = "hcsshim-functional-tests" + +const ( + featureLCOW = "LCOW" + featureWCOW = "WCOW" + featureContainer = "container" + featureHostProcess = "HostProcess" + featureUVMMem = "UVMMem" + featurePlan9 = "Plan9" + featureSCSI = "SCSI" + featureScratch = "Scratch" + featureVSMB = "vSMB" + featureVPMEM = "vPMEM" +) + +var allFeatures = []string{ + featureLCOW, + featureWCOW, + featureHostProcess, + featureContainer, + featureUVMMem, + featurePlan9, + featureSCSI, + featureScratch, + featureVSMB, + featureVPMEM, +} + +// todo: use a new containerd namespace and then nuke everything in it + +var ( + debug bool + pauseDurationOnCreateContainerFailure time.Duration + + // flags + flagFeatures = testflag.NewFeatureFlag(allFeatures) + flagContainerdAddress = flag.String("ctr-address", "tcp://127.0.0.1:2376", "Address for containerd's GRPC server") + flagContainerdNamespace = flag.String("ctr-namespace", "k8s.io", "Containerd namespace") + flagCtrExePath = flag.String("ctr-path", `C:\ContainerPlat\ctr.exe`, "Path to ctr.exe") + flagLinuxBootFilesPath = flag.String("linux-bootfiles", + `C:\\ContainerPlat\\LinuxBootFiles`, + "Path to LCOW UVM boot files (rootfs.vhd, initrd.img, kernel, vmlinux)") +) + +func init() { + if !winapi.IsElevated() { + log.Fatal("tests must be run in an elevated context") + } + + if _, ok := os.LookupEnv("HCSSHIM_FUNCTIONAL_TESTS_DEBUG"); ok { + debug = true + } + flag.BoolVar(&debug, "debug", debug, "Set logging level to debug [%HCSSHIM_FUNCTIONAL_TESTS_DEBUG%]") + + // This allows for debugging a utility VM. + if s := os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES"); s != "" { + if t, err := strconv.Atoi(s); err == nil { + pauseDurationOnCreateContainerFailure = time.Duration(t) * time.Minute + } + } + flag.DurationVar(&pauseDurationOnCreateContainerFailure, + "container-creation-failure-pause", + pauseDurationOnCreateContainerFailure, + "The number of minutes to wait after a container creation failure to try again "+ + "[%HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES%]") +} + +func TestMain(m *testing.M) { + flag.Parse() + + lvl := logrus.WarnLevel + if vf := flag.Lookup("test.v"); debug || (vf != nil && vf.Value.String() == strconv.FormatBool(true)) { + lvl = logrus.DebugLevel + } + logrus.SetLevel(lvl) + logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) + logrus.Infof("using features %q", flagFeatures.S.Strings()) + + e := m.Run() + + // close any uVMs that escaped + cmdStr := ` foreach ($vm in Get-ComputeProcess -Owner '` + hcsOwner + `') { Write-Output "uVM $($vm.Id) was left running" ; Stop-ComputeProcess -Force -Id $vm.Id } ` + cmd := exec.Command("powershell", "-NoLogo", " -NonInteractive", "-Command", cmdStr) + o, err := cmd.CombinedOutput() + if err != nil { + logrus.Warningf("could not call %q to clean up remaining uVMs: %v", cmdStr, err) + } else if len(o) > 0 { + logrus.Warningf(string(o)) + } + + os.Exit(e) +} + +func CreateContainerTestWrapper(ctx context.Context, options *hcsoci.CreateOptions) (cow.Container, *resources.Resources, error) { + if pauseDurationOnCreateContainerFailure != 0 { + options.DoNotReleaseResourcesOnFailure = true + } + s, r, err := hcsoci.CreateContainer(ctx, options) + if err != nil { + logrus.Warnf("Test is pausing for %s for debugging CreateContainer failure", pauseDurationOnCreateContainerFailure) + time.Sleep(pauseDurationOnCreateContainerFailure) + _ = resources.ReleaseResources(ctx, r, options.HostingSystem, true) + } + + return s, r, err +} + +func requireFeatures(t testing.TB, features ...string) { + require.Features(t, flagFeatures.S, features...) +} + +func getContainerdOptions() testctrd.ContainerdClientOptions { + return testctrd.ContainerdClientOptions{ + Address: *flagContainerdAddress, + Namespace: *flagContainerdNamespace, + } +} + +func newContainerdClient(ctx context.Context, t testing.TB) (context.Context, context.CancelFunc, *containerd.Client) { + return getContainerdOptions().NewClient(ctx, t) +} + +func defaultLCOWOptions(t testing.TB) *uvm.OptionsLCOW { + opts := uvm.NewDefaultOptionsLCOW(cleanName(t.Name()), "") + opts.BootFilesPath = *flagLinuxBootFilesPath + + return opts +} + +func defaultWCOWOptions(t testing.TB) *uvm.OptionsWCOW { + opts := uvm.NewDefaultOptionsWCOW(cleanName(t.Name()), "") + + return opts +} + +var _nameRegex = regexp.MustCompile(`[\\\/\s]`) + +func cleanName(n string) string { + return _nameRegex.ReplaceAllString(n, "") +} diff --git a/test/functional/manifest/manifest.go b/test/functional/manifest/manifest.go deleted file mode 100644 index 38dc837c6e..0000000000 --- a/test/functional/manifest/manifest.go +++ /dev/null @@ -1,4 +0,0 @@ -package manifest - -// This is so that tests can include the .syso to manifest them to pick up the right Windows build -// TODO: Auto-generation of the .syso through rsrc or similar. diff --git a/test/functional/manifest/rsrc_amd64.syso b/test/functional/manifest/rsrc_amd64.syso deleted file mode 100644 index 0e9857245ba25586643b662700494e852cfe75ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 372470 zcmeEP2Y405+TL?cLa|{7mEKY8QWWpi`+Kb@7O-FSdhNaU0v4oqkzQ0ldQU=40)fyw z>6PBAs0bwUzwex#WOH)T6G-{a^Sqhao!!~l_bcnA6yoHsZ#?jZ!W;P=KT-e ze?jl{{<$4xw<7x`QvS{YZnv%rXFl^6XnC;F+Oj&GjkJo(>U18rk)DtARirhL4n~T^ z@vEe-`zg+IV{c!itY3(>`!&#dHM2Tv@Na?E`I*&O}tw=CD#Pc8S1?=4SAH!EO%FDqbSA1ffdpA``CqvcuhljVsVWVx3Ow#0W7 zgjhLN7$_nGQQ;rD;?@p#CCAKk$F1n$iCfh_ASr5k!0zAo=EufePr5C4H_Hkbb*&XR zzJpb1RBJ1+r^gC-*Rnh}Rkqw~*XQ4+r+;g?=5+V^hS0uNKv+M^13%%0zi=%YV7ZnK z%70x&q(x2_2Ffi1F-yL3$F2D(FganNC#RcS+|vqJHKy@Z>t{c5*QU@9+Q%&K{ba(X z@lPa0Pk$kPRsWu`%ezjEiTo}!CbILg*kxVT#w_a^6|=m1e9Y3$(b%^xZbi>E@vHi* zSTp^h1z~+X!x#3px-aPE`gm>+>%Xw)Ct%-?EgoP!xb!FM&gDa`7VAe@wGV|_0XenD z?4SI#<%UlPj2LJIAYRCik20r9DhyOe24YwAa3@92a3{wGd6Hv8ve-<-kju{A9RBg8 zi5tgW8NKM^uVNy9n4GwIQb8lNRdU=Ml@cGSQW6)b)TG5ggaeDQ zKU{4|S+2HhS*_yMjZw=cJ)q|IJ73M|W~n*dT>w){!3U^$J-vW0P{1G9-a5CtwH)aj z_=aiW{jEWZf3)6N)Zc1_?dq#XS{EHyWL5GN;H=I%KJcJU_k!M*d&xkF5qvq4?TX1j z?26u=gbiae`qbzpU+VF7RQNmB#IEf7N^;DsZ{ybvi%Q%y@l;B}LfH0v8w*rQLYQsa zOVVNICsPv_ok&RxJCTwQ3M}ODX&H#0pN3c>eM{={^n^_l(pOA*IBjmvz_g&wR$53` zD-AY(%En2gCv2R=T)-(H9mwzvD9e_IPe_Ap?+fi?ErqY>j{g5O#0~A&jIe4`vMYvW zwRzOx#@Hb+QsRS(srVK6uH#_ysVOTxdygyEhTzAvy>FcR?1f32r?iM$-G5-><|%8@ zj~`5mnFSjU_1ZZ4bLL_5u<0<^cGyX5)5g?d?!W7IUK94w|F?_D1TxOqbFq|K9(b?lxH z3VVj_B2GsPPa9`n&VD?rO*`#7r+hr)g@roak6klDEgp5fw)>E-uy<|u+57Pv>_1z7 zpTlwV1@sX*eqh|-dmMa$!aT!v#0Vqc4?e{FKnq_92Y0hPeCFW0U$%vTvdBREy5XMP z2UA@!OS|g6J|?15o%nSl-ouzZE+ua6@#OeBUNJ547GIZKegS5bfeuf)-U!xzc6*u4{!`Y znDf&)W`Hl)0sMkz2QRJ~VVz^-41eL`dIZch%KV`rMbc8qKveigZmh%6@$1xAEqCnl z9xW5rkDitsGdqp(IOgeS^X%h`&*qK4pzWuwP_b)QxigU zth#yOo6Qo|kD8bq6Z9wgdBpeOw0n%{i?OdacFzBt;9*Qqp_EK+w$w2Ywdf-^p4aKQy$!S5 zUlzZ1Xm`Z7CATeo<{cr}OCp`23O3 z$G{h)V;ry$^94^tO|;H7iV)TpbB6Nlz|bM-X<}eQ(4#Jn@p-x5+FS1ErQbY(G5%IQ zr}xJBC2aSm-%nYg)-U)7cJKB+gDX8_{hVyOp!>}nBJa<{1hMb~Uo0QyeV#bx9jr~p z8Y9FB5)Ygv{jbQE*|gwQ*QWWeXxm>mYtAS$Ax)}u`eKDZ)o^-rtpaX#(7#AE$rJUjG)YX?H{&d+oeHn_fbp1-HZ zJU7dhnmCzrXuj~i_mlAqzOeRJTF#^W^Z5b6X9kBjKd^F$bw0IV!snP9z&v5`YJ}8Q zQ5b-|>-qi7i$8CL=ljWaywB%(#cKP;&QY%&ON6N{N#SZ+$`Z9>%Tl##>r%CM&t`>p z2Nd*KEgl@GA_lv2#Nt72MJ)Qst-=SoRoDQxTG-#E7WQ?i`Mq8GJwM|km>M79+*ii( z^aC6lc-Iv5vJ&uZf~Tpc!CfpDe1aSCfnJkUQFK@Cv+xXHjXu7;5u|NDYDveZ6Qiad z!yF&ueLmAGhIrrf>sYI=?L0#5+#0ENZ(phQ?OLM_?p?1A?c1mh@85)Uz54w~vf2?n zQbi6A06ZD#QskZRma#lL^l`&*yp4z<9_=H-e{`$R{%*CP4}1fjyK^m0hCi^M!IdFj zp#1>aul)eV01NQG;kDF+afk)r10)tGyjfI;Gx4sy3%0IpAM5zu!5Ti5lC&88J>Rb_ zyzQGlJ~c5EHosWy+P2(b>zmY(15v`hnl^eDH@2zgShMz}V(HhY|}^ z2x~6*O(w_9)qOzi#`H#VP{vhP27I7P)SdPOb%$wI$gsrYQ)5esDTAAWoS#}SMXICdyw@d2M3 z>UR$C%y0v~X?O=Uf^!1+&XL3d1vig!eMV~XGTkrW8#~`5$Kf5lq$Q^?&yR0y@Vh#` z-`2_cogJ2^@SQ&ITK{y|J$yu18piKwu=!Jm_HR6eKK>N?bsLQDy)k}ff1hdlhxctz zM-L^a-O1Bc6+M}6vqT+Zh-Aghwc9m zH_5w>kZXzY>`-EXa&6W5ySapQziK^5ik{h-F+aY=k(L-WMI~KZ9E?7R{E5T+Hl0LydR9Mx*#A(x+7#06^!5FIY(GnEfO6vtm@XZPu>$577xr^| zeS!VFp_KXpzH5}m_yzLt{V&Yu&7eYOkI7Ao35@-Zn!}z0*SJZLK^}#*fX9VS1x#i|^WB|{HJfZmR{hZELz*MX$&+)(_ z79>Qxs_R}e{o!-s7QcIc>ZTzt?%TEMxxL9#2keX+cVz$0WhdYd(hu!fg}y&QZ4A0A zJz|h`665|ff3{cJ_BnlljtekGV1LUs2b@1FMR9?iA7tAwH#pMF4ZZ25 z(YBBA-?Ml&JQcE^fVglf^kO>thPjw8S^})Wz8E0E##W^Kp51norQrE_;G z*bOA&8T~Shi)SGop1x#Az_g`91Eww?R_V8uBPtJGJ+ktII-EX`R@s#CRF{Em%<--G?jvN*cW80e_KfpP`Gczxkmk%)I#uspWun^+|_PeF% zd7}0Kkmp`J7rcjB&F2I1jIiw6cW!rJz9E7B|8~e;&kM1C$_Y+=HTlp1F3ihkuBkEl zo@+YPa|-AD9XTxU&lSTfB`hBv&<%d#`BfvUKE3Ie^DbwbH~f6A_6hJE+CMEF5?B{` z&i!ZDzxP=n<_7p&0Bz9QSRa(@wevB*SCZ$O@d4Kugke46yk4%-;s-c4$nOuOVJ!GE zWia~(_+OuIm~+4U%02`5zjoTbHvDW|fM-Bav}f4;-vIjpC(Jq$v&P&zhm3FEFB;@d zgFbUy?u%2{chV2I-~Tv%s3%>o?HlGvhMg~3IkIy9Wy32qUN`FO8e}_iXkd#B`?qa; z(LhTr!LzufBdWp&1z;W?+e5Kkoa=*1&|m170Dg<>lex}_e%F>mjy{;jlpwfPK=%Wf z6Zn?A<2&W<+{TAIMt)_FfuOG5c^=5Qx2;b)2Fy#hsGswCy7U@3-}SVx?Sl5b?8^!I ziW30+gQ+8(_Kw&;b=h#wH;Dgt>9XO0>3Tk3coh{J`n1{?KTI9mI#umU8m$r{UQx?N zT?n60%xeTQ+hp2Stf?6raK3PUpEK?U^qc_4Cm0V7OPu6gGbrB`D0?=ZQ;YGPd2dYp zlhp>}_cVT&Ixo9tx!3kBJ)dQC)?u&3WfP1KOq`Gr>w9fK0PAxtwc?i=YERNQy)JOo ziaBc5wBaf;Vx~HHAW0qCF-NT)-=xg>0N$VBbkR>9{VsyxJU_B2=3_F}3&96WOPp+- zOI_msLe5F2WPjOU!2B<3wcGL_?q6)%K4bjceSq%wOWgMT`4IX8#0K8^{|JorcSH|X zd-lYsZ$AH@y8qwTsQd4{Ufp-cHR^-69#zSStJVH3Gq7IpZ1224DW4bAe$6MoFn)k@ zg?uOJ3_d&5F(Kvx7pxoY^#P{uC>z=?t@LF8ah-m?i|4#gGRA}5=PT~#v7ECihM3=< zjpw}&V|uIuP-~~QRDb=Es(N*KN8NRMbM@Fmx2i`UY@^eA{_h&~#YfMoBY0mRdVUAB z81n(A<$VDk`}Jqjh1r+k2e8&y`vLms)4zs5_YLp?i#Ct3E~Gxm|2OuvsgFs2y9^tN%Q9yL#mQR;ok0TU7f8Zr16; zZCj}a@3~RMY+j(%)t9tO>7HaT-uhgC$tJT6eBh~h8n^ek*kMui3W#w65K{(ca0M8A%rjUO8 zH2MJ(2aKoM_j$!~;T>YXYq!dXSyY&2LQk}>#<>5f{Pz9qV+y>sr=x&%?i0#}3EgouL$!82N#B?2qG5OS?+S=jFFv_<==( zJo=r((-aGwVjqIFBVSPWgYC5>Wx)pB0(pRBw2yx+EeKI8Q}e?%SPd~>O? zeTOY@eNWW9_Uh=qrRvXPscO$Id{{@8|a`On*{RdRo+aBRL;{wLpBv$L9iE8^HMi z?l0}{0v7e&^K(As$LEFYqf2m1pnU-JDi!~iye?JWyjN1n%%H+H2haKSydUOiAF${B zaR1we|NI>@tw0)@ zdDc{6jjj~kIDE5?$uL(l&5rGIeCE%7qAc|NC22F|v2GI!%q_WC;3WGE=vXhRKE?zx z4k!gaV7dKW*OaN&*^qs-ZU1KeFWcN7Ww^LhkNe9?TvGblnHdXs-zO-*I6s@7_&GNn zeMcI{8W97n_EdeW6O#GBGwcKKO&=`-yuaNJKAGb{#{PcefA;BZYUp>i5f3olQz9Z1b3RkJ#G3c6M6>l5>|5F*b2)F{c6gUUqzN)}kz}Y2* z*Pg!SJo>VbUX|XM-PNsTcX6i!OtT>f<)H%uRruhN#Uj+77fQMfMX5ihwil(&l0G-5 zHaqvT4c_faYF^*KZ2I9(XY#S{WKcJ^3hr4+j{{zQ;+pe1v@1D%FQ6UK_Gp8&MdmLC zY66!4Rf{VIKwb5L=YT;#1aJ`G_bSqXzk%O@!@#j(0_!_!fx5^ zNe7;IsHGwv*Jy>bm3rb~fN4>IdU&m;9%-qb2E5nxUW30*S6d5r|pkppKdeiH1}-IIeqQ*@||&g2UnU?KkYVhaJHWuKiwX6A=P|4 z^ux%e!0CgnsO!G^ZS?dA$_Uzk6WnL?n8*5_U)XI^?{U-}BUbg!#T>DhC2%dSqlWW#ZzR3;SORnzz_pLh5HuHFL z$~KbYr)VV1Tgz`8fs9FVSb_hLiSjzMTGr{VBle z4iDTsA2vSm*Xbbi5{a*h-{UN9WI2ZcT zTFZ?2+`!{lPF?5uwEHI?Zmr(?Uklaw-5b=vFRxW&yEIp``(35NhBQ+vMm16E$9l1O zd}9?o!HW&!ny59uHo>>Mo2u}k&D8AvSE&izuT~?zxkmN*-8J?ZL{-!sF&1#^1ypwowS=5pKPVRdi_S#@3ZUFFP*PZGx}Vm77c5r zR{hdMtsB!sMWO9cXglwP`yz&;&A7+0U9MIGzqn3)^VSXOwg0r#_0o^Aoe#5}xR&;D zdGPA5jXaNan06Wto_?gYdh@v!>Pxh@=SSD8p&hSLzjeJ@&Fp)X3LV@`MS|1SXd~Oo zHj{I<-^gL*=qCE!f(Klw#&>J327i66>iYim>Z2ELQqOc~ogvqD_oBa`Ke1)*Y@=`9 z%PVvPZGI4a%>(z|nhqFUqFx^WqJc#~2rv~G2TZi{bC9k85&-u9=3JAq4r2qk*$K;N zQ#_CPCBR$26}Am(TeS0T?5C|}?Bo98`mtWS{w&w9_q9PfA297?Tg|X< z1DFkb4BP=U25KM0jk@jg5yEoUqgto-~mG*0{@8r}sZ+NF~ zy7w+`JoKOTwDp#%`-j)VRIX0dOloyPW_e0Ve>XkF*QI zY&%o4e;vvn05YzJd=`7;w!gNN)IP`c^;pB+{s1ZO&Gz6F6=d0Xz&`!1f`{ z()NUj-nrJzW8LS+O|&+}O|TSFZU6Xo8-0O-Ur_W5Tc>+qd$phg4b)cHE&I8l9k10k zZTcG2ujOh|_C4%3dwqPpT8!gqE7T!l;|Jz7(ED}=H-KK+z%e{7FYLiF8aPj%k8RpN z>!goxUgr?@(ciG|?epn%>V+p-={}t8V_W_C0q)bWE{=Kmsh044*Xll(_X_HxAWqg<);0V70=~9)@OVJud^rkG8Hzsn(Fg$ZS|iH zE%i9E{R7_kkL@MedHZ_x;uANho*&jz(|cX0*8UQhj`im%71%z*dW8ClG$S51`U`o{ z=4q2zbeZMOAIdpjcPG#XVE;#1(>9zo!hOc}2OzKK^pNrz0YI+Ta9*#Z^K1*im_T!a z^d-R9J@3KX$1dc*19a?S+J$Z2Qx=TxdWo(pYP_o!`utS(`#>5~_yMK;0enL2cuOTt zt_u0r&yYF$0fPR2A^P*L5qmT4V_ZyqrA{)|WE@QW{vEnO`}EZ_?ki51`kgw>`VY-- zp!+ShC3SiuHRs2x)EBR@&0ZTaHfD6R?cKMi=O1sS`_HY2DcEmQ&rSU%Eo$3MJtn13 zVJx#VsG$nN_~wmgU_)pNWS8NmjBldfegB0PYD%x>im?don|(9e$#$7Kom}iKA->JE z_RV2CYN{PGJ!-|sK=s4B4Kau0Z9nG;80WwDd~*feLwxDc_P%vG^b7qB^eruJf(xl@ z^~PU4`hSUh|4u^>=9;y=g9B}kOnqSVn)addM(>$o--Q6g=9I_ro@m=zF1W}1T&tvx*&EUb@j-pHZAMjORxQpuHvrU~JF(*bC^G390d6c|Myk+#6eZ{M_2icwirV0FR+B*vD;$ zUn>oHGj3s97!&9=m$EkI#1qGjTW4I7U)y);AY%YMca5}2F@V!1U?zpUeE8 zS4KN$Uk>}{c&_mBmveTd?0PWveD*K*i@ZREG)!~WCJ zhYN%Q>cA-*Cy^4|P6P7gCm9Fq4ypwo5Rj#xv31b7&peOrKzl@au{;YX@3x(v`vaGH zJcaZEU=u*z8FSI^X2C9u+c@v+eg4m9fP#w5!QL$LH#X1lVMF==`U9q>ukh;=7zgYL zrVj|rEeGL~hSATQotjoT_oKUab!L^3(4XHO4h}aWc=fIwjy8djPu>pMo{Xv#;9S3M1 z5RlV)Ak+UdZE5=ei2=&9Ly~(da|~?$&8m_*{k+CIXV+-6C!}_J%<=uaC%ASzVx-^q z;=K{C-Rt>dGfv3x1zsOeTaN{})*N>33ljtMw(Y-A&%4VUsi(2Q;G8i)j@UNyA`g5% zP$L&q_Z$toz1U(iT+3~)SDaLsdP|~Cwzs(Ijr`hjZJ%D<1Dh(gI@vaJ|AyNFr?a64 zvo8;TjR#;19|+qHJQmh6i`>y>y$!u-jIrZKhz%wHe|X1Ci~$+e0s*GFytw2c_2kUJ#>w6>8Pa+{cc3%yLm}}!&g%u} z@4_YrpBHd&ah*4T?m%bYheF_coYM>VZ`8B^t(&O7jeQ+nuP|-pJ$3=U+cd>hi|p(R zuUQS(=mc~D3e!%W!?yMX?lp|#8tnmcQk1rK!}%Wp=dfO+S!~u^t1v0-+_inyrEd5H z4{ST&Sa?={uyL#vuzkiwb#?|_{y)S8D=~lYryUc}4{)v^%{wPpQ$HJYybokz0rM>2 zCAN?DU66L-c+MNpPl!(_2yyv57P_QkGNk_k6p(JG%oh|V?Q^uOQ$wm%g)M9X6p&`8 z%ohZw?BnMo%sN}^4zBT@eVtQ)eF&J77Cd8yE|)U6p_opbyX#*bBT0 zTnDhNOuu9v_OIurVEUe; z*V|Iv)?aX)zkv&ZX+Tc=J&m$EfTMQ$B2u>Xd|(2w8_?geLt5hYZ`#ike!vYIM|@D< z+gJQzx#1rgVhk}7^M@E4=&=BOK-$hA?>a%)spnKN|OVSGnC z&KU&UV%meeo6L{r@!p=M9Ygk4C;!*gx-sKOGayYw9_%_EGHz z@P5E+c!s#2egOU$(88={|t{KW(}$@Dkw9{+DC_k3eN0 zH~VM*pOgLHgR-ju+CS4r0Av59rvLX@AnU^%c9APxa0~`xr?MZGn zUYIi4J!1wF4=_IqXvTISWh|imk#9R?+dl))F|mJ8V~_u7^J4%0RVBy0m=n}2fBJzO8!H7_7!doe$TY>p`!lhAfNO%dPS9RA$Z-I_Cs+^Pq_133K7c-f_Rp~Z z^TB}m_K?H@*~OM+kJ!K1e-?!(zy|(@f{y#KcIYE}{{Mv2_A~Ph@jcc$`n_S>Mt%G7 z;@WolfJ(d-TuXBm+=KHl>ZJTjXVg z-`^1{tWPjLUU)0FQy3gt=D4Wdmy81Th9{r``SOZ@k{{U zqb<^N!Tk6HGbg}fUtxQZW?92%?m<$qf3g39TY-76Y5UBJuEW^=cYEBQW8d%e1@r-! zH{e_V?bOWM<;SiIa*U~WAFk1ecZgKR0e(a+@xR3Xe$*n@U56Gl*D>|3;96hUww>`l zpwrfK%co2|%h)sL@6B^Ov#+Qr?f*EydZzPckjnQ*{76;oU+mwHTI9JaW{l;*TAyUw z#`D$pn`0RR>;EPqKM!yNYk(hF9a66OyAQb4&Ogunj`hE09UtGF`3B%RS+4(k z4B+}&K6mFkH*WxZFTf4_3~*g7*W$$i{?`B2#q~Y~xc;{aFapqP@{w}=Z)f0QfNN); z0=Qn7ZRNT_y@uFN{NE4P;`(H^sU2`5z%_l10IpBm&wE4q8?e#Nvz>#0bAa5||8f0v zPW;`EvKxTkf$IUTS2XMYen!4jtpCM&f8FNz$;5&Ye8fRUO z`M}R$myv*(w{!OWv~BJ;?*=e$zv#%j1klIcZ_eZUg3N3Cx8MKCcy|Z3u^s3L(580-TwmV+ z;Cr)|0<^CktP|;AK-)ah^?<+kfB0_Pdq9HSj(*6mw)64Ge+JMm&_8fJE!X|CedhwN zGw);n>9|gJ`}%AX-~ZwIf8+g@VAkjTKDM5*4#2S=>jL>aux40LKkvL; z!X$SU*ys(w`v9NcrvgU-W6w^X;4jbhe@Vb3;8~zHpl#lyY_t5bGXR}3_J14o4H$3Z z1kypke1Q8q0dYVI&<)rPtOfY}4VJTSqpmam0!1w360z7XQ@B=`bTDa`jjJBHhe+J@OivaeSJ%9}W{npn2*U?7;^a+0f zvw$A~wr>Cs$~^49HQS2yzkI)sHqUo?`MnahX$mkJpkEjQ%mLWG?*YEo%=a0XXFJyc zJpi7kzjwuYaSguz%WHlI!~!XRKmI0S|0saGGv&L&&hvI+ySiPcmKpP>`^RsD@S2$S z8)@5gLC5~K{nuI$(%@Xx`e)ee*#Q5y%_G2@Ku4fA@S_cWE8shT-xGKg z;IUPlTwt5;1IfIfcG=Adn6m2lfNhMcNsUVg3m4hn;6R&-vTN zQ@m74@Of2W3!wmQop!<4{tri(DFfzL1DD!$o)P=!w}N>6 zwZB+Z_Jq`l^tay6(a-zq`?(MCKl=ZmS~`woeMyr8^*g+<*CM^kYtFaREIx(TkzdXX zK)22Ut^yc)5?2C^9r;GcH?v{#O|i|qIhM!s-L><1rR1Ps6_U_WcGKuz zIImwE$cVkrzg`B^-aOTFg*SEA%*=b32TXaEd@Ym_Jg*L*zkNl@_J`Lx7pS`*-|PZv zB468^uT?8+z9zPLj4sD<8F}v0=OON|%sOMH2YPS~=QY`8XIs$@XZv`pKA!t|tZ8dq z?APy~uwL-#Dke^|^^>gsbp8;tYy00d`-)buTiQRy@|o}Yb|fY)PNdHtN)$9DR*mGcoiUdOgbb$iQ; z+!6EZ`@x#uTfFvPn7RK3?EAN-PCL6MdD%Ym;xq2o%8PtT+kH^~fz1HtuHOTW0SkaH z0nW{h1vo!Oo9P0~0R9B{OrOsdwgUXV59b;es_0vFTmk!i*S6&@NI56Z`L{2DRlrW* zE#MHqxyNq-&L#c^@Ob(FK2w;@JnUc3|HJ-k;u@T1iv&Ig*e1^1^H~AyoX;0HC(k*? z4}tZ-F5m-T8?YSUeepa!|BtkM*?&kKulLM%e_s>naKM}wh5Qn8Kd@V4|6DWh zA&|;4q+ElepZ_CG0y+X*lf&n8j{-FS+W+~$o6N)h_4j{a|5xA|Y=?f1Z|B=0y#}DK zqJQ`wu!q+|`Wvvu&eLb~19)G2uAra)bDQPm<%c={qS`;k3-uLZ{^!UE;tA>d%ZqaO zxew^Sv46e(58HbI+J7a0v33XGT7Y)PwK=Z=2Ur!-QGhv*>kcB!C1AJ4{%a%u39tn1=ZQTW%~L0IVRQ;Wy}*9*-Bb| z7=RrZ``7E_uuc2_8sNMD?Y}+1=Y_Pj8UXu$fA-H9!k_)ugbjZTYy~`kul;|AeboTk z|6_pZ|EnX%m51@adP1glp0{q!OWB(t4%0Ribf7q+G|Lrp`EzJDC ze(s04gCD5j2j(``b(J4&mpk!g0Jb3Z?@LM<-o~80j@P$NKfek3@uTd|F$a*J=YRSb z#0pa;1THF5f{O*T$p0K{c{|xwq z=i~nbNZ(H)%K1<1zkJz0)%C#Kre^*BulDo)yw?Al_x;u+O{sbw>oDI8W?p{fmjSW= z@@M~8`=i(Ep)c>gH>5t}|J>|f&-vkb|4IB$uzyk0cg(&_;{Nh$@>%Xo?7#fkKkQiB zFaBrvJ&v(|PICY!_>M6CN6@!a4CE#5&tf6vdqeEM{Mo+2mNHZ)%tf`TW6jx&-)C46(AM+FQ4}R_mbPR9phQv5ZnHJ=K!&-pY<5iUFw5*Z_ue5 zk}v-Zi2avO``?QH`=wTQ(ET~=Jk;?V(Aa*`))^OELk-_C+xxx1@~`c3hrSFr?BA^8 zmG!^AB$d%^%>V0oK&%P6&wdwBkN4pZ;0PF#1?>7T{ z|A*-}%;Wu^7OdBP|Bv4YvW0k( zYDhJ`_5^mzcYbNdr|gvZf*>8o6SV)^Z2PCrDQp|f{h056R0#Vw+Ku(RTHZ%OZ*buq zK^+sAGD#~g1F$Fd|8D@@fv!MjpbOB=&U2sH&wVW8G0gv9gXO(|=XQsB*N(w%ZvprX zarXU0XP}!;p5ORj8TYfC-xPP|x&1BdzZNBg{=XWo(GlSFob5AtrlzgDAGVckWS%Lv zi9WzRtP`oW3+Bu7Z^x`lVD~lUJ6`2k$Z~JOfG{8o6p#V*ft(WyECyusFX*@V|H1gZ zkK#au=%WiLHaT1vs1OXmhiLmp$~nVdfT_SVV4_c+`=$Vsfa$4j;L zNj{L4>-qn(DsySKFkl#fjhgq zgU9Ocek){Gze{>yT$`Ul*YL>-C99Y5)5Bzew5t)BdXf zJXY-A5G<)MP>LCVt@^Wn7xvTs?*rKX=hyx#+sBIimtvVq(}V%T0PI!l-;g1xFi^o5 z5c{uS0+-tr1{@5C{X58ztT0f)7!dofU;>xh6$Ts(i2XarkgPCJ!59$xuV4a~+Z6^J z42b$XEm)jKv91Mv4JIIi%Fi^o5@MHh|u#fBbx&E&m!1sT+{*Ukf zybA0mMM!@GHrn|W$o~ZJ{U2HXTfuZlZnx|*0DE<=|Kt0A-EEsbi9FZ;UJd*XR0o~` zw%g@g|F6GKjC3(z-WxiGysZB%yDA|quRsjIR`oYJkX`}sef|`H-vD9%zYGYo^K+5% z+rm6%GO!ikHv#zV4}K$n=S>56p8ozPQhEQc0_l?6YT07|Hfs9+OOam*(2jQjQ2=dx zyPe;ObPEsz>;V#iSUVqwl>cvF8^Gf}AQMRC`#)t*C8Wg_kOA5%!2b)v?|_{LTm)3J zVe;o=o5!%6$C+_Eqr33N6cu_*Q6@={MhgSN zfG{8o2m``^Fdz&F1HynXAPfit!hkR!3)S5C((+ zVL%uV2801&pu#ecYIV%*-op{xR?hi=<2v@Olj8xMbh(vN`6+L)RV{}D4%_89RbW?d zS^is)Zcev6elOr{y*{X;zXMV;&i8+T@XT_*9Ay?;{_Y^Nc{uR4>trc+-n#cvS#L0VdDa{B5f7@!D#7gKStXc7J9Pu{D$i;osoq9A|8{iVpr7*F@+hz4 zEDv|q?9FE(s1W5@2=sG)76LnYuj%}oS9ul!{hXhLz}(9Hm1Q&pEM=E%Myc0MGdHv4 zG>PF>$7}`IQAeeE9o=D%@?U<++Z`RR=C>SmchWLEep^oEi*)%Z<+mIH`dyc&i&Zx4RlI^Su62gZannmwXn@Q&F78U~NTS#*iFnX{Eb`PH*WjL9r# zf8nsf?B%Ihyma<*`mBx_7w})6*?{cjsNc~KW-kv%d1eFsmuKo&_VP|RKT}7tm!mk( z@=P7^e|{zb*~-0LQU*aNhkP9Tr(3R0*{(k}Q$D9G*I@@gUik-P${*#*Yrk3snfd`e zse@fC&+V6Yc_z93>N(}p#nB0P_0xNF_VR`S!y8)L0|w+&4*k^kZ^|=;@7-hcx46ok z=EFLD>#yUzP{!Yamfx0frZH~bsJLYk|Pq9z?RW|WwtzTsmuTOcGJnHY` zTTcHo8~(@MpW<7dRX==7(Oc2~9d_7!_Ct!k2X>NcCGfAgmS^6d|HGu*$iHmzlb8JS z(jWhKEqjFlVL%uV2Fe@*O)*U9q{j*vE?9M$!_VbSmP!S00h zKL?~HMF2|zQWC=hcJ4}W=hV(|pIPpZZdQO3^LtqVp?xjSg5H)Z;z#)?T29=Z>HP`o zhPx6r{OU=Ln-{QU%i0`e%ytg0V|ix1XjKXR*nQU2|5;UcZ~TE@%+2*2bALfyEq8EN zD_{ZiVqqW4EjnSCIWsBj+fBRPzI#g+|GjbEd*{Zk=y84gx>1iOZW{M#{MzB;lA>o$ zN{$Vlm$-RCRKkWa$w|>ux241dZP^(5Npk3q7j2l^-C8iWhifWqemrD580m+=li~fX zJHz@~wbqWZ&h~egvwyHW^b>YLw-9Q|uQsNh=$lvZzL7gqwybHGN$tG8S zvc|r28%9T%W;qx7@F#R36tUv#(1k0^JibL-H>Y(hY2U&Chw!abm!%NEtTW9fJX8nK| zp{G|TxGg{b3?#)&)3T3V`a{#CsHv+{;?cJ!hS9eZxyv2p$&h_AVu7T%c`9t^mD=v- zz~3|0b7D>ou^;WY2%s*|FVH_|p8%PsKo4GBG0dvMJEjgGmduD{{IgIo_rlJ#?Bmvs zcrYnu_Azffr|ms!zi#V6GIYa$x{-?bZ%a~`+PQt5T0QN4H6Jm2*nmJ4+TW$Iu%AmU z=<8DRd%M)UUKufdR=p^i{GGaEVoRe3vCxBuDcMP1>exVHM2DJh+ZnHI0>1w(jNxGO zi)io1fy`JQdC2<|Y@Gc-+ScU7={vVZs=Yf`sRMh~se^mhs-uTf)V9q(sU?Fg6*(eM zMGp65!_r|MMJyTWQ4xbZYSB+_=m~VA4`RcfUi&w`q6ogg=z@tAv>qTn2*Eh95pSJw z0d%6m`+_aIVsw0uG2H;n<2V!=Ib-(>z{VkSwR`(YE$_nzHmjotqtlNZh&pv- z|E7~j(+=%jo4#^veaO{PONR$)IU9NVO?~uZDRg4-AnJo#&4)hBfe$GrUC4+L7$;&b z;a%#$&mUVZ=tPC}1smqSsqZ~8YI@V8m|4RUH&5x7xN%%(%+-C5*zenfjbpw~+%&N> z=52axO%Ct7e`n;({W})LK+cH=cdz*U;O>>`(7tu*(B5_G_oG`?!qN}Y7Z0@3B8OMX zTlTge@YM_BGnPU(mJIQz@PTga8|VwD3q_0_GV}m)pNnz9<wWmP0NN? zS{*q&U=;fJuObFm?PtAb)t+*{Tieg7VHcebA0C2o6*0(lDtv&Y!hf{X5^d|yi5zS` zzx$~RJeGaLLd24$e<);}KwprK{vZu;!0VLj%Hyng zd_n|#0>>ASVPXA&9w+d8%prEA4$KBOc>Yvw{Q&M$_xbR9ck#X5`xvjDAm&^AlchG#Xrb1vn54RV|FQaY^p7faeIFHB406Y{vgspr zfny2Aj2u4{qF>1955UbZ>cFJ0%l~s`^nH=OwomRDZy7)9kMA7zQ7@nifhW-~q$jQI ztzLWiX?52>o2b^;)l@%^?4uGF+@<@1e2okI$T^4WX6QqP9u(dmoFE_2fuD^Il(%_f zGuA_8Br@+!r~_Qt_*QR8K#CkGlWPYgCtSUQu&`$En!)52!^0 zF)vVru|Zxsz;dGpoNuB$3+NBLI?&4+L>-v%UHKXt4td?%uR#wMJ8hpFF|N)JzquVl z*;2-dVUMa=GsdX@KGRMmC9PL~9bBrGjyOjx9cqsa3Sr;5UDsb3^0$( z`hoAM130%_%nxABL*EDHzizQ@U(bozdZ1(KLil|@I)m6B^FQ$mA5uwcda2l@pDK(C z^_<{Rj0^n9s-*Vm`+**?PvbooKo4|03_tlYbpYp<`S}5^q2s;a`A0un=339m3FqaD zNA@N@3($Uma{(L+EE(d)e8B0Iy;G-nZ;TN+X3VD!&^9G5^aYw9@Rbrd$hwp=KhxvF z_7B|bDhx(`dA|1j3;Vck|v3QUNEKV*M@rv`Z}? z;a1Cr=My{@(C65BZs+^_ysqP{vxMqp+n0}YW3B?vqA_Pco3MHDmujvS^|fB5jBxB3 zi3g!O9^f*dK2Q_55I6^@2AuB)+<$I|b~m5%%9B@DW7}tT4IDkIi$~4w;yF3HD^g(L zz)D!B1N@X7Vb~u|4ECnnUO2F_THu3)I9DIb<7{x-U!A`Doacq19%uc`^LS?)e9!l{ z?W~jSF>Uc}cjzG8%YaHMsJoX7@-Y+q%={t1iF9ty>Jo5Z^j8-(ui5;uD$lgP@mz3A z?wJ~J+w99tCOqZ>fHI^kuK)rmpWJ>prZUh1co&!s>;@F@JFp)(WaEIneH7_QfX5K& zkKBKA`onE+R>VX1F>S3LeXyl^`jM9EKkZxS^eN=c_7e}aR1e?ZN*sYIXZ??& ze$-7ohdi&zYqL%s`xNTrxvay~$CPc**FpKCxc0+1p69UM4rtrck9g}d?dCo3Uf4#~ z|KNQ%9(epg@3Buk%rdkQz`5x>zy19;AA7KEo4;UF8Jv<=8)n}rY?FJ+gtBnT1T->Z6Z*2cl-GA?`r(S&W2KB-d zH*9EkZ=0SS+O?+a{s%k*ya;>(3<8z{e*jKd7}*@d{vJRxpulTScn-jRK*t}{3*>79 z8v%2@Xyji9^m^#fpDxtTGLDB?=Wd^2ZQnNCQro7x(wOcIzEJH6Zmd@Q+(Zre`danh z$66_J`=1W2)%b2#tCVSt)X|0Y)t}+@)UmL7>S$!I zi~RmM4OBEZp4Pj$`s|e(pg9De z|JGOy`s!Nviq`4@=qq$v-G9$5(2bkaq#n)H=JAcy-jD|B7}~(L@Laa-2?%e@3}$Coqna)M|IWwAJ07%{hM2Doo>w|?+@OY{TUJNOLL9%Qs8-D zEnwu5jQo|h%(VQlO<5LL59~Uq3;JCcq;~_n&gVczUxK{r^H-bd?G+;{*N&awIszyl zJ!ZV6w$H4g4$Nz$<;VL^o!&?d`20H9a})LV;(91+pt;-=+(7LCrv$n62UBj2-3R%T z??1x4`muOeGx+UR>XoNks&!+Us6XxfdvTmO$6x)jLd?VZAi|T88kDuH~>%?xzrZDyM z9G=T|ra)g;{nAAJg}TkP3R9=^{&@c0kjC&|wbYK80Sf%lUNpd2n$6!jq{=AR$-9a_e_s?yZj{DcZ zy{E%&(wUll#g#&4>F~Mf&@p|DqSniH9+=x89rm8ix{FYsAN{5eX#_p15BXP7aT7FW z^`8Px+Q4>w+6L~WzDk}_)z$%*u^px>@+&9GhB}bn@=6fWH4Yd29eZh5_;rN@jO{?B`>y%6DZ<|`Z!L}*Y8*Bx1%54DK^>@sw zaSh^(>kpwWQ-@CNZBzfjkcJI+1zmpQj+r%@^E^{0>+`L1`}7)3dHwyt^_u9mI_hS9 z<~l6f6I81?kG1d5+@~{bh4%e>Z%Ew-cL&$`1nHQ)A@!o6EA$1nFKbBua1uI@?(GX) zrVr5b!r(FhAisng@BrLL6nlSP9Qxo}$EGP>Ke2kE)nMHut1WP+jk_HAen@#u8yVFPO`4YdG`Tqm$SXBPq%f=MDKgSI5yw@H*UPg16^#kC%DeL(1qpD z1;}6blWECQt6~h05f3n)Zb&Y2lQYYJGwvhocCamg%e`Y(b@#5In(m!Jm%HPqou$W# z8z);sfhZsnSOqKxR@z{G6Y!f-6FmnP*I5g!29^RVeXtzcNk^A9{Ljjqg>g08nZ<_2%TsX|CaX6yhw>U2zSO%;BSTE~h%6;pBpw1mE7tUJt>xM zzYO^p;2X*(Zd%n!uyfpRHbB-jJ+S)#$TMi-&?g`!oT6hwJ%$38 zYk@FeHLwbZ044)nfDV9u$H3?awo9cy08je9k@f{X0^SC`1U>-XwbM_LzQ19THBiOh zeiq~~26!L%0C?9()(ETtDI5_iTKE^h3w?2PhxCXrxu= z@RIsp;(X#`;6va&8($%P$Hvf@>48;n{x86rc0Fw8XTVzk+spbp6679ff!1(fY~)7F z5^!k2wQh`&weQ$9`|^JvR@e-mpb{rnVqZWVFnYk4&E{6T;Y-MWkV^b_RmfsI@Hs%4e+E!)-2Q?oLAb(uP8x1XYX4CGJyr~D~vr(V$yd;?Gq2qXWoIQMFhfck04!Y`12erAY_CkI0`w@$-Yv*WBz0 zuKf~a4{$u7=T0%cG4n&l#(m}Q+@>CI`v7naFNL&J!~-IKKZ3_D#EaVQx6QopJoNj? zPWdwy+&{Om+BV}tqXX;~U*{AsP|AmVG4D0as z+V&9xJZ;Nf&j;CgJwL!XRLl(=vExA_XQs~BfO%79@{jU5*srf)u6ZU>hx{4;vF|kT zv*}MwpZY2K|G_H$U+n)G|FJ)&{F!F-{m6fU{=c`Q{~v}rKL(sJ0Jquyf9&Z0zs9*7 z0~+}=uAuxo0lW?|B6?Z?gNUuvEcuBorCkQ1nvhO0B#33_nPs1p2wHAANaZeXav;v zgL>H45V(l#PTT&_St{k8YQTAplzENx)O)Kb)ydtks^yy4%XQwAK9(=4VV}nMp00CP zr#kUk0M0*OrQU1mVg1$Y?P@Cd?(@~2$MpA{(2nYMd)O}D_L{by%X$m*!<@0U&uz0V zsR{WXv*pj{0`|OcW-N%<@OEBf>-6*eelNj!edqvnGwb{WYkuTA>34Okn|E}O#4KW5VayKb;K<6<}bZqCW8g<#GU9+#? z{HpUgfR?o_b6v*cO6w1HD6;%FB**L)(q1Q-o)PWOGl%n>tQd3wvygwA1B+@8gq+P>E@MT`8Jq3*Q9eqozo8D~Z{`_IY{>cv#&+)joR?-SS!lWD zmj^x@;j!ok>O;mR?$0KF`2H7o9L5R-jxl&%S^62?kzNdN{6%?DR*bhf{xa$3kbi&3 z|2bR!BOq^%P8fD)Jgj@dQzHMY#S z;6nKLJwE-vo&$njY)qa~S;r$g?c>XmJPX=Z!#h&iKm9((US^C)*=Nk{LjHYi`QHlp z4u_nx%@JcC{lKRvALEcepF2>mC~MZqvFJAd^?*1n@+UV(LT}VM068zSO{_Jiw`>3EL!ke?s8*L)zY5c1zQw~_Yg@a?^9`Ey;!3Faa59^?$q0xE>;d9Ob5 zr~jvY(k5y5v{{pW2K(>l5PWV=-_N=U#*=&|&6Keq=aNqw?OLRqgic#KN`np<-jQAe%mVrVJ%GMIf1qC`T;pu{--7YoG|0KHjs8f9en1bPH_AiQ zhMQ_3{$oGSwF1t%dSiPaz-zJoai^0D#P{BDU~sKA7!wXjo^iIG<1x>MPG?)r^tHgf zv45_=p#KjALV&qG2*&m@jQ{_2eC3t(A>+ls9AK^wxNjcjbmwAC$vK$Qc@F1sO;`}X zdgnM&?&F+8U-E$TwAvn?_37)OzjvcwIIY({({h*VTNcKEF0K#6e1OQ_Yk!<$<9s2J zG2iCghW)$#T;9|L|L%sJ8@L>K?&G$8E`s!JU?Z>ySmFTJm~!75KtF#%Dsm_%h6!2f zaTe0<0ONwL0M`L_0k|HlC%|<9-1loaxox=(n*qqw$o~iAI|3Xla&FMf6>}fwiU`gn zitU#p?SVXv{Auss0-Pt{IFTv!;2VItK#T%J{uM+1W{ybYUykJMdn>T#oa|rZ?@LxW z-1d<_<299^WhDIsxhf$A-jEK>Gi3qyvzr zkw3?I3xTP?bQ{di0Hy%5fT_H7q@n}mL_FQ1TN7&NKt* z7QW%__=Q_m9mnP=n3Kz~eVFETTjmZNZdsKxx4G#!Azg1<9nAr(4!=5P`?z;Iqk>fL zw#Ss)+m_i@DqJ7Y(caO?<*ng1d)u<~SGzLq$y;4U3+?7umVJSYZTlAO=HV=FeZJdv zb5Y(=ACS3yo6mL~zuTHU-6oQrS-ak)ly#lX@Q>>-ni!FJyhq`TL(c6K)R4LDy@8y! zy*HFu?qHNHJ)S8dZsM!%9d&ZtW&O8`3j$B|o^Ud)<2#4aPV4(@n?p&_Gu+8>a|2S7mj~=W zyvOhTj{3lI&Hlmigmkk4=Jv1x=Jm8Z3wod4@9`I>h!Z!BccsKH2-vb^ZMOTJ`h?{Q z?HO>^fjz6vJ$^Ky>fmd0zRwxoS#I*NpjYYn-@bYAloCJBvu&IAzM~g+^dv=1zAhsGk0e0 z%sFSuy*o2!COvhJEGsR6?jhn;q8_N>&$80uS$6sXmXmpiod~mM(TjCi zSM=_89nJ1wwm`k!oEZ}wGZ?$Jp#@S%m_2PTHuERYx>talSM&?AwPKbYm z`$t*n2TD@5_bCmVC6q=kl&PZ@iKUQV!XvmPkY7dJJcKUdP%j1?`(AU_E&2*M?)g4% zl5SoHO-=UYa$h*MqH*T=uy878$XuFvDgJ)O#kdEckUg&Sa_$NBv7pab)O>-(Ez@N& z--#@GiAF}q!X#9X8~PDSv^pU-0rf^W4Y@xdSwrRsBJN*XT{wSqIoE&Y`Ox<0YL1 z4Jo{w;C#2}qSxIC@}p)L$29Uz_#`phct8wz_BS2+41N*Ddp>d5)R<*+&E#}^7&2iFMMDQkuE z$*Y844y+Wyl2!<-Qr7Bx0slqEe{953_gj@7*=ztEh#L7NJyf8i57JHYS{aL9R$eD6 z=zwG=J@9@4Jj^6s@Q#otE`RkpKz_W-$eDwa1izU#5TRs zeOv9B>o`YtFmNzCvO!=e8xXgG+?Cx@y*Rk;iFzRUsn2BK$zRd|(N}qGo21b(+FcTt zf3ffo^;Qjj)sTTusznLtql4>ZC8xFQq~EO{ps@KS&e?Q?c?A& zIg9ldE4K;i4<2fDFrVbcJMu60i9=s~InV@U^_ryJ>f+P|0h zug`3&m7}6;6u+idthTRB^cUvD;{Anh(?>sVkg=aPYH(zej2+!vJ$~MRHji>YT!Z+O zj+7>+xaVnu9@~QSLD9dkIN|{SFY1`Zf*SO}f#pJXk|%!Yi+vwxb<+&g40sDL0vZhc z)UNTr2iX{R>^7+R4qwBA0e<>y-vYgogtdAs2}lH@SL(Ace*+e}^l^kOGhk6G3|Rac zLzc8opCzo(JQK4@pM@ig;_uf!L-*uQX-e_p*XZN(W!mt6h+vm8N-Sr)3&fG>@s)rBk#s+=doelZ48yi2$iY@Z&!Z!MKVn5F9$bR^? zBU|s&i7oW#!aT-wWg`dMutA@6XP?tEpw1(#*}RG6@i)%u#1`L-I4r2^#Rhs&d&#SZ%biU_6d!$gM`A&y#l+u-?;RbA3Lb$PUx!MwNRzb zi!f8)jx$x?O;D#cxI%o2OZU0^&3Sqw zhuEoC&#+McwzC~iyJB2QpQjyfinD{2Y{z^B^Zv$M9lxeNJGWDIGbdPhoA}MZd0OC~ zU zcZ*`oZdQn2jPqdsq?Zle8{qRzlJ)9-LG&AMF5fh}h2F+*TgoPm?(|f-=^5!F9Y}7XX@PPP={8Rb z5Z}i6b&7%r6XTryO*&r+sW&(+q}~8v;HBWYgL6WgSloy=`DbB#tHFhFtp063^gQpQv=j#Q(8|t6)3nO&Ng7x={@4= zlL5lYtFcW#FFL5$j5sSPF5FuS65sv;VG9svy*AwqpmUzZAL`^gs4KBq!Sxt)Py`)Z zgsd6BQD8mLhtdFjVI9!3D1TX6NL_x`9?s`|4`=nw1?aq9&(ndz?2G&Kzl8kLaUL(7 zvqa}Fc^4mSO|rWn4xP(88P9p03J|;@|44+)|bBbe<~CQI+9LbvfFbPGPheZ=?B4|_F6`5%@~88t>HGN|4hg~9AR`W z5y?LZ^7}#lkqGlTvsa|^i%GAf2TH$Me)A$)asSM{b#!pPB-#P`gUHq;>3eh~nMwXf zT9Eqy!1)KQHqB?%HSi+I-w*E9z#IZ(Tmkt&7;*xF`0e(;7iRu7_;cVsi=xk-+5&8m# zvvtp+KirRYa0hadE(kY)D4-cV1Nx%!^L`Y3wF9Vp+_ji;E?CzR@{fi*bT&1eH8SjFN&Y`_;-bYhqA#ofbZ|AJl? z1HPyavvEKD;{G~)aK7g}Z9Iy*1n5Qlf!63*F6iKe z`#((z<^QQX0J#=CAGn(XRA;IE&O|-_7Viajl=C-e6Lj{(O!N)YpodxMp>;>0UCc)u z)C++?b^bHT0aOo=(&lx%Jm+#(x%{o?Xyq@#`|vjU1qH>rysxobEAvZ}yFB-D7kK66 z584Dk8|Qr%>A)BAkAwVUAv^VTQ*f5+WXSKG7FuUG_d)r80r}^l z{7FxQ`M{^d189wS$9tdWdp+#!#whnY1yRb)^!&}3b~S206!&FZ;8&8r0o-eW^}rh7 zNAzdz=;x{Yw*zZ{_2>guUfN%$KidB?wDkqRd|)-=zDN9k)<}Mei}L1qLl<4~ZNT5} zN* zjGwFg^0f5;bspbGfA4c`AU`i#&_2M%qGSbqC(xNa>wtkIKWL51pW@SnKA*k|d0o)> zyq^Ckb9yh7fzGfdP9T%01@E)qeg`1A>3lN!UZC$F-v5C1w@!zBYlr{B-crQMN z*WQ2sH|emJ*IEtKYM@pFwHm0^K&=L9HSi)Dc=bCG{M8&vwA+bfYd2o;(`*4l+pG8@ zZQAI>gz~^HAGGmFF@9d_Yi@bOY|;wuC7ML{zS_NB`bfK+m^|SACVez}z#t)hPYKcm zs&1Iv&^O7Ksk794M-b2~mSm*$*G4>uxDU@|vH75XUeeHenOW=<-D_!avI*`gTxt`| zjXJ9mHPPI^(*Lwigat4fDBCB(3^%n`tp;i}P^*Dj4b*C&Rs*#fsMWwr(Lma-Tg6K! z*NG>?MvAbX_}h?%X(2Z=YxN>r6uD2_ne@S*pnUQYyhS5`j&b{L1xAo_EICAVubdh;Lv= zxdS$aCvvinWS&2^DnD_Z;(F9VAq{o`zrZeIKd>0<;r|WWL1o3{A;*8zZZ7LmET6kh zPYom+u>i>OfNTRZFU7H(tYhrg_^B*vu^x+nT^#HIE800pZnFJ53i}pM$immMMxdRL z92Mny74GCsop>JUXM=q+FUEl9SeAK_5M6RP_vHOE2WFJOp06}!iA){y9oaLIT_Z;c z%7W}M$VQ3e;5KWZcOZ*D}y201pTtPym7nj(lVtDk~iG`uerIw7BXWaFE(N`y_64E8~= z`GOr#@*45pk?)0b5u5uySh>)N9p7Qh{SUz&sLbXGc0`i>R|;$b$v(1z9K=7jd4deF zu(9LoTwsgy+;yqK`gU>DP%ClcG$Xzq0qJfd+rxyFvipZO>A;Rlf^KQb27TDz2rOiM zKelPzY<4oxoY$i&_Dp0C1xR*FWZM`Adp=5+Kr)bxBjk!qSZgS^AJ{>h=42*&n$6PF z+WEm18u^c`5t_$;C)|R4DEoPnObt}EN!lc;PyV3K&g^asdnA$KRgD8XCdm#-vTGvS zqrTw+{Y&*(#4;V&9qF_9)p{(vLLAAKC{}AnMD|2vmqh-O zeNw<2J+{L~mj!*RTe5$?4qNT{#)wABubO@_#JU~$AzKmRi)@9OYwsIWlmmW-z&pSm zfWg-2*oPnV_W9P$Z2$M(EnwHvsbusp8}*k%ZPfl#I;;0A?x2oX(M}x>>|WGCy>fbI z^+X3NHSB<34`ZWV;MqkTh&Yie+o?mAwujw|N)7uU^=JLNtH**a^6CtmnU3nvW$k#J zeg5s$>%BX&u_G+l=L2li)4y)_0C_DQ@KKNPu=DT*Pocmg;5T3h(4-RoyuHAm33$WX9tsb4-)sCIq-WIkhO4zs%$kxSFa}T>4*n6TTJaVOripdivbAIKv+64=G_t*lpO5_)GmshwC?bO(WSPr;a)uEvw(}1pJeYjeM4?IggL^`gJb_ ziY@ZPjay^{w@{wlW9WDx*udp{kiKjBzWN{CN@_joMuN#VHxo@<^J9%a{w=`ZOT_u+ zf+qZ9gnuc8=f$-8%=-<4_AC%5ZPuY z?th1E3)u@yN)ME^0qt;LU0p+x<$2c%!^VNj0@@f@2+YCS&n4$VbX{rPGuFM%#X8XM z3KCn4hmD6HY&_;8e9_rp9XG58r0{tNUjiEt3in3%JfyJ{uqS>%%j&|dnD#uc+womY zuZOkBy@av{zqH2vA&>=31$f;6<#8#ECtZ+RJRd0X^i%iBX$|X4fNVOZ!=_~{_?v{a ziIcEa&l_v>UEn_{b+3%ptj;(etmlGoFIv}%@RIlS^hu5qfM}G&Z%t%CK3q=F#=t6IIrv9}aa=iN!+;V$4GYjMRDNMi%wNPK{n>HlU- zTkgi1+i+TUOfuwzG}s6|27}kLz%gJgP@kR!eP$U@Jh~_!pgo`hI;RH7j5EUOeGN=b z534r=wBME3CJvWl-k5SFrpXM@_pz4P9{y8+ndxD5`(iDv37+ku1^$LH9gA(8kc`LSPp7T6{K0*Y(a}Ghfv8xo8&)vF_Cs z<=}HV5NqWFgr#T`Us9fMSIb8w{x4|wSDu9Kbz$?tpCNq%I?to`igH3%R0jNe8uSgo z8<+@LW}FYvrS{;-+rvS{G_>cj@b^M{AloLgT^UYkf$HEHN~cQM-;8d<(}u1?iGTD- z@6j``;o<(|{;QXiFR0WWmVwXj&jssDEQa2}KiQnELw_>??O`Uh2b9k;=>H4K8}4fL zPhk?DXvc#AZhsCro*18=dC^O&!w#TSpDocIT7dty=r1j>cFY26rCVK&eq$`P2T*U& zzLXc-HRcEF=4%}HMdhgc9CXj?1*j3~(*j^N;E#Ta*MGR5$rHXsWYGIdt232(l{7dP-4hsIb{y~j^<-k&41K^;Q>u=4! zq%-hKWzh++1grsW>kL|@-;tiFr7u?l&^48Tgj&77T=mwbR;z(p4ZLOzXci%94pPEl zQrb0H{|QQqwUEz!THOU&Osl#;vng-j038_y>PO4KK%pliE!D3;bKx&y$1GYlCfE9l zL~-3FdfC*f#6)vnf?ucXvX$w5qPXr;xZ0`@r;H^&;P<9|c>WZD?wR@-_fotUqqI&- z0lW*8t<&P8F7T+mYBf-+f&WSkWS$Eae~tb|j9yfAoZwXmg!yDK^HL1t_Q&14xKrZOMe@qO{T32$+Jv3FuKq?d|$v)ts>!UgP!dJc2Bu`%Cw zxV-QCzS!GORP%X5%;)lb6)|r}dn`sT5lc`me4j(`kpUgGp}f{iY9jyWpS?xQKS_Ia zRr{L@^L2a-JmEXRhvx5&Zq|8pY>N&%vRRiM-z>1iE$vwJj|13=tpcC-llB|r`x$Q0 z%nyQJ+Hde7&HZ72!K@>zHG8ZjFMo1bK>j9}k1`oHpu6H9@AWg@?rnG?XaVN#mK!|6 zd|WALDfav;3HR46jb1KOM=aM@!ynW@jk!9GNUdKu`7PB~@1CXmaF?Gh=I`o6={I$1 zf;l_d%a7*lXrDh5fRCw@e&BA3{5N1fydRuKSPb94w9)NtbK0>vZWc5bhy5tc*^w>g zEM|2(wq#NlHpSVJg)P_kAKKK0t(mD}o{nAFu7w>~@`g6-_z&jn@a8sb)$|SzX1glb z@~I}9di8#<$H*Zz9+Pu^>U=RuHe4o)gi;{z9za zW25r{#x1Xenp)k6ZtIg7RKM`tPRuRrl#R{YD_WDhPrER8!|NyLR~Uo+7IQvc826iu zF+QK{$cFBi@A1L7>y-S2w>;tJOJlMaf2H}8`FN%`rHyfOdDdPWSBeYs4Kdll^=aOU z=6h(aWHHMBjeSDIDH!9Q znH6C)24kMy(1!=+k|d|Fv+^13=@LFIB()YkK-S%R{3JHM(HR#xPQ% z#%M?hhE!lihanch_lO{L2kpari?#rBSds5q98|r975XNEB54V5R&&uf1r19mfUAUH72=!uO5{TE5O|m#y+qcO z(podQiM(IGz9PO!IX?$ih@8&X5#&d=ciMD5@vCd6BL-hhUS9|6-LPhHwWy|VX8bNK zOk67blhO#ESN=Yn{}>jG0TLK$mK@reJfgFZJG}Dvo^#&*M{mS ze?8i(`oY| zrvl~w$OvyRH6z?;dS-;tw6xHAuE%x=J|5vj z9?hDkGkbOFU>_p{z1 znlV!vH!Vl8uOMWb07in4gkLlFLJ@YiJiKCDQYr~cjjkm?KQ?Hwq94!@A?AW57T?st z9fgZn#N#A52$O`dLJvVvA(X-0%>WAZy6x~mB?^9inm)LlfryIq?z`H%z8C=)EMW9Y zF?X4h4iyUR9lCB^p&|ZL*w;|5G$@nKf94Dmh6rB?!-V&72+0tkzc2tBR(=KgneYMJ zbj9lwVTfJi?wbDVf#@RA^AbEz3T`N)4?xGG#QNjTS#U=}w41jGOJ(WLTIkE|mV2Vq zMPVSGbVnIaLK%928+*YOPt(;8{$qr3xbuKS@AK3t4+^zL{SeFY8ph*zB4>Am(`6xa z)aBECeg$<$!!G1r_9S z6Fl*M0{+~Qh8JQvavuBPzlajChp&`cxz+^1jJJ@X2&Fpii9D%3m&=G#g;2_Z(6i4} z*YS$UP9B~v?ry!5mYpq>3MaQQ?v5^QV|yw8HS`0Oty1CXW$)%_Ki=KVsh4t^lc%yz z@A`Fm+IxCBxjKxWra%I2p1qWlJluMCj``Zj)!tL(>N3W|-P7IKOEt#bwTHc@Yv;+9 zN`*CU}93-u>LijC1nn*`=bNHX72KVDIJP;4T3x!IhWt0Nx>ghRY z!UT5@FDJ(VLlpfTdnsqwI$1b(x3}!7a&UIAQdv7Y*{U4u>}^!mw$6?=7WS4l_ICE( z$}W}DshE*%XUm%8Wa-?kd$%#&yQyp~oGew=U9Id?b~X+UDqAZ@3+HZ*7LNAaZJ&|T zFc%bRP4aT;YGG+_U5ob9ZhslJ-zV{PBnxvQ;{mCDA-!A@mu zZEdM?u(unda_na3Xl-w8-_6OYlJa_Xsivaz?DAOMq88CbY7!W^Iil?f<-&zBS6-Rx rewoXFe>A?pqcwaziP?P>Yq#EJ> diff --git a/test/functional/manifest_test.go b/test/functional/manifest_test.go deleted file mode 100644 index d7be25b1d2..0000000000 --- a/test/functional/manifest_test.go +++ /dev/null @@ -1,3 +0,0 @@ -package functional - -import _ "github.com/Microsoft/hcsshim/test/functional/manifest" diff --git a/test/functional/test.go b/test/functional/test.go deleted file mode 100644 index e0e8f0545b..0000000000 --- a/test/functional/test.go +++ /dev/null @@ -1,49 +0,0 @@ -package functional - -import ( - "context" - "os" - "os/exec" - "strconv" - "time" - - "github.com/Microsoft/hcsshim/internal/cow" - "github.com/Microsoft/hcsshim/internal/hcsoci" - "github.com/Microsoft/hcsshim/internal/resources" - "github.com/sirupsen/logrus" -) - -var pauseDurationOnCreateContainerFailure time.Duration - -func init() { - if len(os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_DEBUG")) > 0 { - logrus.SetLevel(logrus.DebugLevel) - logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) - } - - // This allows for debugging a utility VM. - s := os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES") - if s != "" { - if t, err := strconv.Atoi(s); err == nil { - pauseDurationOnCreateContainerFailure = time.Duration(t) * time.Minute - } - } - - // Try to stop any pre-existing compute processes - cmd := exec.Command("powershell", `get-computeprocess | stop-computeprocess -force`) - _ = cmd.Run() - -} - -func CreateContainerTestWrapper(ctx context.Context, options *hcsoci.CreateOptions) (cow.Container, *resources.Resources, error) { - if pauseDurationOnCreateContainerFailure != 0 { - options.DoNotReleaseResourcesOnFailure = true - } - s, r, err := hcsoci.CreateContainer(ctx, options) - if err != nil { - logrus.Warnf("Test is pausing for %s for debugging CreateContainer failure", pauseDurationOnCreateContainerFailure) - time.Sleep(pauseDurationOnCreateContainerFailure) - _ = resources.ReleaseResources(ctx, r, options.HostingSystem, true) - } - return s, r, err -} diff --git a/test/functional/utilities/layerfolders.go b/test/functional/utilities/layerfolders.go deleted file mode 100644 index 0d009b0718..0000000000 --- a/test/functional/utilities/layerfolders.go +++ /dev/null @@ -1,53 +0,0 @@ -package testutilities - -import ( - "bytes" - "encoding/json" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" -) - -var imageLayers map[string][]string - -func init() { - imageLayers = make(map[string][]string) -} - -func LayerFolders(t *testing.T, imageName string) []string { - if _, ok := imageLayers[imageName]; !ok { - imageLayers[imageName] = getLayers(t, imageName) - } - return imageLayers[imageName] -} - -func getLayers(t *testing.T, imageName string) []string { - cmd := exec.Command("docker", "inspect", imageName, "-f", `"{{.GraphDriver.Data.dir}}"`) - var out bytes.Buffer - cmd.Stdout = &out - if err := cmd.Run(); err != nil { - t.Skipf("Failed to find layers for %q. Check docker images", imageName) - } - imagePath := strings.Replace(strings.TrimSpace(out.String()), `"`, ``, -1) - layers := getLayerChain(t, imagePath) - return append([]string{imagePath}, layers...) -} - -func getLayerChain(t *testing.T, layerFolder string) []string { - jPath := filepath.Join(layerFolder, "layerchain.json") - content, err := os.ReadFile(jPath) - if os.IsNotExist(err) { - t.Fatalf("layerchain not found") - } else if err != nil { - t.Fatalf("failed to read layerchain") - } - - var layerChain []string - err = json.Unmarshal(content, &layerChain) - if err != nil { - t.Fatalf("failed to unmarshal layerchain") - } - return layerChain -} diff --git a/test/functional/utilities/requiresbuild.go b/test/functional/utilities/requiresbuild.go deleted file mode 100644 index 47930994d0..0000000000 --- a/test/functional/utilities/requiresbuild.go +++ /dev/null @@ -1,19 +0,0 @@ -package testutilities - -import ( - "testing" - - "github.com/Microsoft/hcsshim/osversion" -) - -func RequiresBuild(t *testing.T, b uint16) { - if osversion.Build() < b { - t.Skipf("Requires build %d+", b) - } -} - -func RequiresExactBuild(t *testing.T, b uint16) { - if osversion.Build() != b { - t.Skipf("Requires exact build %d", b) - } -} diff --git a/test/functional/utilities/tempdir.go b/test/functional/utilities/tempdir.go deleted file mode 100644 index 574e31279d..0000000000 --- a/test/functional/utilities/tempdir.go +++ /dev/null @@ -1,15 +0,0 @@ -package testutilities - -import ( - "os" - "testing" -) - -// CreateTempDir creates a temporary directory -func CreateTempDir(t *testing.T) string { - tempDir, err := os.MkdirTemp("", "test") - if err != nil { - t.Fatalf("failed to create temporary directory: %s", err) - } - return tempDir -} diff --git a/test/functional/uvm_mem_backingtype_test.go b/test/functional/uvm_mem_backingtype_test.go index 2eeda361c5..b2c83a8cc5 100644 --- a/test/functional/uvm_mem_backingtype_test.go +++ b/test/functional/uvm_mem_backingtype_test.go @@ -1,4 +1,5 @@ -//go:build functional || uvmmem +//go:build windows && (functional || uvmmem) +// +build windows // +build functional uvmmem package functional @@ -6,23 +7,23 @@ package functional import ( "context" "io" - "os" "testing" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" "github.com/sirupsen/logrus" + + tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) func runMemStartLCOWTest(t *testing.T, opts *uvm.OptionsLCOW) { - u := testutilities.CreateLCOWUVMFromOpts(context.Background(), t, opts) + u := tuvm.CreateAndStartLCOWFromOpts(context.Background(), t, opts) u.Close() } func runMemStartWCOWTest(t *testing.T, opts *uvm.OptionsWCOW) { - u, _, scratchDir := testutilities.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") - defer os.RemoveAll(scratchDir) + u, _, _ := tuvm.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") u.Close() } @@ -46,7 +47,7 @@ func runMemTests(t *testing.T, os string) { wopts.EnableDeferredCommit = bt.enableDeferredCommit runMemStartWCOWTest(t, wopts) } else { - lopts := uvm.NewDefaultOptionsLCOW(t.Name(), "") + lopts := defaultLCOWOptions(t) lopts.MemorySizeInMB = 512 lopts.AllowOvercommit = bt.allowOvercommit lopts.EnableDeferredCommit = bt.enableDeferredCommit @@ -56,12 +57,18 @@ func runMemTests(t *testing.T, os string) { } func TestMemBackingTypeWCOW(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureWCOW) runMemTests(t, "windows") } func TestMemBackingTypeLCOW(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW) runMemTests(t, "linux") } @@ -88,21 +95,30 @@ func runBenchMemStartLcowTest(b *testing.B, allowOvercommit bool, enableDeferred } func BenchmarkMemBackingTypeVirtualLCOW(b *testing.B) { - //testutilities.RequiresBuild(t, osversion.RS5) + b.Skip("not yet updated") + + require.Build(b, osversion.RS5) + requireFeatures(b, featureLCOW) logrus.SetOutput(io.Discard) runBenchMemStartLcowTest(b, true, false) } func BenchmarkMemBackingTypeVirtualDeferredLCOW(b *testing.B) { - //testutilities.RequiresBuild(t, osversion.RS5) + b.Skip("not yet updated") + + require.Build(b, osversion.RS5) + requireFeatures(b, featureLCOW) logrus.SetOutput(io.Discard) runBenchMemStartLcowTest(b, true, true) } func BenchmarkMemBackingTypePhyscialLCOW(b *testing.B) { - //testutilities.RequiresBuild(t, osversion.RS5) + b.Skip("not yet updated") + + require.Build(b, osversion.RS5) + requireFeatures(b, featureLCOW) logrus.SetOutput(io.Discard) runBenchMemStartLcowTest(b, false, false) diff --git a/test/functional/uvm_memory_test.go b/test/functional/uvm_memory_test.go index aad97c271b..748ddde8f6 100644 --- a/test/functional/uvm_memory_test.go +++ b/test/functional/uvm_memory_test.go @@ -1,26 +1,32 @@ +//go:build windows && functional + package functional import ( "context" - "os" "testing" "time" "github.com/Microsoft/hcsshim/internal/memory" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + + "github.com/Microsoft/hcsshim/test/internal/require" + tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) func TestUVMMemoryUpdateLCOW(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW) ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) defer cancel() - opts := uvm.NewDefaultOptionsLCOW(t.Name(), "") + opts := defaultLCOWOptions(t) opts.MemorySizeInMB = 1024 * 2 - u := testutilities.CreateLCOWUVMFromOpts(ctx, t, opts) + u := tuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) defer u.Close() newMemorySize := uint64(opts.MemorySizeInMB/2) * memory.MiB @@ -38,7 +44,10 @@ func TestUVMMemoryUpdateLCOW(t *testing.T) { } func TestUVMMemoryUpdateWCOW(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureWCOW) ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) defer cancel() @@ -46,8 +55,7 @@ func TestUVMMemoryUpdateWCOW(t *testing.T) { opts := uvm.NewDefaultOptionsWCOW(t.Name(), "") opts.MemorySizeInMB = 1024 * 2 - u, _, uvmScratchDir := testutilities.CreateWCOWUVMFromOptsWithImage(ctx, t, opts, "mcr.microsoft.com/windows/nanoserver:1909") - defer os.RemoveAll(uvmScratchDir) + u, _, _ := tuvm.CreateWCOWUVMFromOptsWithImage(ctx, t, opts, "mcr.microsoft.com/windows/nanoserver:1909") defer u.Close() newMemoryInBytes := uint64(opts.MemorySizeInMB/2) * memory.MiB diff --git a/test/functional/uvm_plannine_test.go b/test/functional/uvm_plannine_test.go index 714a17b050..ae3aa5c392 100644 --- a/test/functional/uvm_plannine_test.go +++ b/test/functional/uvm_plannine_test.go @@ -1,5 +1,5 @@ -//go:build functional || uvmp9 -// +build functional uvmp9 +//go:build windows && functional +// +build windows,functional // This file isn't called uvm_plan9_test.go as go test skips when a number is in it... go figure (pun intended) @@ -15,16 +15,23 @@ import ( "github.com/Microsoft/hcsshim/internal/hcs" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + + "github.com/Microsoft/hcsshim/test/internal/require" + testuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) // TestPlan9 tests adding/removing Plan9 shares to/from a v2 Linux utility VM // TODO: This is very basic. Need multiple shares and so-on. Can be iterated on later. func TestPlan9(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW, featurePlan9) + ctx := context.Background() - vm := testutilities.CreateLCOWUVM(context.Background(), t, t.Name()) + vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, defaultLCOWOptions(t)) defer vm.Close() + testuvm.SetSecurityPolicy(ctx, t, vm, "") dir := t.TempDir() var iterations uint32 = 64 @@ -46,12 +53,17 @@ func TestPlan9(t *testing.T) { } func TestPlan9_Writable(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) + // t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW, featurePlan9) + ctx := context.Background() - opts := uvm.NewDefaultOptionsLCOW(t.Name(), "") + opts := defaultLCOWOptions(t) opts.NoWritableFileShares = true - vm := testutilities.CreateLCOWUVMFromOpts(context.Background(), t, opts) + vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) defer vm.Close() + testuvm.SetSecurityPolicy(ctx, t, vm, "") dir := t.TempDir() diff --git a/test/functional/uvm_properties_test.go b/test/functional/uvm_properties_test.go index c9f732e4f1..5bf784ddc5 100644 --- a/test/functional/uvm_properties_test.go +++ b/test/functional/uvm_properties_test.go @@ -1,21 +1,25 @@ -//go:build functional || uvmproperties +//go:build windows && (functional || uvmproperties) +// +build windows // +build functional uvmproperties package functional import ( "context" - "os" "testing" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" + tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) func TestPropertiesGuestConnection_LCOW(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) + t.Skip("not yet updated") - uvm := testutilities.CreateLCOWUVM(context.Background(), t, t.Name()) + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW) + + uvm := tuvm.CreateAndStartLCOWFromOpts(context.Background(), t, defaultLCOWOptions(t)) defer uvm.Close() p, gc := uvm.Capabilities() @@ -27,9 +31,12 @@ func TestPropertiesGuestConnection_LCOW(t *testing.T) { } func TestPropertiesGuestConnection_WCOW(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) - uvm, _, uvmScratchDir := testutilities.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") - defer os.RemoveAll(uvmScratchDir) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureWCOW) + + uvm, _, _ := tuvm.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") defer uvm.Close() p, gc := uvm.Capabilities() diff --git a/test/functional/uvm_scratch_test.go b/test/functional/uvm_scratch_test.go index 47ae7a79d8..bd3d7b07fe 100644 --- a/test/functional/uvm_scratch_test.go +++ b/test/functional/uvm_scratch_test.go @@ -1,4 +1,5 @@ -//go:build functional || uvmscratch +//go:build windows && (functional || uvmscratch) +// +build windows // +build functional uvmscratch package functional @@ -12,14 +13,18 @@ import ( "github.com/Microsoft/hcsshim/internal/lcow" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" + tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) func TestScratchCreateLCOW(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) - tempDir := t.TempDir() + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW, featureScratch) - firstUVM := testutilities.CreateLCOWUVM(context.Background(), t, "TestCreateLCOWScratch") + tempDir := t.TempDir() + firstUVM := tuvm.CreateAndStartLCOW(context.Background(), t, "TestCreateLCOWScratch") defer firstUVM.Close() cacheFile := filepath.Join(tempDir, "cache.vhdx") @@ -36,7 +41,7 @@ func TestScratchCreateLCOW(t *testing.T) { t.Fatalf("cacheFile wasn't created!") } - targetUVM := testutilities.CreateLCOWUVM(context.Background(), t, "TestCreateLCOWScratch_target") + targetUVM := tuvm.CreateAndStartLCOW(context.Background(), t, "TestCreateLCOWScratch_target") defer targetUVM.Close() // A non-cached create diff --git a/test/functional/uvm_scsi_test.go b/test/functional/uvm_scsi_test.go index b25a0db38a..38c233219d 100644 --- a/test/functional/uvm_scsi_test.go +++ b/test/functional/uvm_scsi_test.go @@ -1,4 +1,5 @@ -//go:build functional || uvmscsi +//go:build windows && (functional || uvmscsi) +// +build windows // +build functional uvmscsi package functional @@ -18,15 +19,21 @@ import ( "github.com/Microsoft/hcsshim/internal/lcow" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + testutilities "github.com/Microsoft/hcsshim/test/internal" + "github.com/Microsoft/hcsshim/test/internal/require" + tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" "github.com/sirupsen/logrus" ) // TestSCSIAddRemovev2LCOW validates adding and removing SCSI disks // from a utility VM in both attach-only and with a container path. func TestSCSIAddRemoveLCOW(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) - u := testutilities.CreateLCOWUVM(context.Background(), t, t.Name()) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW, featureSCSI) + + u := tuvm.CreateAndStartLCOWFromOpts(context.Background(), t, defaultLCOWOptions(t)) defer u.Close() testSCSIAddRemoveMultiple(t, u, `/run/gcs/c/0/scsi`, "linux", []string{}) @@ -36,10 +43,13 @@ func TestSCSIAddRemoveLCOW(t *testing.T) { // TestSCSIAddRemoveWCOW validates adding and removing SCSI disks // from a utility VM in both attach-only and with a container path. func TestSCSIAddRemoveWCOW(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureWCOW, featureSCSI) + // TODO make the image configurable to the build we're testing on - u, layers, uvmScratchDir := testutilities.CreateWCOWUVM(context.Background(), t, t.Name(), "mcr.microsoft.com/windows/nanoserver:1903") - defer os.RemoveAll(uvmScratchDir) + u, layers, _ := tuvm.CreateWCOWUVM(context.Background(), t, t.Name(), "mcr.microsoft.com/windows/nanoserver:1903") defer u.Close() testSCSIAddRemoveSingle(t, u, `c:\`, "windows", layers) @@ -89,7 +99,6 @@ func testSCSIAddRemoveSingle(t *testing.T, u *uvm.UtilityVM, pathPrefix string, } else { tempDir = testutilities.CreateLCOWBlankRWLayer(context.Background(), t) } - defer os.RemoveAll(tempDir) disks[i] = filepath.Join(tempDir, `sandbox.vhdx`) } @@ -140,7 +149,6 @@ func testSCSIAddRemoveMultiple(t *testing.T, u *uvm.UtilityVM, pathPrefix string } else { tempDir = testutilities.CreateLCOWBlankRWLayer(context.Background(), t) } - defer os.RemoveAll(tempDir) disks[i] = filepath.Join(tempDir, `sandbox.vhdx`) } @@ -214,8 +222,12 @@ func testSCSIAddRemoveMultiple(t *testing.T, u *uvm.UtilityVM, pathPrefix string } func TestParallelScsiOps(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) - u := testutilities.CreateLCOWUVM(context.Background(), t, t.Name()) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW, featureSCSI) + + u := tuvm.CreateAndStartLCOWFromOpts(context.Background(), t, defaultLCOWOptions(t)) defer u.Close() // Create a sandbox to use diff --git a/test/functional/uvm_virtualdevice_test.go b/test/functional/uvm_virtualdevice_test.go index 73f8ede868..59c6c581c8 100644 --- a/test/functional/uvm_virtualdevice_test.go +++ b/test/functional/uvm_virtualdevice_test.go @@ -1,5 +1,5 @@ -//go:build functional -// +build functional +//go:build windows && functional +// +build windows,functional package functional @@ -12,7 +12,8 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/require" + tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) const lcowGPUBootFilesPath = "C:\\ContainerPlat\\LinuxBootFiles\\nvidiagpu" @@ -30,7 +31,11 @@ func findTestVirtualDevice() (string, error) { } func TestVirtualDevice(t *testing.T) { - testutilities.RequiresBuild(t, osversion.V20H1) + t.Skip("not yet updated") + + require.Build(t, osversion.V20H1) + requireFeatures(t, featureLCOW) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -54,7 +59,7 @@ func TestVirtualDevice(t *testing.T) { opts.BootFilesPath = lcowGPUBootFilesPath // create test uvm and ensure we can assign and remove the device - vm := testutilities.CreateLCOWUVMFromOpts(ctx, t, opts) + vm := tuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) defer vm.Close() vpci, err := vm.AssignDevice(ctx, testDeviceInstanceID, 0, "") if err != nil { diff --git a/test/functional/uvm_vpmem_test.go b/test/functional/uvm_vpmem_test.go index e437a488f3..79e62095d0 100644 --- a/test/functional/uvm_vpmem_test.go +++ b/test/functional/uvm_vpmem_test.go @@ -1,4 +1,5 @@ -//go:build functional || uvmvpmem +//go:build windows && (functional || uvmvpmem) +// +build windows // +build functional uvmvpmem package functional @@ -11,16 +12,22 @@ import ( "github.com/Microsoft/hcsshim/internal/copyfile" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/layers" + "github.com/Microsoft/hcsshim/test/internal/require" + tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) // TestVPMEM tests adding/removing VPMem Read-Only layers from a v2 Linux utility VM func TestVPMEM(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) - alpineLayers := testutilities.LayerFolders(t, "alpine") + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureLCOW, featureVPMEM) + + alpineLayers := layers.LayerFolders(t, "alpine") ctx := context.Background() - u := testutilities.CreateLCOWUVM(ctx, t, t.Name()) + u := tuvm.CreateAndStartLCOW(ctx, t, t.Name()) defer u.Close() var iterations uint32 = uvm.MaxVPMEMCount diff --git a/test/functional/uvm_vsmb_test.go b/test/functional/uvm_vsmb_test.go index 368392f054..74315cb104 100644 --- a/test/functional/uvm_vsmb_test.go +++ b/test/functional/uvm_vsmb_test.go @@ -1,4 +1,5 @@ -//go:build functional || uvmvsmb +//go:build windows && (functional || uvmvsmb) +// +build windows // +build functional uvmvsmb package functional @@ -6,20 +7,24 @@ package functional import ( "context" "errors" - "os" "testing" "github.com/Microsoft/hcsshim/internal/hcs" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + + "github.com/Microsoft/hcsshim/test/internal/require" + tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) // TestVSMB tests adding/removing VSMB layers from a v2 Windows utility VM func TestVSMB(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) - uvm, _, uvmScratchDir := testutilities.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") - defer os.RemoveAll(uvmScratchDir) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureWCOW, featureVSMB) + + uvm, _, _ := tuvm.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") defer uvm.Close() dir := t.TempDir() @@ -43,12 +48,14 @@ func TestVSMB(t *testing.T) { // TODO: VSMB for mapped directories func TestVSMB_Writable(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureWCOW, featureVSMB) opts := uvm.NewDefaultOptionsWCOW(t.Name(), "") opts.NoWritableFileShares = true - vm, _, uvmScratchDir := testutilities.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") - defer os.RemoveAll(uvmScratchDir) + vm, _, _ := tuvm.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") defer vm.Close() dir := t.TempDir() diff --git a/test/functional/wcow_test.go b/test/functional/wcow_test.go index e182d9f4fb..256a79da5d 100644 --- a/test/functional/wcow_test.go +++ b/test/functional/wcow_test.go @@ -1,4 +1,5 @@ -//go:build functional || wcow +//go:build windows && (functional || wcow) +// +build windows // +build functional wcow package functional @@ -23,7 +24,8 @@ import ( "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/internal/wcow" "github.com/Microsoft/hcsshim/osversion" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/layers" + "github.com/Microsoft/hcsshim/test/internal/require" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -363,7 +365,11 @@ func generateShimLayersStruct(t *testing.T, imageLayers []string) []hcsshim.Laye // Argon through HCSShim interface (v1) func TestWCOWArgonShim(t *testing.T) { - imageLayers := testutilities.LayerFolders(t, imageName) + t.Skip("not yet updated") + + requireFeatures(t, featureWCOW) + + imageLayers := layers.LayerFolders(t, imageName) argonShimMounted := false argonShimScratchDir := t.TempDir() @@ -372,9 +378,6 @@ func TestWCOWArgonShim(t *testing.T) { } hostRWSharedDirectory, hostROSharedDirectory := createTestMounts(t) - defer os.RemoveAll(hostRWSharedDirectory) - defer os.RemoveAll(hostROSharedDirectory) - layers := generateShimLayersStruct(t, imageLayers) // For cleanup on failure @@ -428,7 +431,11 @@ func TestWCOWArgonShim(t *testing.T) { // Xenon through HCSShim interface (v1) func TestWCOWXenonShim(t *testing.T) { - imageLayers := testutilities.LayerFolders(t, imageName) + t.Skip("not yet updated") + + requireFeatures(t, featureWCOW) + + imageLayers := layers.LayerFolders(t, imageName) xenonShimScratchDir := t.TempDir() if err := wclayer.CreateScratchLayer(context.Background(), xenonShimScratchDir, imageLayers); err != nil { @@ -436,9 +443,6 @@ func TestWCOWXenonShim(t *testing.T) { } hostRWSharedDirectory, hostROSharedDirectory := createTestMounts(t) - defer os.RemoveAll(hostRWSharedDirectory) - defer os.RemoveAll(hostROSharedDirectory) - uvmImagePath, err := uvmfolder.LocateUVMFolder(context.Background(), imageLayers) if err != nil { t.Fatalf("LocateUVMFolder failed %s", err) @@ -497,7 +501,11 @@ func generateWCOWOciTestSpec(t *testing.T, imageLayers []string, scratchPath, ho // Argon through HCSOCI interface (v1) func TestWCOWArgonOciV1(t *testing.T) { - imageLayers := testutilities.LayerFolders(t, imageName) + t.Skip("not yet updated") + + requireFeatures(t, featureWCOW) + + imageLayers := layers.LayerFolders(t, imageName) argonOci1Mounted := false argonOci1ScratchDir := t.TempDir() if err := wclayer.CreateScratchLayer(context.Background(), argonOci1ScratchDir, imageLayers); err != nil { @@ -505,9 +513,6 @@ func TestWCOWArgonOciV1(t *testing.T) { } hostRWSharedDirectory, hostROSharedDirectory := createTestMounts(t) - defer os.RemoveAll(hostRWSharedDirectory) - defer os.RemoveAll(hostROSharedDirectory) - // For cleanup on failure var argonOci1Resources *resources.Resources var argonOci1 cow.Container @@ -544,7 +549,11 @@ func TestWCOWArgonOciV1(t *testing.T) { // Xenon through HCSOCI interface (v1) func TestWCOWXenonOciV1(t *testing.T) { - imageLayers := testutilities.LayerFolders(t, imageName) + t.Skip("not yet updated") + + requireFeatures(t, featureWCOW) + + imageLayers := layers.LayerFolders(t, imageName) xenonOci1Mounted := false xenonOci1ScratchDir := t.TempDir() @@ -553,9 +562,6 @@ func TestWCOWXenonOciV1(t *testing.T) { } hostRWSharedDirectory, hostROSharedDirectory := createTestMounts(t) - defer os.RemoveAll(hostRWSharedDirectory) - defer os.RemoveAll(hostROSharedDirectory) - // TODO: This isn't currently used. // uvmImagePath, err := uvmfolder.LocateUVMFolder(imageLayers) // if err != nil { @@ -599,8 +605,12 @@ func TestWCOWXenonOciV1(t *testing.T) { // Argon through HCSOCI interface (v2) func TestWCOWArgonOciV2(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) - imageLayers := testutilities.LayerFolders(t, imageName) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureWCOW) + + imageLayers := layers.LayerFolders(t, imageName) argonOci2Mounted := false argonOci2ScratchDir := t.TempDir() @@ -609,9 +619,6 @@ func TestWCOWArgonOciV2(t *testing.T) { } hostRWSharedDirectory, hostROSharedDirectory := createTestMounts(t) - defer os.RemoveAll(hostRWSharedDirectory) - defer os.RemoveAll(hostROSharedDirectory) - // For cleanup on failure var argonOci2Resources *resources.Resources var argonOci2 cow.Container @@ -649,8 +656,12 @@ func TestWCOWArgonOciV2(t *testing.T) { // Xenon through HCSOCI interface (v2) func TestWCOWXenonOciV2(t *testing.T) { - testutilities.RequiresBuild(t, osversion.RS5) - imageLayers := testutilities.LayerFolders(t, imageName) + t.Skip("not yet updated") + + require.Build(t, osversion.RS5) + requireFeatures(t, featureWCOW) + + imageLayers := layers.LayerFolders(t, imageName) xenonOci2Mounted := false xenonOci2UVMCreated := false @@ -660,9 +671,6 @@ func TestWCOWXenonOciV2(t *testing.T) { } hostRWSharedDirectory, hostROSharedDirectory := createTestMounts(t) - defer os.RemoveAll(hostRWSharedDirectory) - defer os.RemoveAll(hostROSharedDirectory) - uvmImagePath, err := uvmfolder.LocateUVMFolder(context.Background(), imageLayers) if err != nil { t.Fatalf("LocateUVMFolder failed %s", err) diff --git a/test/go.mod b/test/go.mod index a11c839dea..36f0938582 100644 --- a/test/go.mod +++ b/test/go.mod @@ -48,6 +48,7 @@ require ( github.com/moby/sys/mountinfo v0.4.1 // indirect github.com/opencontainers/runc v1.0.3 // indirect github.com/opencontainers/selinux v1.8.2 // indirect + github.com/pelletier/go-toml v1.8.1 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect diff --git a/test/go.sum b/test/go.sum index 1a83c829a7..692ab67812 100644 --- a/test/go.sum +++ b/test/go.sum @@ -497,6 +497,7 @@ github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3 github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/test/internal/cmd/cmd.go b/test/internal/cmd/cmd.go new file mode 100644 index 0000000000..8689822cee --- /dev/null +++ b/test/internal/cmd/cmd.go @@ -0,0 +1,89 @@ +//go:build windows + +package cmd + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/Microsoft/hcsshim/internal/cmd" + "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/log" +) + +const CopyAfterExitTimeout = time.Second + +func desc(c *cmd.Cmd) string { + desc := "init command" + if c.Spec != nil { + desc = strings.Join(c.Spec.Args, " ") + } + + return desc +} + +func Create(ctx context.Context, _ testing.TB, c cow.ProcessHost, p *specs.Process, io *BufferedIO) *cmd.Cmd { + cc := &cmd.Cmd{ + Host: c, + Context: ctx, + Spec: p, + Log: log.G(ctx), + CopyAfterExitTimeout: CopyAfterExitTimeout, + ExitState: &cmd.ExitState{}, + } + io.AddToCmd(cc) + + return cc +} + +func Start(_ context.Context, t testing.TB, c *cmd.Cmd) { + if err := c.Start(); err != nil { + t.Helper() + t.Fatalf("failed to start %q: %v", desc(c), err) + } +} + +func Run(ctx context.Context, t testing.TB, c *cmd.Cmd) int { + Start(ctx, t, c) + e := Wait(ctx, t, c) + + return e +} + +func Wait(_ context.Context, t testing.TB, c *cmd.Cmd) int { + // todo, wait on context.Done + if err := c.Wait(); err != nil { + t.Helper() + + ee := &cmd.ExitError{} + if errors.As(err, &ee) { + return ee.ExitCode() + } + t.Fatalf("failed to wait on %q: %v", desc(c), err) + } + + return 0 +} + +func WaitExitCode(ctx context.Context, t testing.TB, c *cmd.Cmd, e int) { + if ee := Wait(ctx, t, c); ee != e { + t.Helper() + t.Errorf("got exit code %d, wanted %d", ee, e) + } +} + +func Kill(ctx context.Context, t testing.TB, c *cmd.Cmd) { + ok, err := c.Process.Kill(ctx) + if !ok { + t.Helper() + t.Fatalf("could not deliver kill to %q", desc(c)) + } else if err != nil { + t.Helper() + t.Fatalf("could not kill %q: %v", desc(c), err) + } +} diff --git a/test/internal/cmd/io.go b/test/internal/cmd/io.go new file mode 100644 index 0000000000..9035d920df --- /dev/null +++ b/test/internal/cmd/io.go @@ -0,0 +1,64 @@ +//go:build windows + +package cmd + +import ( + "bytes" + "errors" + "testing" + + "github.com/Microsoft/hcsshim/internal/cmd" +) + +type BufferedIO struct { + in *bytes.Buffer + out, err bytes.Buffer +} + +func NewBufferedIOFromString(in string) *BufferedIO { + b := NewBufferedIO() + b.in = bytes.NewBufferString(in) + + return b +} + +func NewBufferedIO() *BufferedIO { + return &BufferedIO{} +} + +func (b *BufferedIO) Output() (_ string, err error) { + o := b.out.String() + if e := b.err.String(); len(e) != 0 { + err = errors.New(e) + } + + return o, err +} + +func (b *BufferedIO) TestOutput(t testing.TB, out string, err error) { + t.Helper() + + outGive, errGive := b.Output() + if !errors.Is(errGive, err) { + t.Fatalf("got stderr: %v; wanted: %v", errGive, err) + } + if outGive != out { + t.Fatalf("got stdout %q; wanted %q", outGive, out) + } +} + +func (b *BufferedIO) AddToCmd(c *cmd.Cmd) { + if b == nil { + return + } + + if c.Stdin == nil && b.in != nil { + c.Stdin = b.in + } + if c.Stdout == nil { + c.Stdout = &b.out + } + if c.Stderr == nil { + c.Stderr = &b.err + } +} diff --git a/test/internal/constants/constants.go b/test/internal/constants/constants.go new file mode 100644 index 0000000000..0f668cd7fd --- /dev/null +++ b/test/internal/constants/constants.go @@ -0,0 +1,30 @@ +package constants + +import ( + "fmt" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/platforms" +) + +const ( + PlatformWindows = "windows" + PlatformLinux = "linux" + SnapshotterWindows = "windows" + SnapshotterLinux = "windows-lcow" +) + +func SnapshotterFromPlatform(platform string) (string, error) { + p, err := platforms.Parse(platform) + if err != nil { + return "", err + } + switch p.OS { + case PlatformWindows: + return SnapshotterWindows, nil + case PlatformLinux: + return SnapshotterLinux, nil + default: + } + return "", fmt.Errorf("unknown platform os %q: %v", p.OS, errdefs.ErrInvalidArgument) +} diff --git a/test/internal/constants/images.go b/test/internal/constants/images.go new file mode 100644 index 0000000000..18d056c926 --- /dev/null +++ b/test/internal/constants/images.go @@ -0,0 +1,86 @@ +package constants + +// not technically constants, but close enough ... + +import ( + "errors" + "fmt" + + "github.com/Microsoft/hcsshim/osversion" +) + +const ( + DockerImageRepo = "docker.io/library" + McrWindowsImageRepo = "mcr.microsoft.com/windows" + + ImageLinuxAlpineLatest = "docker.io/library/alpine:latest" + ImageLinuxPause31 = "k8s.gcr.io/pause:3.1" +) + +var ErrUnsupportedBuild = errors.New("unsupported build") + +var ( + ImageWindowsNanoserver1709 = NanoserverImage("1709") + ImageWindowsNanoserver1803 = NanoserverImage("1803") + ImageWindowsNanoserver1809 = NanoserverImage("1809") + ImageWindowsNanoserver1903 = NanoserverImage("1903") + ImageWindowsNanoserver1909 = NanoserverImage("1909") + ImageWindowsNanoserver2004 = NanoserverImage("2004") + ImageWindowsNanoserver2009 = NanoserverImage("2009") + ImageWindowsNanoserverLTSC2022 = NanoserverImage("ltsc2022") + + ImageWindowsServercore1709 = ServercoreImage("1709") + ImageWindowsServercore1803 = ServercoreImage("1803") + ImageWindowsServercore1809 = ServercoreImage("1809") + ImageWindowsServercore1903 = ServercoreImage("1903") + ImageWindowsServercore1909 = ServercoreImage("1909") + ImageWindowsServercore2004 = ServercoreImage("2004") + ImageWindowsServercore2009 = ServercoreImage("2009") + ImageWindowsServercoreLTSC2022 = ServercoreImage("ltsc2022") +) + +// all inputs should be predefined and vetted +// may not be formatted correctly for arbitrary inputs +func makeImageURL(repo, image, tag string) string { + r := fmt.Sprintf("%s/%s", repo, image) + if tag != "" { + r = fmt.Sprintf("%s:%s", r, tag) + } + + return r +} + +func NanoserverImage(tag string) string { + return makeImageURL(McrWindowsImageRepo, "nanoserver", tag) +} + +func ServercoreImage(tag string) string { + return makeImageURL(McrWindowsImageRepo, "servercore", tag) +} + +var _buildToTag = map[uint16]string{ + osversion.RS1: "1607", + osversion.RS2: "1703", + osversion.RS3: "1709", + osversion.RS4: "1803", + osversion.RS5: "1809", + osversion.V19H1: "1903", + osversion.V19H2: "1909", + osversion.V20H1: "2004", + osversion.V21H2Server: "ltsc2022", +} + +func ImageFromBuild(build uint16) (string, error) { + if tag, ok := _buildToTag[build]; ok { + return tag, nil + } + + // Due to some efforts in improving down-level compatibility for Windows containers (see + // https://techcommunity.microsoft.com/t5/containers/windows-server-2022-and-beyond-for-containers/ba-p/2712487) + // the ltsc2022 image should continue to work on builds ws2022 and onwards. With this in mind, + // if there's no mapping for the host build, just use the Windows Server 2022 image. + if build > osversion.V21H2Server { + return "ltsc2022", nil + } + return "", ErrUnsupportedBuild +} diff --git a/test/internal/container/container.go b/test/internal/container/container.go new file mode 100644 index 0000000000..440f55585a --- /dev/null +++ b/test/internal/container/container.go @@ -0,0 +1,83 @@ +//go:build windows + +package container + +import ( + "context" + "testing" + + "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/Microsoft/hcsshim/internal/cmd" + "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/hcsoci" + "github.com/Microsoft/hcsshim/internal/resources" + "github.com/Microsoft/hcsshim/internal/uvm" + + testcmd "github.com/Microsoft/hcsshim/test/internal/cmd" +) + +func Create( + ctx context.Context, + t testing.TB, + vm *uvm.UtilityVM, + spec *specs.Spec, + name, owner string, +) (cow.Container, *resources.Resources, func()) { + t.Helper() + + if spec.Windows == nil || spec.Windows.Network == nil || spec.Windows.LayerFolders == nil { + t.Fatalf("improperly configured windows spec for container %q: %#+v", name, spec.Windows) + } + + co := &hcsoci.CreateOptions{ + ID: name, + HostingSystem: vm, + Owner: owner, + Spec: spec, + // dont create a network namespace on the host side + NetworkNamespace: "", //spec.Windows.Network.NetworkNamespace, + } + + c, r, err := hcsoci.CreateContainer(ctx, co) + if err != nil { + t.Fatalf("could not create container %q: %v", co.ID, err) + } + f := func() { + if err := resources.ReleaseResources(ctx, r, vm, true); err != nil { + t.Errorf("failed to release container resources: %v", err) + } + if err := c.Close(); err != nil { + t.Errorf("could not close container %q: %v", c.ID(), err) + } + } + + return c, r, f +} + +func Start(ctx context.Context, t testing.TB, c cow.Container, io *testcmd.BufferedIO) *cmd.Cmd { + t.Helper() + + if err := c.Start(ctx); err != nil { + t.Fatalf("could not start %q: %v", c.ID(), err) + } + + // OCI spec is nil to tell bridge to start container's init process + init := testcmd.Create(ctx, t, c, nil, io) + testcmd.Start(ctx, t, init) + + return init +} + +func Wait(_ context.Context, t testing.TB, c cow.Container) { + // todo: add wait on ctx.Done + if err := c.Wait(); err != nil { + t.Fatalf("could not wait on container %q: %v", c.ID(), err) + } +} + +func Kill(ctx context.Context, t testing.TB, c cow.Container) { + if err := c.Shutdown(ctx); err != nil { + t.Fatalf("could not terminate container %q: %v", c.ID(), err) + } +} diff --git a/test/internal/containerd/containerd.go b/test/internal/containerd/containerd.go new file mode 100644 index 0000000000..f29b0e2918 --- /dev/null +++ b/test/internal/containerd/containerd.go @@ -0,0 +1,207 @@ +//go:build windows + +package containerd + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/errdefs" + kubeutil "github.com/containerd/containerd/integration/remote/util" + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/remotes" + "github.com/containerd/containerd/remotes/docker" + "github.com/containerd/containerd/remotes/docker/config" + "github.com/containerd/containerd/snapshots" + "github.com/opencontainers/image-spec/identity" + "google.golang.org/grpc" + + "github.com/Microsoft/hcsshim/test/internal/constants" + "github.com/Microsoft/hcsshim/test/internal/timeout" +) + +// images maps image refs -> chain ID +var images sync.Map + +// default containerd.New(address) does not connect to tcp endpoints on windows +func createGRPCConn(ctx context.Context, address string) (*grpc.ClientConn, error) { + addr, dialer, err := kubeutil.GetAddressAndDialer(address) + if err != nil { + return nil, err + } + + return grpc.DialContext(ctx, addr, grpc.WithInsecure(), grpc.WithContextDialer(dialer)) +} + +type ContainerdClientOptions struct { + Address string + Namespace string +} + +// NewClient returns a containerd client, a context with the namespace set, and the +// context's cancel function. The context should be used for containerd operations, and +// cancel function will terminate those operations early. +func (cco ContainerdClientOptions) NewClient( + ctx context.Context, + t testing.TB, + opts ...containerd.ClientOpt, +) (context.Context, context.CancelFunc, *containerd.Client) { + t.Helper() + + // regular `New` does not work on windows, need to use `WithConn` + cctx, ccancel := context.WithTimeout(ctx, timeout.ConnectTimeout) + defer ccancel() + + conn, err := createGRPCConn(cctx, cco.Address) + if err != nil { + t.Fatalf("failed to dial runtime client: %v", err) + } + + defOpts := []containerd.ClientOpt{ + containerd.WithDefaultNamespace(cco.Namespace), + } + opts = append(defOpts, opts...) + c, err := containerd.NewWithConn(conn, opts...) + if err != nil { + t.Fatalf("could not create new containerd client: %v", err) + } + t.Cleanup(func() { + c.Close() + }) + + ctx = namespaces.WithNamespace(ctx, cco.Namespace) + ctx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + + return ctx, cancel, c +} + +func GetPlatformComparer(t testing.TB, platform string) platforms.MatchComparer { + var p platforms.MatchComparer + if platform == "" { + p = platforms.All + } else { + pp, err := platforms.Parse(platform) + if err != nil { + t.Helper() + t.Fatalf("could not parse platform %q: %v", platform, err) + } + p = platforms.Only(pp) + } + + return p +} + +// GetImageChainID gets the chain id of an image. platform can be "". +func GetImageChainID(ctx context.Context, t testing.TB, client *containerd.Client, image, platform string) string { + t.Helper() + is := client.ImageService() + + i, err := is.Get(ctx, image) + if err != nil { + t.Fatalf("could not retrieve image %q: %v", image, err) + } + + p := GetPlatformComparer(t, platform) + + diffIDs, err := i.RootFS(ctx, client.ContentStore(), p) + if err != nil { + t.Fatalf("could not retrieve unpacked diff ids: %v", err) + } + chainID := identity.ChainID(diffIDs).String() + + return chainID +} + +func CreateActiveSnapshot(ctx context.Context, t testing.TB, client *containerd.Client, snapshotter, parent, key string, opts ...snapshots.Opt) []mount.Mount { + t.Helper() + + ss := client.SnapshotService(snapshotter) + + ms, err := ss.Prepare(ctx, key, parent, opts...) + if err != nil { + t.Fatalf("could not make active snapshot %q from %q: %v", key, parent, err) + } + + t.Cleanup(func() { + if err := ss.Remove(ctx, key); err != nil && !errors.Is(err, errdefs.ErrNotFound) { + // remove is not idempotent, so do not Fail test + t.Logf("failed to remove active snapshot %q: %v", key, err) + } + }) + + return ms +} + +// a view will not not create a new scratch layer/vhd, but instead return only the directory of the +// committed snapshot `parent` +func CreateViewSnapshot(ctx context.Context, t testing.TB, client *containerd.Client, snapshotter, parent, key string, opts ...snapshots.Opt) []mount.Mount { + t.Helper() + + ss := client.SnapshotService(snapshotter) + + ms, err := ss.View(ctx, key, parent, opts...) + if err != nil { + t.Fatalf("could not make active snapshot %q from %q: %v", key, parent, err) + } + + t.Cleanup(func() { + if err := ss.Remove(ctx, key); err != nil && !errors.Is(err, errdefs.ErrNotFound) { + // remove is not idempotent, so do not Fail test + t.Logf("failed to remove view snapshot %q: %v %T", key, err, err) + } + }) + + return ms +} + +// copied from https://github.com/containerd/containerd/blob/main/cmd/ctr/commands/images/pull.go + +// PullImage pulls the image for the specified platform and returns the chain ID +func PullImage(ctx context.Context, t testing.TB, client *containerd.Client, ref, plat string) string { + if chainID, ok := images.Load(ref); ok { + return chainID.(string) + } + + opts := []containerd.RemoteOpt{ + containerd.WithSchema1Conversion, + containerd.WithPlatform(plat), + containerd.WithPullUnpack, + } + + if s, err := constants.SnapshotterFromPlatform(plat); err == nil { + opts = append(opts, containerd.WithPullSnapshotter(s)) + } + + img, err := client.Pull(ctx, ref, opts...) + if err != nil { + t.Fatalf("could not pull image %q: %v", ref, err) + } + + name := img.Name() + diffIDs, err := img.RootFS(ctx) + if err != nil { + t.Fatalf("could not fetch RootFS diff IDs for %q: %v", name, err) + } + + chainID := identity.ChainID(diffIDs).String() + images.Store(ref, chainID) + t.Logf("unpacked image %q with ID %q", name, chainID) + + return chainID +} + +func GetResolver(ctx context.Context, _ testing.TB) remotes.Resolver { + options := docker.ResolverOptions{ + Tracker: docker.NewInMemoryTracker(), + } + hostOptions := config.HostOptions{} + options.Hosts = config.ConfigureHosts(ctx, hostOptions) + + return docker.NewResolver(options) +} diff --git a/test/functional/utilities/defaultlinuxspec.go b/test/internal/defaultlinuxspec.go similarity index 92% rename from test/functional/utilities/defaultlinuxspec.go rename to test/internal/defaultlinuxspec.go index 682e1fe36d..25f301705a 100644 --- a/test/functional/utilities/defaultlinuxspec.go +++ b/test/internal/defaultlinuxspec.go @@ -1,4 +1,6 @@ -package testutilities +//go:build windows + +package internal import ( "encoding/json" diff --git a/test/functional/utilities/defaultwindowsspec.go b/test/internal/defaultwindowsspec.go similarity index 93% rename from test/functional/utilities/defaultwindowsspec.go rename to test/internal/defaultwindowsspec.go index 84f3870f4d..88581039a9 100644 --- a/test/functional/utilities/defaultwindowsspec.go +++ b/test/internal/defaultwindowsspec.go @@ -1,4 +1,6 @@ -package testutilities +//go:build windows + +package internal import ( "encoding/json" diff --git a/test/internal/doc.go b/test/internal/doc.go new file mode 100644 index 0000000000..853e32afaf --- /dev/null +++ b/test/internal/doc.go @@ -0,0 +1,16 @@ +// This package provides helper functions for testing hcsshim, primarily aimed at the +// end-to-end, integration, and functional tests in ./test. It can, however, be used +// by unit tests. +// +// These files are primarily intended for tests, and using them in code will cause test +// dependencies to be treated normally, which may cause circular import issues with upstream +// packages that vendor hcsshim. +// See https://github.com/microsoft/hcsshim/issues/1148. +// +// Even though this package is meant for testing, appending _test to all files causes issues +// when running or building tests in other folders (ie, ./test/cri-containerd). +// See https://github.com/golang/go/issues/8279. +// Additionally, adding a `//go:build functional` constraint would require internal tests +// (ie, schemaversion_test.go) and unit tests that import this package to be build with +// the `functional` tag. +package internal diff --git a/test/internal/flag/flag.go b/test/internal/flag/flag.go new file mode 100644 index 0000000000..da4daf6af4 --- /dev/null +++ b/test/internal/flag/flag.go @@ -0,0 +1,77 @@ +// This package augments the default "flags" package with functionality similar +// to that in "github.com/urfave/cli", since the two packages do not mix easily +// and the "testing" package uses a default flagset that we cannot easily update. +package flag + +import ( + "flag" + "strings" +) + +const FeatureFlagName = "feature" + +func NewFeatureFlag(all []string) *StringSlice { + ff := NewStringSlice() + flag.Var(ff, FeatureFlagName, + "The sets of functionality to test; can be set multiple times, or separated with commas."+ + "Supported features: "+strings.Join(all, ","), + ) + + return ff +} + +// StringSlice is a type to be used with the standard library's flag.Var +// function as a custom flag value, similar to "github.com/urfave/cli".StringSlice. +// It takes either a comma-separated list of strings, or repeat invocations. +type StringSlice struct { + S StringSet +} + +var _ flag.Value = &StringSlice{} + +// NewStringSetFlag returns a new StringSetFlag with an empty set. +func NewStringSlice() *StringSlice { + return &StringSlice{ + S: make(StringSet), + } +} + +// Strings returns a string slice of the flags provided to the flag +func (ss *StringSlice) Strings() []string { + return ss.S.Strings() +} + +func (ss *StringSlice) String() string { + return ss.S.String() +} + +// Set is called by `flag` each time the flag is seen when parsing the +// command line. +func (ss *StringSlice) Set(s string) error { + for _, f := range strings.Split(s, ",") { + f = Standardize(f) + ss.S[f] = struct{}{} + } + + return nil +} + +type StringSet map[string]struct{} + +func (ss StringSet) Strings() []string { + a := make([]string, 0, len(ss)) + for k := range ss { + a = append(a, k) + } + + return a +} + +func (ss StringSet) String() string { + return "[" + strings.Join(ss.Strings(), ", ") + "]" +} + +// Standardize formats the feature flag s to be consistent (ie, trim and to lowercase) +func Standardize(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} diff --git a/test/internal/layers/layerfolders.go b/test/internal/layers/layerfolders.go new file mode 100644 index 0000000000..347e22024b --- /dev/null +++ b/test/internal/layers/layerfolders.go @@ -0,0 +1,93 @@ +//go:build windows + +package layers + +import ( + "bytes" + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/mount" + + testctrd "github.com/Microsoft/hcsshim/test/internal/containerd" +) + +var imageLayers map[string][]string + +func init() { + imageLayers = make(map[string][]string) +} + +// FromImage returns thee layer paths of a given image, pulling it if necessary +func FromImage(ctx context.Context, t testing.TB, client *containerd.Client, ref, platform, snapshotter string) []string { + chainID := testctrd.PullImage(ctx, t, client, ref, platform) + return FromChainID(ctx, t, client, chainID, snapshotter) +} + +// FromChainID returns thee layer paths of a given image chain ID +func FromChainID(ctx context.Context, t testing.TB, client *containerd.Client, chainID, snapshotter string) []string { + ms := testctrd.CreateViewSnapshot(ctx, t, client, snapshotter, chainID, chainID+"view") + if len(ms) != 1 { + t.Fatalf("Rootfs does not contain exactly 1 mount for the root file system") + } + + return FromMount(ctx, t, ms[0]) +} + +// FromMount returns the layer paths of a given mount +func FromMount(_ context.Context, t testing.TB, m mount.Mount) (layers []string) { + for _, option := range m.Options { + if strings.HasPrefix(option, mount.ParentLayerPathsFlag) { + err := json.Unmarshal([]byte(option[len(mount.ParentLayerPathsFlag):]), &layers) + if err != nil { + t.Fatalf("failed to unmarshal parent layer paths from mount: %v", err) + } + } + } + layers = append(layers, m.Source) + + return layers +} + +// Deprecated: This relies on docker. Use [FromChainID] or [FromMount] instead. +func LayerFolders(t testing.TB, imageName string) []string { + if _, ok := imageLayers[imageName]; !ok { + imageLayers[imageName] = getLayers(t, imageName) + } + return imageLayers[imageName] +} + +func getLayers(t testing.TB, imageName string) []string { + cmd := exec.Command("docker", "inspect", imageName, "-f", `"{{.GraphDriver.Data.dir}}"`) + var out bytes.Buffer + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + t.Skipf("Failed to find layers for %q. Check docker images", imageName) + } + imagePath := strings.Replace(strings.TrimSpace(out.String()), `"`, ``, -1) + layers := getLayerChain(t, imagePath) + return append([]string{imagePath}, layers...) +} + +func getLayerChain(t testing.TB, layerFolder string) []string { + jPath := filepath.Join(layerFolder, "layerchain.json") + content, err := os.ReadFile(jPath) + if os.IsNotExist(err) { + t.Fatalf("layerchain not found") + } else if err != nil { + t.Fatalf("failed to read layerchain") + } + + var layerChain []string + err = json.Unmarshal(content, &layerChain) + if err != nil { + t.Fatalf("failed to unmarshal layerchain") + } + return layerChain +} diff --git a/test/internal/layers/scratch.go b/test/internal/layers/scratch.go new file mode 100644 index 0000000000..6d0dd4b6e2 --- /dev/null +++ b/test/internal/layers/scratch.go @@ -0,0 +1,49 @@ +//go:build windows + +package layers + +import ( + "context" + "path/filepath" + "testing" + + "github.com/Microsoft/hcsshim/internal/lcow" + "github.com/Microsoft/hcsshim/internal/uvm" +) + +const ( + CacheFileName = "cache.vhdx" + ScratchSpaceName = "sandbox.vhdx" + UVMScratchSpaceName = "uvmscratch.vhdx" +) + +func CacheFile(_ context.Context, t testing.TB, dir string) string { + if dir == "" { + dir = t.TempDir() + } + cache := filepath.Join(dir, CacheFileName) + return cache +} + +// ScratchSpace creates an LCOW scratch space VHD at `dir\name`, and returns the dir and name. +// If name, dir, or chache are empty, ScratchSpace uses a default name or creates a temporary +// directory, respectively. +func ScratchSpace(ctx context.Context, t testing.TB, vm *uvm.UtilityVM, name, dir, cache string) (string, string) { + if dir == "" { + dir = t.TempDir() + } + if cache == "" { + cache = CacheFile(ctx, t, dir) + } + if name == "" { + name = ScratchSpaceName + } + scratch := filepath.Join(dir, name) + + if err := lcow.CreateScratch(ctx, vm, scratch, lcow.DefaultScratchSizeGB, cache); err != nil { + t.Helper() + t.Fatalf("could not create scratch space %q using vm %q and cache file %q: %v", scratch, vm.ID(), cache, err) + } + + return dir, scratch +} diff --git a/test/internal/manifest/manifest.go b/test/internal/manifest/manifest.go new file mode 100644 index 0000000000..386de8fa57 --- /dev/null +++ b/test/internal/manifest/manifest.go @@ -0,0 +1,6 @@ +//go:build windows + +// This package allows tests can include the .syso to manifest them to pick up the right Windows build +package manifest + +//go:generate go run github.com/josephspurrier/goversioninfo/cmd/goversioninfo -platform-specific diff --git a/test/internal/manifest/manifest.xml b/test/internal/manifest/manifest.xml new file mode 100644 index 0000000000..acf4843ede --- /dev/null +++ b/test/internal/manifest/manifest.xml @@ -0,0 +1,10 @@ + + + containerd-shim-runhcs-v1 + + + + + + + diff --git a/test/internal/manifest/resource_windows_386.syso b/test/internal/manifest/resource_windows_386.syso new file mode 100644 index 0000000000000000000000000000000000000000..c4ee2aa22a60895bda36a4fd86fc8a8adc2ceb1f GIT binary patch literal 998 zcma)5!EVz)5FNLXs^-L@2ab!yb(0tp;Z{z95LAkUXhlmnqK&<7Rq<#XQ zz$b9z09VfZ34efiwy}~D4valJvomkrj(5G!J&!8+u}{=^9j|Adjq7ML+#nah{R&&j z9`tejR92RCvyqZ7!NBpp}6GUBuU+$zls_-9y_yj#_nubU81^)M~ z`H?q7bWRua2IDh&32#Pc^aAr4##8u}eCz!D)qH(h+}LTICF4&4Yaw~T~+`&*d$6{jr~`b zr#$qTD}_!(X0#mp*V6gZNjDPC$$XJrGt_H0_E%P)xOgdZ;driMYn(|-9-I6`xV*Ox ze3lEXQt8V1_F|JRgG7mRu~kKhCML1brBGVhguA86xn1dH?D#r}`rELk5DOHdR11}< z@;`dH1+Thl)~|aIMUkl(WayzLucDPi_c&*-l}^lC$ASUFW^ASb!$Kr$E-_S(;oo1r`t=@40{q_hZ Y-Oa@~sBc4cvIn$%cdLJPzy2qF1L`}#X#fBK literal 0 HcmV?d00001 diff --git a/test/internal/manifest/resource_windows_amd64.syso b/test/internal/manifest/resource_windows_amd64.syso new file mode 100644 index 0000000000000000000000000000000000000000..5e5c9c9ceed15a89e7392e9a4e3150d29748e547 GIT binary patch literal 998 zcma)5v5wO~5FKCATq=qyC>M)%lNb`=oSYm&a8erX#2ukT8+(0O$= z_ykG{XsP)V{s8gT#!8M*F!t=s&b)a$-t{IQJaY1*L)3U(uV30SuA|LxgIomnE3A_P z=;QizvsMo14qQfba~Jj;7G_7hnm_zU{II(hL|ubl?WH_&_>Vw*f*v(Z!z20v|NGAT z$QvQLpbL79@fp2Wdtj`WMZKP#@@45u&G1 z4@d#z{%S<`(I?R*vH~0y-3TzZjD&sr>;M#5ePyL`j^qJJLNtC{t`Y$g} zdFV4+37v?{XgT$-rS(r{y+~Lq^JR9;P_OOOUzhsW#w(c%%X1Z%#+tO^vB{5x&HJ0c zXSvWSm9|>!E;j2iNR&t$m#V1H#4I+t5=zT5;dZ5RUas{jwtN#rgI!othy@B!sin$P z^&h?5g1fGo_3Iu)QDiCx8G5M6UDQc*k8}1)>BPLTEEqCu#&#Mo&i{hjby1kIlF8Wx zn6a1S|EzuGpmzs_7nI7 zK7k_#xFOE`34efiPKes29GGN0Wdtj`WMZKP#@@45u&G1 z4@d#z{%S<`(I?R*vH~0y-3TzZjD&sr5ePyL`j^qJJLNtC{t`Y$g} zdFV4+37v?{XgT$-rS(r{y+~Lq^JR9;P_OOOUzhsW#w(c%%X1Z%#+tO^vB{5x&HJ0c zXSvWSm9|>!E;j2iNR&t$m#V1H#4I+t5=zT5;dZ5RUas{jwtN#rgI!othy@B!sin$P z^&h?5g1fGo_3Iu)QDiCx8G5M6UDQc*k8}1)>BPLTEEqCu#&#Mo&i{hjby1kIlF8Wx znM)%lNb`=oSYm&a8erX#2ukT8+(0O$={0AUDLXVoJ;Sqg-|9xkE z5k_o6{LR$9#_Q6uy&h-JidjuWySRds1hKx_H{mpAc1WJn&Uik|Ir4 zU{jQl1hb};4D#3mV1_h;PHOuEBT-0)h@Ydoh z2c!UUe>I}}=#%IYSpg1*b^wY_-sTzG$kHmKr+&~M`b_HBBuZaR{g;=g zJoK5Zgib_ew4D0a()uT}UL>rQ`7*m^sMmJtuS<7n}7MBub=>OI1{8Vip@+38iJ3aJy1DFV}h%TfPaR!7i*R!~%t=)KX=t z`j1|2!ChC)`gISYC^8j;3_aB3F6tz@$2ohYbYk9E77Q6SV>=BP=YPTNx+qLp$>i*U z&6BDBE|kOckq92}Wx9OGN2v_?aw5ij6sE~|D1xz=i1+?rOA`(Dnu!LtddCS2+9R0t WwilyQ-v)QG9ooIS?w>uV|B2t(1Hm8w literal 0 HcmV?d00001 diff --git a/test/internal/manifest/versioninfo.json b/test/internal/manifest/versioninfo.json new file mode 100644 index 0000000000..7c4fddcf92 --- /dev/null +++ b/test/internal/manifest/versioninfo.json @@ -0,0 +1,43 @@ +{ + "FixedFileInfo": { + "FileVersion": { + "Major": 1, + "Minor": 0, + "Patch": 0, + "Build": 0 + }, + "ProductVersion": { + "Major": 1, + "Minor": 0, + "Patch": 0, + "Build": 0 + }, + "FileFlagsMask": "3f", + "FileFlags ": "00", + "FileOS": "040004", + "FileType": "01", + "FileSubType": "00" + }, + "StringFileInfo": { + "Comments": "", + "CompanyName": "", + "FileDescription": "", + "FileVersion": "", + "InternalName": "", + "LegalCopyright": "", + "LegalTrademarks": "", + "OriginalFilename": "", + "PrivateBuild": "", + "ProductName": "", + "ProductVersion": "v1.0.0.0", + "SpecialBuild": "" + }, + "VarFileInfo": { + "Translation": { + "LangID": "0409", + "CharsetID": "04B0" + } + }, + "IconPath": "", + "ManifestPath": "manifest.xml" +} \ No newline at end of file diff --git a/test/internal/oci/cri.go b/test/internal/oci/cri.go new file mode 100644 index 0000000000..5157755cfb --- /dev/null +++ b/test/internal/oci/cri.go @@ -0,0 +1,50 @@ +package oci + +import ( + imagespec "github.com/opencontainers/image-spec/specs-go/v1" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" +) + +// +// not technically OCI, but making a CRI package seems overkill +// + +func LinuxWorkloadRuntimeConfig(name string, cmd, args []string, wd string) *runtime.ContainerConfig { + return &runtime.ContainerConfig{ + Metadata: &runtime.ContainerMetadata{ + Name: name, + }, + Command: cmd, + Args: args, + WorkingDir: wd, + } +} + +func LinuxWorkloadImageConfig() *imagespec.ImageConfig { + return LinuxImageConfig([]string{""}, []string{"/bin/sh"}, "/") +} + +func LinuxSandboxRuntimeConfig(name string) *runtime.PodSandboxConfig { + return &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: name, + Namespace: "default", + }, + Hostname: "", + Windows: &runtime.WindowsPodSandboxConfig{}, + } +} + +// based off of: +// containerd\pkg\cri\server\sandbox_run_windows.go +// containerd\pkg\cri\server\container_create.go +// containerd\pkg\cri\server\container_create_windows.go + +func LinuxSandboxImageConfig(pause bool) *imagespec.ImageConfig { + entry := []string{"/bin/sh", "-c", TailNullArgs} + if pause { + entry = []string{"/pause"} + } + + return LinuxImageConfig(entry, []string{}, "/") +} diff --git a/test/internal/oci/images.go b/test/internal/oci/images.go new file mode 100644 index 0000000000..4a87bd566b --- /dev/null +++ b/test/internal/oci/images.go @@ -0,0 +1,19 @@ +package oci + +import ( + imagespec "github.com/opencontainers/image-spec/specs-go/v1" +) + +var DefaultUnixEnv = []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +} + +func LinuxImageConfig(entry, cmd []string, wd string) *imagespec.ImageConfig { + return &imagespec.ImageConfig{ + WorkingDir: wd, + Entrypoint: entry, + Cmd: cmd, + User: "", + Env: DefaultUnixEnv, + } +} diff --git a/test/internal/oci/oci.go b/test/internal/oci/oci.go new file mode 100644 index 0000000000..d3af84cb20 --- /dev/null +++ b/test/internal/oci/oci.go @@ -0,0 +1,125 @@ +package oci + +import ( + "context" + "errors" + "testing" + + "github.com/Microsoft/hcsshim/test/internal/constants" + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/namespaces" + ctrdoci "github.com/containerd/containerd/oci" + criconstants "github.com/containerd/containerd/pkg/cri/constants" + "github.com/opencontainers/runtime-spec/specs-go" +) + +// +// testing helper functions for OCI spec creation +// + +const ( + TailNullArgs = "tail -f /dev/null" + + DefaultNamespace = namespaces.Default + CRINamespace = criconstants.K8sContainerdNamespace + + // from containerd\pkg\cri\server\helpers_linux.go + + HostnameEnv = "HOSTNAME" +) + +func DefaultLinuxSpecOpts(nns string, extra ...ctrdoci.SpecOpts) []ctrdoci.SpecOpts { + opts := []ctrdoci.SpecOpts{ + WithoutRunMount, + ctrdoci.WithRootFSReadonly(), + WithDisabledCgroups, // we set our own cgroups + ctrdoci.WithDefaultUnixDevices, + ctrdoci.WithDefaultPathEnv, + WithWindowsNetworkNamespace(nns), + } + opts = append(opts, extra...) + + return opts +} + +// DefaultLinuxSpec returns a default OCI spec for a Linux container. +// See CreateSpecWithPlatform for more details. +func DefaultLinuxSpec(ctx context.Context, t testing.TB, nns string) *specs.Spec { + return CreateLinuxSpec(ctx, t, t.Name(), DefaultLinuxSpecOpts(nns)...) +} + +// CreateLinuxSpec returns the OCI spec for a Linux container. +// See CreateSpecWithPlatform for more details. +func CreateLinuxSpec(ctx context.Context, t testing.TB, id string, opts ...ctrdoci.SpecOpts) *specs.Spec { + return CreateSpecWithPlatform(ctx, t, constants.PlatformLinux, id, opts...) +} + +// CreateWindowsSpec returns the OCI spec for a Windows container. +// See CreateSpecWithPlatform for more details. +func CreateWindowsSpec(ctx context.Context, t testing.TB, id string, opts ...ctrdoci.SpecOpts) *specs.Spec { + return CreateSpecWithPlatform(ctx, t, constants.PlatformWindows, id, opts...) +} + +// CreateSpecWithPlatform returns the OCI spec for the specified platform. +// The context must contain a containerd namespace (via "github.com/containerd/containerd/namespaces".WithNamespace) +func CreateSpecWithPlatform(ctx context.Context, t testing.TB, plat, id string, opts ...ctrdoci.SpecOpts) *specs.Spec { + // ctx = namespaces.WithNamespace(ctx, ns) + container := &containers.Container{ID: id} + + spec, err := ctrdoci.GenerateSpecWithPlatform(ctx, nil, plat, container, opts...) + if err != nil { + t.Helper() + t.Fatalf("failed to generate spec for container %q: %v", id, err) + } + + return spec +} + +func WithWindowsLayerFolders(layers []string) ctrdoci.SpecOpts { + return func(ctx context.Context, client ctrdoci.Client, c *containers.Container, s *specs.Spec) error { + if len(layers) < 2 { + return errors.New("at least two layers are required, including the sandbox path") + } + + if s.Windows == nil { + s.Windows = &specs.Windows{} + } + s.Windows.LayerFolders = layers + + return nil + } +} + +//defined in containerd\pkg\cri\opts\spec_windows.go + +func WithWindowsNetworkNamespace(path string) ctrdoci.SpecOpts { + return func(ctx context.Context, client ctrdoci.Client, c *containers.Container, s *specs.Spec) error { + if s.Windows == nil { + s.Windows = &specs.Windows{} + } + if s.Windows.Network == nil { + s.Windows.Network = &specs.WindowsNetwork{} + } + s.Windows.Network.NetworkNamespace = path + + return nil + } +} + +//defined in containerd\pkg\cri\opts\spec_linux.go + +// WithDisabledCgroups clears the Cgroups Path from the spec +func WithDisabledCgroups(_ context.Context, _ ctrdoci.Client, c *containers.Container, s *specs.Spec) error { + if s.Linux == nil { + s.Linux = &specs.Linux{} + } + s.Linux.CgroupsPath = "" + return nil +} + +// defined in containerd\oci\spec_opts_linux.go + +// WithoutRunMount removes the `/run` inside the spec +func WithoutRunMount(ctx context.Context, client ctrdoci.Client, c *containers.Container, s *specs.Spec) error { + return ctrdoci.WithoutMounts("/run")(ctx, client, c, s) +} diff --git a/test/internal/require/build.go b/test/internal/require/build.go new file mode 100644 index 0000000000..d4659f08e5 --- /dev/null +++ b/test/internal/require/build.go @@ -0,0 +1,23 @@ +//go:build windows + +package require + +import ( + "testing" + + "github.com/Microsoft/hcsshim/osversion" + + _ "github.com/Microsoft/hcsshim/test/internal/manifest" // manifest test binary automatically +) + +func Build(t testing.TB, b uint16) { + if osversion.Build() < b { + t.Skipf("Requires build %d+", b) + } +} + +func ExactBuild(t testing.TB, b uint16) { + if osversion.Build() != b { + t.Skipf("Requires exact build %d", b) + } +} diff --git a/test/internal/require/requires.go b/test/internal/require/requires.go new file mode 100644 index 0000000000..16eadba2d4 --- /dev/null +++ b/test/internal/require/requires.go @@ -0,0 +1,61 @@ +package require + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Microsoft/hcsshim/test/internal/flag" +) + +// Features checks the wanted features are present in given, +// and skips the test if any are missing. If the given set is empty, +// the function returns (by default all features are enabled). +func Features(t testing.TB, given flag.StringSet, want ...string) { + if len(given) == 0 { + return + } + for _, f := range want { + ff := flag.Standardize(f) + if _, ok := given[ff]; !ok { + t.Skipf("skipping test due to feature not set: %s", f) + } + } +} + +// Binary checks if `binary` exists in the same directory as the test +// binary. +// Returns full binary path if it exists, otherwise, skips the test. +func Binary(t testing.TB, binary string) string { + executable, err := os.Executable() + if err != nil { + t.Skipf("error locating executable: %s", err) + return "" + } + + baseDir := filepath.Dir(executable) + return File(t, baseDir, binary) +} + +// File checks if `file` exists in `path`, and returns the full path +// if it exists. Otherwise, it skips the test. +func File(t testing.TB, path, file string) string { + path, err := filepath.Abs(path) + if err != nil { + t.Fatalf("could not resolve path %q: %v", path, err) + } + + p := filepath.Join(path, file) + fi, err := os.Stat(p) + switch { + case os.IsNotExist(err): + t.Skipf("file %q not found", p) + case err != nil: + t.Fatalf("could not stat file %q: %v", p, err) + case fi.IsDir(): + t.Fatalf("path %q is a directory", p) + default: + } + + return p +} diff --git a/test/internal/schemaversion_test.go b/test/internal/schemaversion_test.go index f004d2ea23..3e549e7457 100644 --- a/test/internal/schemaversion_test.go +++ b/test/internal/schemaversion_test.go @@ -9,8 +9,9 @@ import ( hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/osversion" - _ "github.com/Microsoft/hcsshim/test/functional/manifest" "github.com/sirupsen/logrus" + + _ "github.com/Microsoft/hcsshim/test/internal/manifest" ) func init() { diff --git a/test/functional/utilities/scratch.go b/test/internal/scratch.go similarity index 91% rename from test/functional/utilities/scratch.go rename to test/internal/scratch.go index c35d4e91db..333a4cd478 100644 --- a/test/functional/utilities/scratch.go +++ b/test/internal/scratch.go @@ -1,4 +1,6 @@ -package testutilities +//go:build windows + +package internal import ( "context" @@ -9,6 +11,8 @@ import ( "github.com/Microsoft/hcsshim/internal/lcow" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/wclayer" + + tuvm "github.com/Microsoft/hcsshim/test/internal/uvm" ) const lcowGlobalSVMID = "test.lcowglobalsvm" @@ -47,7 +51,7 @@ func CreateWCOWBlankRWLayer(t *testing.T, imageLayers []string) string { // for a "service VM". func CreateLCOWBlankRWLayer(ctx context.Context, t *testing.T) string { if lcowGlobalSVM == nil { - lcowGlobalSVM = CreateLCOWUVM(ctx, t, lcowGlobalSVMID) + lcowGlobalSVM = tuvm.CreateAndStartLCOW(ctx, t, lcowGlobalSVMID) lcowCacheScratchFile = filepath.Join(t.TempDir(), "sandbox.vhdx") } tempDir := t.TempDir() diff --git a/test/functional/utilities/stringsetflag.go b/test/internal/stringsetflag.go similarity index 82% rename from test/functional/utilities/stringsetflag.go rename to test/internal/stringsetflag.go index eeb06bf4cf..561f4ed275 100644 --- a/test/functional/utilities/stringsetflag.go +++ b/test/internal/stringsetflag.go @@ -1,10 +1,14 @@ -package testutilities +//go:build windows + +package internal + +import "github.com/Microsoft/hcsshim/test/internal/flag" // StringSetFlag is a type to be used with the standard library's flag.Var // function as a custom flag value. It accumulates the arguments passed each // time the flag is used into a set. type StringSetFlag struct { - set map[string]struct{} + set flag.StringSet } // NewStringSetFlag returns a new StringSetFlag with an empty set. @@ -36,6 +40,6 @@ func (ssf StringSetFlag) Set(s string) error { } // ValueSet returns the internal set of what values have been seen. -func (ssf StringSetFlag) ValueSet() map[string]struct{} { +func (ssf StringSetFlag) ValueSet() flag.StringSet { return ssf.set } diff --git a/test/functional/assets/defaultlinuxspec.json b/test/internal/testdata/defaultlinuxspec.json similarity index 100% rename from test/functional/assets/defaultlinuxspec.json rename to test/internal/testdata/defaultlinuxspec.json diff --git a/test/functional/assets/defaultwindowsspec.json b/test/internal/testdata/defaultwindowsspec.json similarity index 100% rename from test/functional/assets/defaultwindowsspec.json rename to test/internal/testdata/defaultwindowsspec.json diff --git a/test/functional/assets/samples/config.justin.lcow.working.json b/test/internal/testdata/samples/config.justin.lcow.working.json similarity index 100% rename from test/functional/assets/samples/config.justin.lcow.working.json rename to test/internal/testdata/samples/config.justin.lcow.working.json diff --git a/test/functional/assets/samples/from-docker-linux/privileged.json b/test/internal/testdata/samples/from-docker-linux/privileged.json similarity index 100% rename from test/functional/assets/samples/from-docker-linux/privileged.json rename to test/internal/testdata/samples/from-docker-linux/privileged.json diff --git a/test/functional/assets/samples/from-docker-linux/sh.json b/test/internal/testdata/samples/from-docker-linux/sh.json similarity index 100% rename from test/functional/assets/samples/from-docker-linux/sh.json rename to test/internal/testdata/samples/from-docker-linux/sh.json diff --git a/test/internal/timeout/timeout.go b/test/internal/timeout/timeout.go new file mode 100644 index 0000000000..38f8212d39 --- /dev/null +++ b/test/internal/timeout/timeout.go @@ -0,0 +1,45 @@ +// Package provides functionality for timing out operations and waiting +// for deadlines. +package timeout + +import ( + "context" + "testing" + "time" +) + +const ConnectTimeout = time.Second * 10 + +type ErrorFunc func(error) error + +// WaitForError waits until f returns or the context is done. +// The returned error will be passed to the error processing functions fe, and (if non-nil), +// then passed to to [testing.Fatal]. +// +// fe should expect nil errors. +func WaitForError(ctx context.Context, t testing.TB, f func() error, fe ErrorFunc) { + var err error + ch := make(chan struct{}) + + go func() { + err = f() + close(ch) + }() + + select { + case <-ch: + fatalOnError(t, fe, err) + case <-ctx.Done(): + fatalOnError(t, fe, ctx.Err()) + } +} + +func fatalOnError(t testing.TB, f func(error) error, err error) { + if f != nil { + err = f(err) + } + if err != nil { + t.Helper() + t.Fatal(err.Error()) + } +} diff --git a/test/internal/uvm/lcow.go b/test/internal/uvm/lcow.go new file mode 100644 index 0000000000..12c58d6a36 --- /dev/null +++ b/test/internal/uvm/lcow.go @@ -0,0 +1,47 @@ +//go:build windows + +package uvm + +import ( + "context" + "testing" + + "github.com/Microsoft/hcsshim/internal/uvm" +) + +// CreateAndStartLCOW with all default options. +func CreateAndStartLCOW(ctx context.Context, t testing.TB, id string) *uvm.UtilityVM { + return CreateAndStartLCOWFromOpts(ctx, t, uvm.NewDefaultOptionsLCOW(id, "")) +} + +// CreateAndStartLCOWFromOpts creates an LCOW utility VM with the specified options. +func CreateAndStartLCOWFromOpts(ctx context.Context, t testing.TB, opts *uvm.OptionsLCOW) *uvm.UtilityVM { + t.Helper() + + if opts == nil { + t.Fatal("opts must be set") + } + + vm := CreateLCOW(ctx, t, opts) + cleanup := Start(ctx, t, vm) + t.Cleanup(cleanup) + + return vm +} + +func CreateLCOW(ctx context.Context, t testing.TB, opts *uvm.OptionsLCOW) *uvm.UtilityVM { + vm, err := uvm.CreateLCOW(ctx, opts) + if err != nil { + t.Helper() + t.Fatalf("could not create LCOW UVM: %v", err) + } + + return vm +} + +func SetSecurityPolicy(ctx context.Context, t testing.TB, vm *uvm.UtilityVM, policy string) { + if err := vm.SetSecurityPolicy(ctx, "allow_all", policy); err != nil { + t.Helper() + t.Fatalf("could not set vm security policy to %q: %v", policy, err) + } +} diff --git a/test/internal/uvm/uvm.go b/test/internal/uvm/uvm.go new file mode 100644 index 0000000000..02bf301b2d --- /dev/null +++ b/test/internal/uvm/uvm.go @@ -0,0 +1,59 @@ +//go:build windows + +package uvm + +import ( + "context" + "fmt" + "testing" + + "github.com/Microsoft/hcsshim/internal/uvm" + + "github.com/Microsoft/hcsshim/test/internal/timeout" +) + +func Start(ctx context.Context, t testing.TB, vm *uvm.UtilityVM) func() { + err := vm.Start(ctx) + f := func() { + if err := vm.Close(); err != nil { + t.Logf("could not close vm %q: %v", vm.ID(), err) + } + } + + if err != nil { + t.Helper() + t.Fatalf("could not start UVM: %v", err) + } + + return f +} + +func Wait(ctx context.Context, t testing.TB, vm *uvm.UtilityVM) { + fe := func(err error) error { + if err != nil { + err = fmt.Errorf("could not wait for uvm %q: %w", vm.ID(), err) + } + + return err + } + timeout.WaitForError(ctx, t, vm.Wait, fe) +} + +func Kill(ctx context.Context, t testing.TB, vm *uvm.UtilityVM) { + if err := vm.Terminate(ctx); err != nil { + t.Helper() + t.Fatalf("could not kill uvm %q: %v", vm.ID(), err) + } +} + +func Close(ctx context.Context, t testing.TB, vm *uvm.UtilityVM) { + // Terminate will error on context cancellation, but close does not accept contexts + fe := func(err error) error { + if err != nil { + err = fmt.Errorf("could not close uvm %q: %w", vm.ID(), err) + } + + return err + } + timeout.WaitForError(ctx, t, vm.Close, fe) +} diff --git a/test/functional/utilities/createuvm.go b/test/internal/uvm/wcow.go similarity index 51% rename from test/functional/utilities/createuvm.go rename to test/internal/uvm/wcow.go index d88307bde9..32172da04c 100644 --- a/test/functional/utilities/createuvm.go +++ b/test/internal/uvm/wcow.go @@ -1,22 +1,26 @@ -package testutilities +//go:build windows + +package uvm import ( "context" - "os" "testing" "github.com/Microsoft/hcsshim/internal/uvm" + + "github.com/Microsoft/hcsshim/test/internal/layers" ) // CreateWCOWUVM creates a WCOW utility VM with all default options. Returns the // UtilityVM object; folder used as its scratch -func CreateWCOWUVM(ctx context.Context, t *testing.T, id, image string) (*uvm.UtilityVM, []string, string) { +func CreateWCOWUVM(ctx context.Context, t testing.TB, id, image string) (*uvm.UtilityVM, []string, string) { return CreateWCOWUVMFromOptsWithImage(ctx, t, uvm.NewDefaultOptionsWCOW(id, ""), image) - } // CreateWCOWUVMFromOpts creates a WCOW utility VM with the passed opts. -func CreateWCOWUVMFromOpts(ctx context.Context, t *testing.T, opts *uvm.OptionsWCOW) *uvm.UtilityVM { +func CreateWCOWUVMFromOpts(ctx context.Context, t testing.TB, opts *uvm.OptionsWCOW) *uvm.UtilityVM { + t.Helper() + if opts == nil || len(opts.LayerFolders) < 2 { t.Fatalf("opts must bet set with LayerFolders") } @@ -29,49 +33,24 @@ func CreateWCOWUVMFromOpts(ctx context.Context, t *testing.T, opts *uvm.OptionsW uvm.Close() t.Fatal(err) } + return uvm } // CreateWCOWUVMFromOptsWithImage creates a WCOW utility VM with the passed opts // builds the LayerFolders based on `image`. Returns the UtilityVM object; // folder used as its scratch -func CreateWCOWUVMFromOptsWithImage(ctx context.Context, t *testing.T, opts *uvm.OptionsWCOW, image string) (*uvm.UtilityVM, []string, string) { +func CreateWCOWUVMFromOptsWithImage(ctx context.Context, t testing.TB, opts *uvm.OptionsWCOW, image string) (*uvm.UtilityVM, []string, string) { + t.Helper() + if opts == nil { t.Fatal("opts must be set") } - uvmLayers := LayerFolders(t, image) - scratchDir := CreateTempDir(t) - defer func() { - if t.Failed() { - os.RemoveAll(scratchDir) - } - }() - + uvmLayers := layers.LayerFolders(t, image) + scratchDir := t.TempDir() opts.LayerFolders = append(opts.LayerFolders, uvmLayers...) opts.LayerFolders = append(opts.LayerFolders, scratchDir) return CreateWCOWUVMFromOpts(ctx, t, opts), uvmLayers, scratchDir } - -// CreateLCOWUVM with all default options. -func CreateLCOWUVM(ctx context.Context, t *testing.T, id string) *uvm.UtilityVM { - return CreateLCOWUVMFromOpts(ctx, t, uvm.NewDefaultOptionsLCOW(id, "")) -} - -// CreateLCOWUVMFromOpts creates an LCOW utility VM with the specified options. -func CreateLCOWUVMFromOpts(ctx context.Context, t *testing.T, opts *uvm.OptionsLCOW) *uvm.UtilityVM { - if opts == nil { - t.Fatal("opts must be set") - } - - uvm, err := uvm.CreateLCOW(ctx, opts) - if err != nil { - t.Fatalf("could not create LCOW UVM: %v", err) - } - if err := uvm.Start(ctx); err != nil { - uvm.Close() - t.Fatalf("could not start LCOW UVM: %v", err) - } - return uvm -} diff --git a/test/runhcs/e2e_matrix_test.go b/test/runhcs/e2e_matrix_test.go index 3837a880ac..a45a8b664f 100644 --- a/test/runhcs/e2e_matrix_test.go +++ b/test/runhcs/e2e_matrix_test.go @@ -18,7 +18,8 @@ import ( "github.com/Microsoft/go-winio/vhd" "github.com/Microsoft/hcsshim/osversion" runhcs "github.com/Microsoft/hcsshim/pkg/go-runhcs" - testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" + "github.com/Microsoft/hcsshim/test/internal/layers" + "github.com/Microsoft/hcsshim/test/internal/require" runc "github.com/containerd/go-runc" "github.com/opencontainers/runtime-tools/generate" "github.com/pkg/errors" @@ -166,7 +167,7 @@ func testWindows(t *testing.T, version int, isolated bool) { var err error // Make the bundle - bundle := testutilities.CreateTempDir(t) + bundle := t.TempDir() defer func() { if err == nil { os.RemoveAll(bundle) @@ -174,7 +175,7 @@ func testWindows(t *testing.T, version int, isolated bool) { t.Errorf("additional logs at bundle path: %v", bundle) } }() - scratch := testutilities.CreateTempDir(t) + scratch := t.TempDir() defer func() { vhd.DetachVhd(filepath.Join(scratch, "sandbox.vhdx")) os.RemoveAll(scratch) @@ -194,7 +195,7 @@ func testWindows(t *testing.T, version int, isolated bool) { // Get the LayerFolders imageName := getWindowsImageNameByVersion(t, version) - layers := testutilities.LayerFolders(t, imageName) + layers := layers.LayerFolders(t, imageName) for _, layer := range layers { g.AddWindowsLayerFolders(layer) } @@ -310,25 +311,25 @@ func testLCOWPod(t *testing.T) { } func Test_RS1_Argon(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS1) + require.ExactBuild(t, osversion.RS1) testWindows(t, osversion.RS1, false) } func Test_RS1_Xenon(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS1) + require.ExactBuild(t, osversion.RS1) testWindows(t, osversion.RS1, true) } func Test_RS3_Argon(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS3) + require.ExactBuild(t, osversion.RS3) testWindows(t, osversion.RS3, false) } func Test_RS3_Xenon(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS3) + require.ExactBuild(t, osversion.RS3) guests := []int{osversion.RS1, osversion.RS3} for _, g := range guests { @@ -337,13 +338,13 @@ func Test_RS3_Xenon(t *testing.T) { } func Test_RS4_Argon(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS4) + require.ExactBuild(t, osversion.RS4) testWindows(t, osversion.RS4, false) } func Test_RS4_Xenon(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS4) + require.ExactBuild(t, osversion.RS4) guests := []int{osversion.RS1, osversion.RS3, osversion.RS4} for _, g := range guests { @@ -352,19 +353,19 @@ func Test_RS4_Xenon(t *testing.T) { } func Test_RS5_Argon(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS5) + require.ExactBuild(t, osversion.RS5) testWindows(t, osversion.RS5, false) } func Test_RS5_ArgonPods(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS5) + require.ExactBuild(t, osversion.RS5) testWindowsPod(t, osversion.RS5, false) } func Test_RS5_UVMAndContainer(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS5) + require.ExactBuild(t, osversion.RS5) guests := []int{osversion.RS1, osversion.RS3, osversion.RS4, osversion.RS5} for _, g := range guests { @@ -373,19 +374,19 @@ func Test_RS5_UVMAndContainer(t *testing.T) { } func Test_RS5_UVMPods(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS5) + require.ExactBuild(t, osversion.RS5) testWindowsPod(t, osversion.RS5, true) } func Test_RS5_LCOW(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS5) + require.ExactBuild(t, osversion.RS5) testLCOW(t) } func Test_RS5_LCOW_UVMPods(t *testing.T) { - testutilities.RequiresExactBuild(t, osversion.RS5) + require.ExactBuild(t, osversion.RS5) testLCOWPod(t) } diff --git a/test/runhcs/runhcs_test.go b/test/runhcs/runhcs_test.go index 6e7b98152e..24cf5ed1bd 100644 --- a/test/runhcs/runhcs_test.go +++ b/test/runhcs/runhcs_test.go @@ -4,5 +4,5 @@ package runhcs import ( - _ "github.com/Microsoft/hcsshim/test/functional/manifest" + _ "github.com/Microsoft/hcsshim/test/internal/manifest" )