Merged PR 6117553: Sync ADO to 671ec23

Sync ADO to 671ec23

Related work items: #1258, #1273, #1304, #1359, #1360, #1362, #1363, #1367, #1372, #1374, #1377, #1380, #1381, #1382, #1384, #1387, #1388, #1389, #1390, #1391, #1393, #1394, #1395, #1396, #1398, #1400, #1401, #1402, #1404, #1406

.gitattributes

@@ -1 +1,3 @@
-* text=auto eol=lf
+* text=auto eol=lf
+vendor/** -text
+test/vendor/** -text

.github/workflows/ci.yml

@@ -3,10 +3,6 @@ on:
   - push
   - pull_request
 
-env:
-  GOFLAGS: -mod=vendor
-  GOPROXY: off
-
 jobs:
   protos:
     runs-on: 'windows-2019'
@@ -88,25 +84,6 @@ jobs:
           }
           exit $process.ExitCode
 
-  verify-test-vendor:
-    runs-on: 'windows-2019'
-    env:
-      GOPROXY: "https://proxy.golang.org,direct"
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '^1.17.0'
-      - name: Validate test modules
-        shell: powershell
-        run: |
-          $currentPath = (Get-Location).Path
-          $process = Start-Process powershell.exe -PassThru -Verb runAs -Wait -ArgumentList $currentPath/scripts/Verify-GoModules.ps1, $currentPath, "test"
-          if ($process.ExitCode -ne 0) {
-            Write-Error "Test package modules are not up to date. Please validate your go version >= this job's and run `go mod vendor` followed by `go mod tidy` in hcsshim/test directory."
-          }
-          exit $process.ExitCode
-
   test:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -119,17 +96,17 @@ jobs:
           go-version: '^1.17.0'
 
       - run: go test -gcflags=all=-d=checkptr -v ./... -tags admin
-      - run: go test -gcflags=all=-d=checkptr -v ./internal -tags admin
+      - run: go test -mod=mod -gcflags=all=-d=checkptr -v ./internal -tags admin
+        working-directory: test  
+      - run: go test -mod=mod -gcflags=all=-d=checkptr -c ./containerd-shim-runhcs-v1/ -tags functional
         working-directory: test
-      - run: go test -gcflags=all=-d=checkptr -c ./containerd-shim-runhcs-v1/ -tags functional
+      - run: go test -mod=mod -gcflags=all=-d=checkptr -c ./cri-containerd/ -tags functional
         working-directory: test
-      - run: go test -gcflags=all=-d=checkptr -c ./cri-containerd/ -tags functional
+      - run: go test -mod=mod -gcflags=all=-d=checkptr -c ./functional/ -tags functional
         working-directory: test
-      - run: go test -gcflags=all=-d=checkptr -c ./functional/ -tags functional
+      - run: go test -mod=mod -gcflags=all=-d=checkptr -c ./runhcs/ -tags functional
         working-directory: test
-      - run: go test -gcflags=all=-d=checkptr -c ./runhcs/ -tags functional
-        working-directory: test
-      - run: go build -o sample-logging-driver.exe ./cri-containerd/helpers/log.go
+      - run: go build -mod=mod -o sample-logging-driver.exe ./cri-containerd/helpers/log.go
         working-directory: test
 
       - uses: actions/upload-artifact@v2
@@ -170,13 +147,15 @@ jobs:
             containerd-shim-runhcs-v1.exe
             runhcs.exe
             tar2ext4.exe
-            device-util.exe
             wclayer.exe
+            device-util.exe
+            ncproxy.exe
+            dmverity-vhd.exe
             grantvmgroupaccess.exe
             networkagent.exe
+            securitypolicy.exe
             uvmboot.exe
             zapdir.exe
-            ncproxy.exe
 
   build_gcs:
     runs-on: ubuntu-latest

.gitignore

@@ -24,6 +24,7 @@ service/pkg/
 *.img
 *.vhd
 *.tar.gz
+*.tar
 
 # Make stuff
 .rootfs-done
@@ -36,5 +37,9 @@ rootfs-conv/*
 deps/*
 out/*
 
+# test results
+test/results
+
+# go workspace files
 go.work
-go.work.sum
+go.work.sum

README.md

@@ -75,24 +75,6 @@ certify they either authored the work themselves or otherwise have permission to
 more info, as well as to make sure that you can attest to the rules listed. Our CI uses the [DCO Github app](https://github.com/apps/dco) to ensure
 that all commits in a given PR are signed-off.
 
-### Test Directory (Important to note)
-
-This project has tried to trim some dependencies from the root Go modules file that would be cumbersome to get transitively included if this
-project is being vendored/used as a library. Some of these dependencies were only being used for tests, so the /test directory in this project also has
-its own go.mod file where these are now included to get around this issue. Our tests rely on the code in this project to run, so the test Go modules file
-has a relative path replace directive to pull in the latest hcsshim code that the tests actually touch from this project
-(which is the repo itself on your disk).
-
-```
-replace (
-	github.com/Microsoft/hcsshim => ../
-)
-```
-
-Because of this, for most code changes you may need to run `go mod vendor` + `go mod tidy` in the /test directory in this repository, as the
-CI in this project will check if the files are out of date and will fail if this is true.
-
-
 ## Code of Conduct
 
 This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).

cmd/containerd-shim-runhcs-v1/delete.go

@@ -18,7 +18,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
-	"go.opencensus.io/trace"
 )
 
 // LimitedRead reads at max `readLimitBytes` bytes from the file at path `filePath`. If the file has
@@ -56,7 +55,7 @@ The delete command will be executed in the container's bundle as its cwd.
 		// task.DeleteResponse by protocol. We can write to stderr which will be
 		// logged as a warning in containerd.
 
-		ctx, span := trace.StartSpan(gcontext.Background(), "delete")
+		ctx, span := oc.StartSpan(gcontext.Background(), "delete")
 		defer span.End()
 		defer func() { oc.SetSpanStatus(span, err) }()
 

cmd/containerd-shim-runhcs-v1/events.go

@@ -39,7 +39,7 @@ func (e *eventPublisher) close() error {
 }
 
 func (e *eventPublisher) publishEvent(ctx context.Context, topic string, event interface{}) (err error) {
-	ctx, span := trace.StartSpan(ctx, "publishEvent")
+	ctx, span := oc.StartSpan(ctx, "publishEvent")
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(

cmd/containerd-shim-runhcs-v1/exec_hcs.go

@@ -20,6 +20,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/cmd"
 	"github.com/Microsoft/hcsshim/internal/cow"
 	"github.com/Microsoft/hcsshim/internal/log"
+	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
 	"github.com/Microsoft/hcsshim/internal/signals"
 	"github.com/Microsoft/hcsshim/internal/uvm"
@@ -416,13 +417,15 @@ func (he *hcsExec) exitFromCreatedL(ctx context.Context, status int) {
 //
 // 7. Finally, save the UVM and this container as a template if specified.
 func (he *hcsExec) waitForExit() {
-	ctx, span := trace.StartSpan(context.Background(), "hcsExec::waitForExit")
+	var err error // this will only save the last error, since we dont return early on error
+	ctx, span := oc.StartSpan(context.Background(), "hcsExec::waitForExit")
 	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(
 		trace.StringAttribute("tid", he.tid),
 		trace.StringAttribute("eid", he.id))
 
-	err := he.p.Process.Wait()
+	err = he.p.Process.Wait()
 	if err != nil {
 		log.G(ctx).WithError(err).Error("failed process Wait")
 	}
@@ -478,7 +481,7 @@ func (he *hcsExec) waitForExit() {
 //
 // This MUST be called via a goroutine at exec create.
 func (he *hcsExec) waitForContainerExit() {
-	ctx, span := trace.StartSpan(context.Background(), "hcsExec::waitForContainerExit")
+	ctx, span := oc.StartSpan(context.Background(), "hcsExec::waitForContainerExit")
 	defer span.End()
 	span.AddAttributes(
 		trace.StringAttribute("tid", he.tid),

cmd/containerd-shim-runhcs-v1/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/Microsoft/go-winio/pkg/etw"
 	"github.com/Microsoft/go-winio/pkg/etwlogrus"
 	"github.com/Microsoft/go-winio/pkg/guid"
+	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/internal/shimdiag"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -64,6 +65,8 @@ func etwCallback(sourceID guid.GUID, state etw.ProviderState, level etw.Level, m
 }
 
 func main() {
+	logrus.AddHook(log.NewHook())
+
 	// Provider ID: 0b52781f-b24d-5685-ddf6-69830ed40ec3
 	// Provider and hook aren't closed explicitly, as they will exist until process exit.
 	provider, err := etw.NewProvider("Microsoft.Virtualization.RunHCS", etwCallback)
@@ -86,7 +89,7 @@ func main() {
 	)
 
 	// Register our OpenCensus logrus exporter
-	trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+	trace.ApplyConfig(trace.Config{DefaultSampler: oc.DefaultSampler})
 	trace.RegisterExporter(&oc.LogrusExporter{})
 
 	app := cli.NewApp()

cmd/containerd-shim-runhcs-v1/options/runhcs.proto

@@ -106,7 +106,7 @@ message Options {
 	// UTC.
 	bool no_inherit_host_timezone = 19;
 
-	// scrub_logs enables removing environment variables and other protentially sensitive information from logs
+	// scrub_logs enables removing environment variables and other potentially sensitive information from logs
 	bool scrub_logs = 20;
 }