Merge #11532

11532: fix(testing): Adds cached python venvs for integration tests r=kpitzen a=kpitzen

The cache key is based on hashing the contents of "requirements.txt" for each python test.  This should result in unchanged test results among tests sharing venvs, and maintains environment isolation across requirement sets

<!--- 
Thanks so much for your contribution! If this is your first time contributing, please ensure that you have read the [CONTRIBUTING](https://github.com/pulumi/pulumi/blob/master/CONTRIBUTING.md) documentation.
-->

# Description

<!--- Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. -->

Fixes #11485 

## Checklist

<!--- Please provide details if the checkbox below is to be left unchecked. -->
- [x] I have added tests that prove my fix is effective or that my feature works
<!--- 
User-facing changes require a CHANGELOG entry.
-->
- [x] I have run `make changelog` and committed the `changelog/pending/<file>` documenting my change
<!--
If the change(s) in this PR is a modification of an existing call to the Pulumi Service,
then the service should honor older versions of the CLI where this change would not exist.
You must then bump the API version in /pkg/backend/httpstate/client/api.go, as well as add
it to the service.
-->
- [ ] Yes, there are changes in this PR that warrants bumping the Pulumi Service API version
  <!-- `@Pulumi` employees: If yes, you must submit corresponding changes in the service repo. -->


Co-authored-by: Kyle Pitzen <kylepitzen@Kyles-MBP.fios-router.home>

.github/workflows/ci-run-test.yml

@@ -64,6 +64,7 @@ env:
   PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_PROD_ACCESS_TOKEN }}
   # Release builds use the service, PR checks and snapshots will use the local backend if possible.
   PULUMI_TEST_USE_SERVICE: ${{ !contains(inputs.version, '-') }}
+  PULUMI_TEST_PYTHON_SHARED_VENV: "true"
   PYTHON: python
   GO_TEST_PARALLELISM: 8
   GO_TEST_PKG_PARALLELISM: 2

.vscode/settings.json

@@ -1,6 +1,9 @@
 {
     "go.buildTags": "all",
     "go.testTimeout": "1h",
+    "go.testEnvVars": {
+        "PULUMI_TEST_PYTHON_SHARED_VENV": "true",
+    },
     "gopls": {
         // A couple of modules get copied as part of builds and this confuse gopls as it sees the module name twice, just ignore the copy in the build folders.
         "build.directoryFilters": [

changelog/pending/20221205--pkg-testing--optionally-caches-python-venvs-for-testing.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: fix
+  scope: pkg/testing
+  description: Optionally caches python venvs for testing

pkg/testing/integration/program.go

@@ -17,6 +17,7 @@ package integration
 import (
 	"context"
 	cryptorand "crypto/rand"
+	sha256 "crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -292,6 +293,12 @@ type ProgramTestOptions struct {
 	UseAutomaticVirtualEnv bool
 	// Use the Pipenv tool to manage the virtual environment.
 	UsePipenv bool
+	// Use a shared virtual environment for tests based on the contents of the requirements file. Defaults to false.
+	UseSharedVirtualEnv *bool
+	// Shared venv path when UseSharedVirtualEnv is true. Defaults to $HOME/.pulumi-test-venvs.
+	SharedVirtualEnvPath string
+	// Refers to the shared venv directory when UseSharedVirtualEnv is true. Otherwise defaults to venv
+	virtualEnvDir string
 
 	// If set, this hook is called after the `pulumi preview` command has completed.
 	PreviewCompletedHook func(dir string) error
@@ -314,6 +321,13 @@ type ProgramTestOptions struct {
 	LocalProviders []LocalDependency
 }
 
+func (opts *ProgramTestOptions) GetUseSharedVirtualEnv() bool {
+	if opts.UseSharedVirtualEnv != nil {
+		return *opts.UseSharedVirtualEnv
+	}
+	return false
+}
+
 type LocalDependency struct {
 	Package string
 	Path    string
@@ -375,6 +389,34 @@ func (opts *ProgramTestOptions) GetStackName() tokens.QName {
 	return tokens.QName(opts.StackName)
 }
 
+// Returns the md5 hash of the file at the given path as a string
+func hashFile(path string) (string, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	buf := make([]byte, 32*1024)
+	hash := sha256.New()
+	for {
+		n, err := file.Read(buf)
+		if n > 0 {
+			_, err := hash.Write(buf[:n])
+			if err != nil {
+				return "", err
+			}
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+	}
+	sum := string(hash.Sum(nil))
+	return sum, nil
+}
+
 // GetStackNameWithOwner gets the name of the stack prepended with an owner, if PULUMI_TEST_OWNER is set.
 // We use this in CI to create test stacks in an organization that all developers have access to, for debugging.
 func (opts *ProgramTestOptions) GetStackNameWithOwner() tokens.QName {
@@ -547,6 +589,12 @@ func (opts ProgramTestOptions) With(overrides ProgramTestOptions) ProgramTestOpt
 	if overrides.UsePipenv {
 		opts.UsePipenv = overrides.UsePipenv
 	}
+	if overrides.UseSharedVirtualEnv != nil {
+		opts.UseSharedVirtualEnv = overrides.UseSharedVirtualEnv
+	}
+	if overrides.SharedVirtualEnvPath != "" {
+		opts.SharedVirtualEnvPath = overrides.SharedVirtualEnvPath
+	}
 	if overrides.PreviewCompletedHook != nil {
 		opts.PreviewCompletedHook = overrides.PreviewCompletedHook
 	}
@@ -700,6 +748,24 @@ func prepareProgram(t *testing.T, opts *ProgramTestOptions) {
 		}
 	}
 
+	if opts.UseSharedVirtualEnv == nil {
+		if sharedVenv := os.Getenv("PULUMI_TEST_PYTHON_SHARED_VENV"); sharedVenv != "" {
+			useSharedVenvBool := sharedVenv == "true"
+			opts.UseSharedVirtualEnv = &useSharedVenvBool
+		}
+	}
+
+	if opts.virtualEnvDir == "" && !opts.GetUseSharedVirtualEnv() {
+		opts.virtualEnvDir = "venv"
+	}
+
+	if opts.SharedVirtualEnvPath == "" {
+		opts.SharedVirtualEnvPath = filepath.Join(os.Getenv("HOME"), ".pulumi-test-venvs")
+		if sharedVenvPath := os.Getenv("PULUMI_TEST_PYTHON_SHARED_VENV_PATH"); sharedVenvPath != "" {
+			opts.SharedVirtualEnvPath = sharedVenvPath
+		}
+	}
+
 	if opts.Quick {
 		opts.SkipPreview = true
 		opts.SkipExportImport = true
@@ -1025,7 +1091,7 @@ func (pt *ProgramTester) runVirtualEnvCommand(name string, args []string, wd str
 		}()
 	}
 
-	virtualenvBinPath, err := getVirtualenvBinPath(wd, args[0])
+	virtualenvBinPath, err := getVirtualenvBinPath(wd, args[0], pt)
 	if err != nil {
 		return err
 	}
@@ -1985,11 +2051,21 @@ func (pt *ProgramTester) preparePythonProject(projinfo *engine.Projinfo) error {
 			return err
 		}
 	} else {
-		if err = pt.runPythonCommand("python-venv", []string{"-m", "venv", "venv"}, cwd); err != nil {
+		venvPath := "venv"
+		if pt.opts.GetUseSharedVirtualEnv() {
+			requirementsPath := filepath.Join(cwd, "requirements.txt")
+			requirementsmd5, err := hashFile(requirementsPath)
+			if err != nil {
+				return err
+			}
+			pt.opts.virtualEnvDir = fmt.Sprintf("pulumi-venv-%x", requirementsmd5)
+			venvPath = filepath.Join(pt.opts.SharedVirtualEnvPath, pt.opts.virtualEnvDir)
+		}
+		if err = pt.runPythonCommand("python-venv", []string{"-m", "venv", venvPath}, cwd); err != nil {
 			return err
 		}
 
-		projinfo.Proj.Runtime.SetOption("virtualenv", "venv")
+		projinfo.Proj.Runtime.SetOption("virtualenv", venvPath)
 		projfile := filepath.Join(projinfo.Root, workspace.ProjectFile+".yaml")
 		if err = projinfo.Proj.Save(projfile); err != nil {
 			return fmt.Errorf("saving project: %w", err)
@@ -2077,10 +2153,14 @@ func (pt *ProgramTester) installPipPackageDeps(cwd string) error {
 	return nil
 }
 
-func getVirtualenvBinPath(cwd, bin string) (string, error) {
-	virtualenvBinPath := filepath.Join(cwd, "venv", "bin", bin)
+func getVirtualenvBinPath(cwd, bin string, pt *ProgramTester) (string, error) {
+	virtualEnvBasePath := filepath.Join(cwd, pt.opts.virtualEnvDir)
+	if pt.opts.GetUseSharedVirtualEnv() {
+		virtualEnvBasePath = filepath.Join(pt.opts.SharedVirtualEnvPath, pt.opts.virtualEnvDir)
+	}
+	virtualenvBinPath := filepath.Join(virtualEnvBasePath, "bin", bin)
 	if runtime.GOOS == windowsOS {
-		virtualenvBinPath = filepath.Join(cwd, "venv", "Scripts", fmt.Sprintf("%s.exe", bin))
+		virtualenvBinPath = filepath.Join(virtualEnvBasePath, "Scripts", fmt.Sprintf("%s.exe", bin))
 	}
 	if info, err := os.Stat(virtualenvBinPath); err != nil || info.IsDir() {
 		return "", fmt.Errorf("Expected %s to exist in virtual environment at %q", bin, virtualenvBinPath)

tests/integration/integration_python_smoke_test.go

@@ -31,6 +31,10 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/python"
 )
 
+func boolPointer(b bool) *bool {
+	return &b
+}
+
 // TestEmptyPython simply tests that we can run an empty Python project.
 func TestEmptyPython(t *testing.T) {
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
@@ -83,6 +87,7 @@ func TestDynamicPython(t *testing.T) {
 				assert.Equal(t, randomVal, stack.Outputs["random_val"].(string))
 			},
 		}},
+		UseSharedVirtualEnv: boolPointer(false),
 	})
 }
 
@@ -144,7 +149,8 @@ func optsForConstructPython(t *testing.T, expectedResourceCount int, localProvid
 		Secrets: map[string]string{
 			"secret": "this super secret is encrypted",
 		},
-		Quick: true,
+		Quick:               true,
+		UseSharedVirtualEnv: boolPointer(false),
 		ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) {
 			assert.NotNil(t, stackInfo.Deployment)
 			if assert.Equal(t, expectedResourceCount, len(stackInfo.Deployment.Resources)) {