fix(testing): Adds cached python venvs for integration tests

.github/workflows/ci-run-test.yml

@@ -64,6 +64,7 @@ env:
   PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_PROD_ACCESS_TOKEN }}
   # Release builds use the service, PR checks and snapshots will use the local backend if possible.
   PULUMI_TEST_USE_SERVICE: ${{ !contains(inputs.version, '-') }}
+  PULUMI_TEST_PYTHON_SHARED_VENV: "true"
   PYTHON: python
   GO_TEST_PARALLELISM: 8
   GO_TEST_PKG_PARALLELISM: 2

.vscode/settings.json

@@ -1,6 +1,9 @@
 {
     "go.buildTags": "all",
     "go.testTimeout": "1h",
+    "go.testEnvVars": {
+        "PULUMI_TEST_PYTHON_SHARED_VENV": "true",
+    },
     "gopls": {
         // A couple of modules get copied as part of builds and this confuse gopls as it sees the module name twice, just ignore the copy in the build folders.
         "build.directoryFilters": [

changelog/pending/20221205--pkg-testing--optionally-caches-python-venvs-for-testing.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: fix
+  scope: pkg/testing
+  description: Optionally caches python venvs for testing

pkg/testing/integration/program.go

@@ -17,6 +17,7 @@ package integration
 import (
 	"context"
 	cryptorand "crypto/rand"
+	sha256 "crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -292,6 +293,12 @@ type ProgramTestOptions struct {
 	UseAutomaticVirtualEnv bool
 	// Use the Pipenv tool to manage the virtual environment.
 	UsePipenv bool
+	// Use a shared virtual environment for tests based on the contents of the requirements file. Defaults to false.
+	UseSharedVirtualEnv *bool
+	// Shared venv path when UseSharedVirtualEnv is true. Defaults to $HOME/.pulumi-test-venvs.
+	SharedVirtualEnvPath string
+	// Refers to the shared venv directory when UseSharedVirtualEnv is true. Otherwise defaults to venv
+	virtualEnvDir string
 
 	// If set, this hook is called after the `pulumi preview` command has completed.
 	PreviewCompletedHook func(dir string) error
@@ -314,6 +321,13 @@ type ProgramTestOptions struct {
 	LocalProviders []LocalDependency
 }
 
+func (opts *ProgramTestOptions) GetUseSharedVirtualEnv() bool {
+	if opts.UseSharedVirtualEnv != nil {
+		return *opts.UseSharedVirtualEnv
+	}
+	return false
+}
+
 type LocalDependency struct {
 	Package string
 	Path    string
@@ -375,6 +389,34 @@ func (opts *ProgramTestOptions) GetStackName() tokens.QName {
 	return tokens.QName(opts.StackName)
 }
 
+// Returns the md5 hash of the file at the given path as a string
+func hashFile(path string) (string, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	buf := make([]byte, 32*1024)
+	hash := sha256.New()
+	for {
+		n, err := file.Read(buf)
+		if n > 0 {
+			_, err := hash.Write(buf[:n])
+			if err != nil {
+				return "", err
+			}
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+	}
+	sum := string(hash.Sum(nil))
+	return sum, nil
+}
+
 // GetStackNameWithOwner gets the name of the stack prepended with an owner, if PULUMI_TEST_OWNER is set.
 // We use this in CI to create test stacks in an organization that all developers have access to, for debugging.
 func (opts *ProgramTestOptions) GetStackNameWithOwner() tokens.QName {
@@ -547,6 +589,12 @@ func (opts ProgramTestOptions) With(overrides ProgramTestOptions) ProgramTestOpt
 	if overrides.UsePipenv {
 		opts.UsePipenv = overrides.UsePipenv
 	}
+	if overrides.UseSharedVirtualEnv != nil {
+		opts.UseSharedVirtualEnv = overrides.UseSharedVirtualEnv
+	}
+	if overrides.SharedVirtualEnvPath != "" {
+		opts.SharedVirtualEnvPath = overrides.SharedVirtualEnvPath
+	}
 	if overrides.PreviewCompletedHook != nil {
 		opts.PreviewCompletedHook = overrides.PreviewCompletedHook
 	}
@@ -700,6 +748,24 @@ func prepareProgram(t *testing.T, opts *ProgramTestOptions) {
 		}
 	}
 
+	if opts.UseSharedVirtualEnv == nil {
+		if sharedVenv := os.Getenv("PULUMI_TEST_PYTHON_SHARED_VENV"); sharedVenv != "" {
+			useSharedVenvBool := sharedVenv == "true"
+			opts.UseSharedVirtualEnv = &useSharedVenvBool
+		}
+	}
+
+	if opts.virtualEnvDir == "" && !opts.GetUseSharedVirtualEnv() {
+		opts.virtualEnvDir = "venv"
+	}
+
+	if opts.SharedVirtualEnvPath == "" {
+		opts.SharedVirtualEnvPath = filepath.Join(os.Getenv("HOME"), ".pulumi-test-venvs")
+		if sharedVenvPath := os.Getenv("PULUMI_TEST_PYTHON_SHARED_VENV_PATH"); sharedVenvPath != "" {
+			opts.SharedVirtualEnvPath = sharedVenvPath
+		}
+	}
+
 	if opts.Quick {
 		opts.SkipPreview = true
 		opts.SkipExportImport = true
@@ -1025,7 +1091,7 @@ func (pt *ProgramTester) runVirtualEnvCommand(name string, args []string, wd str
 		}()
 	}
 
-	virtualenvBinPath, err := getVirtualenvBinPath(wd, args[0])
+	virtualenvBinPath, err := getVirtualenvBinPath(wd, args[0], pt)
 	if err != nil {
 		return err
 	}
@@ -1985,11 +2051,21 @@ func (pt *ProgramTester) preparePythonProject(projinfo *engine.Projinfo) error {
 			return err
 		}
 	} else {
-		if err = pt.runPythonCommand("python-venv", []string{"-m", "venv", "venv"}, cwd); err != nil {
+		venvPath := "venv"
+		if pt.opts.GetUseSharedVirtualEnv() {
+			requirementsPath := filepath.Join(cwd, "requirements.txt")
+			requirementsmd5, err := hashFile(requirementsPath)
+			if err != nil {
+				return err
+			}
+			pt.opts.virtualEnvDir = fmt.Sprintf("pulumi-venv-%x", requirementsmd5)
+			venvPath = filepath.Join(pt.opts.SharedVirtualEnvPath, pt.opts.virtualEnvDir)
+		}
+		if err = pt.runPythonCommand("python-venv", []string{"-m", "venv", venvPath}, cwd); err != nil {
 			return err
 		}
 
-		projinfo.Proj.Runtime.SetOption("virtualenv", "venv")
+		projinfo.Proj.Runtime.SetOption("virtualenv", venvPath)
 		projfile := filepath.Join(projinfo.Root, workspace.ProjectFile+".yaml")
 		if err = projinfo.Proj.Save(projfile); err != nil {
 			return fmt.Errorf("saving project: %w", err)
@@ -2077,10 +2153,14 @@ func (pt *ProgramTester) installPipPackageDeps(cwd string) error {
 	return nil
 }
 
-func getVirtualenvBinPath(cwd, bin string) (string, error) {
-	virtualenvBinPath := filepath.Join(cwd, "venv", "bin", bin)
+func getVirtualenvBinPath(cwd, bin string, pt *ProgramTester) (string, error) {
+	virtualEnvBasePath := filepath.Join(cwd, pt.opts.virtualEnvDir)
+	if pt.opts.GetUseSharedVirtualEnv() {
+		virtualEnvBasePath = filepath.Join(pt.opts.SharedVirtualEnvPath, pt.opts.virtualEnvDir)
+	}
+	virtualenvBinPath := filepath.Join(virtualEnvBasePath, "bin", bin)
 	if runtime.GOOS == windowsOS {
-		virtualenvBinPath = filepath.Join(cwd, "venv", "Scripts", fmt.Sprintf("%s.exe", bin))
+		virtualenvBinPath = filepath.Join(virtualEnvBasePath, "Scripts", fmt.Sprintf("%s.exe", bin))
 	}
 	if info, err := os.Stat(virtualenvBinPath); err != nil || info.IsDir() {
 		return "", fmt.Errorf("Expected %s to exist in virtual environment at %q", bin, virtualenvBinPath)

tests/integration/integration_python_smoke_test.go

@@ -31,6 +31,10 @@ import (
 	"github.com/pulumi/pulumi/sdk/v3/python"
 )
 
+func boolPointer(b bool) *bool {
+	return &b
+}
+
 // TestEmptyPython simply tests that we can run an empty Python project.
 func TestEmptyPython(t *testing.T) {
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
@@ -83,6 +87,7 @@ func TestDynamicPython(t *testing.T) {
 				assert.Equal(t, randomVal, stack.Outputs["random_val"].(string))
 			},
 		}},
+		UseSharedVirtualEnv: boolPointer(false),
 	})
 }
 
@@ -144,7 +149,8 @@ func optsForConstructPython(t *testing.T, expectedResourceCount int, localProvid
 		Secrets: map[string]string{
 			"secret": "this super secret is encrypted",
 		},
-		Quick: true,
+		Quick:               true,
+		UseSharedVirtualEnv: boolPointer(false),
 		ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) {
 			assert.NotNil(t, stackInfo.Deployment)
 			if assert.Equal(t, expectedResourceCount, len(stackInfo.Deployment.Resources)) {