From 4cfe8ccedcc863e77ebe1208b4ca293219fac229 Mon Sep 17 00:00:00 2001 From: Kyle Pitzen Date: Mon, 5 Dec 2022 12:23:37 -0500 Subject: [PATCH] fix(testing): Adds cached python venvs for integration tests The cache key is based on hashing the contents of "requirements.txt" for each python test. This should result in unchanged test results among tests sharing venvs, and maintains environment isolation across requirement sets --- .github/workflows/ci-run-test.yml | 1 + ...nally-caches-python-venvs-for-testing.yaml | 4 + pkg/testing/integration/program.go | 84 +++++++++++++++++-- 3 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 changelog/pending/20221205--pkg-testing--optionally-caches-python-venvs-for-testing.yaml diff --git a/.github/workflows/ci-run-test.yml b/.github/workflows/ci-run-test.yml index f882ef78d56c..46dba21b8a06 100644 --- a/.github/workflows/ci-run-test.yml +++ b/.github/workflows/ci-run-test.yml @@ -64,6 +64,7 @@ env: PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_PROD_ACCESS_TOKEN }} # Release builds use the service, PR checks and snapshots will use the local backend if possible. PULUMI_TEST_USE_SERVICE: ${{ !contains(inputs.version, '-') }} + PULUMI_TEST_PYTHON_SHARED_VENV: "true" PYTHON: python GO_TEST_PARALLELISM: 8 GO_TEST_PKG_PARALLELISM: 2 diff --git a/changelog/pending/20221205--pkg-testing--optionally-caches-python-venvs-for-testing.yaml b/changelog/pending/20221205--pkg-testing--optionally-caches-python-venvs-for-testing.yaml new file mode 100644 index 000000000000..374d3cff10d3 --- /dev/null +++ b/changelog/pending/20221205--pkg-testing--optionally-caches-python-venvs-for-testing.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: pkg/testing + description: Optionally caches python venvs for testing diff --git a/pkg/testing/integration/program.go b/pkg/testing/integration/program.go index d902e231c32b..0620c26efc4c 100644 --- a/pkg/testing/integration/program.go +++ b/pkg/testing/integration/program.go @@ -17,6 +17,7 @@ package integration import ( "context" cryptorand "crypto/rand" + sha256 "crypto/sha256" "encoding/hex" "encoding/json" "errors" @@ -292,6 +293,12 @@ type ProgramTestOptions struct { UseAutomaticVirtualEnv bool // Use the Pipenv tool to manage the virtual environment. UsePipenv bool + // Use a shared virtual environment for tests based on the contents of the requirements file. Defaults to false. + UseSharedVirtualEnv bool + // Shared venv path when UseSharedVirtualEnv is true. Defaults to $HOME/.pulumi-test-venvs. + SharedVirtualEnvPath string + // Refers to the shared venv directory when UseSharedVirtualEnv is true. Otherwise defaults to venv + virtualEnvDir string // If set, this hook is called after the `pulumi preview` command has completed. PreviewCompletedHook func(dir string) error @@ -375,6 +382,34 @@ func (opts *ProgramTestOptions) GetStackName() tokens.QName { return tokens.QName(opts.StackName) } +// Returns the md5 hash of the file at the given path as a string +func hashFile(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + buf := make([]byte, 32*1024) + hash := sha256.New() + for { + n, err := file.Read(buf) + if n > 0 { + _, err := hash.Write(buf[:n]) + if err != nil { + return "", err + } + } + if err == io.EOF { + break + } + if err != nil { + return "", err + } + } + sum := string(hash.Sum(nil)) + return sum, nil +} + // GetStackNameWithOwner gets the name of the stack prepended with an owner, if PULUMI_TEST_OWNER is set. // We use this in CI to create test stacks in an organization that all developers have access to, for debugging. func (opts *ProgramTestOptions) GetStackNameWithOwner() tokens.QName { @@ -547,6 +582,12 @@ func (opts ProgramTestOptions) With(overrides ProgramTestOptions) ProgramTestOpt if overrides.UsePipenv { opts.UsePipenv = overrides.UsePipenv } + if overrides.UseSharedVirtualEnv { + opts.UseSharedVirtualEnv = overrides.UseSharedVirtualEnv + } + if overrides.SharedVirtualEnvPath != "" { + opts.SharedVirtualEnvPath = overrides.SharedVirtualEnvPath + } if overrides.PreviewCompletedHook != nil { opts.PreviewCompletedHook = overrides.PreviewCompletedHook } @@ -700,6 +741,23 @@ func prepareProgram(t *testing.T, opts *ProgramTestOptions) { } } + if opts.UseSharedVirtualEnv == false { + if sharedVenv := os.Getenv("PULUMI_TEST_PYTHON_SHARED_VENV"); sharedVenv != "" { + opts.UseSharedVirtualEnv = sharedVenv == "true" + } + } + + if opts.virtualEnvDir == "" && !opts.UseSharedVirtualEnv { + opts.virtualEnvDir = "venv" + } + + if opts.SharedVirtualEnvPath == "" { + opts.SharedVirtualEnvPath = filepath.Join(os.Getenv("HOME"), ".pulumi-test-venvs") + if sharedVenvPath := os.Getenv("PULUMI_TEST_PYTHON_SHARED_VENV_PATH"); sharedVenvPath != "" { + opts.SharedVirtualEnvPath = sharedVenvPath + } + } + if opts.Quick { opts.SkipPreview = true opts.SkipExportImport = true @@ -1025,7 +1083,7 @@ func (pt *ProgramTester) runVirtualEnvCommand(name string, args []string, wd str }() } - virtualenvBinPath, err := getVirtualenvBinPath(wd, args[0]) + virtualenvBinPath, err := getVirtualenvBinPath(wd, args[0], pt) if err != nil { return err } @@ -1985,11 +2043,21 @@ func (pt *ProgramTester) preparePythonProject(projinfo *engine.Projinfo) error { return err } } else { - if err = pt.runPythonCommand("python-venv", []string{"-m", "venv", "venv"}, cwd); err != nil { + venvPath := "venv" + if pt.opts.UseSharedVirtualEnv { + requirementsPath := filepath.Join(cwd, "requirements.txt") + requirementsmd5, err := hashFile(requirementsPath) + if err != nil { + return err + } + pt.opts.virtualEnvDir = fmt.Sprintf("pulumi-venv-%x", requirementsmd5) + venvPath = filepath.Join(pt.opts.SharedVirtualEnvPath, pt.opts.virtualEnvDir) + } + if err = pt.runPythonCommand("python-venv", []string{"-m", "venv", venvPath}, cwd); err != nil { return err } - projinfo.Proj.Runtime.SetOption("virtualenv", "venv") + projinfo.Proj.Runtime.SetOption("virtualenv", venvPath) projfile := filepath.Join(projinfo.Root, workspace.ProjectFile+".yaml") if err = projinfo.Proj.Save(projfile); err != nil { return fmt.Errorf("saving project: %w", err) @@ -2077,10 +2145,14 @@ func (pt *ProgramTester) installPipPackageDeps(cwd string) error { return nil } -func getVirtualenvBinPath(cwd, bin string) (string, error) { - virtualenvBinPath := filepath.Join(cwd, "venv", "bin", bin) +func getVirtualenvBinPath(cwd, bin string, pt *ProgramTester) (string, error) { + virtualEnvBasePath := filepath.Join(cwd, pt.opts.virtualEnvDir) + if pt.opts.UseSharedVirtualEnv { + virtualEnvBasePath = filepath.Join(pt.opts.SharedVirtualEnvPath, pt.opts.virtualEnvDir) + } + virtualenvBinPath := filepath.Join(virtualEnvBasePath, "bin", bin) if runtime.GOOS == windowsOS { - virtualenvBinPath = filepath.Join(cwd, "venv", "Scripts", fmt.Sprintf("%s.exe", bin)) + virtualenvBinPath = filepath.Join(virtualEnvBasePath, "Scripts", fmt.Sprintf("%s.exe", bin)) } if info, err := os.Stat(virtualenvBinPath); err != nil || info.IsDir() { return "", fmt.Errorf("Expected %s to exist in virtual environment at %q", bin, virtualenvBinPath)