diff --git a/changelog/pending/20221108--yaml1-0-2.yaml b/changelog/pending/20221108--yaml1-0-2.yaml new file mode 100644 index 000000000000..dcc302358de6 --- /dev/null +++ b/changelog/pending/20221108--yaml1-0-2.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: yaml + description: "[Updates Pulumi YAML to v1.0.2](https://github.com/pulumi/pulumi-yaml/releases/tag/v1.0.2) which fixes a bug encountered using templates with project level config." diff --git a/pkg/go.mod b/pkg/go.mod index c3ba58e40325..613d1da58191 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -68,7 +68,7 @@ require ( github.com/muesli/cancelreader v0.2.2 github.com/natefinch/atomic v1.0.1 github.com/pulumi/pulumi-java/pkg v0.6.0 - github.com/pulumi/pulumi-yaml v1.0.1 + github.com/pulumi/pulumi-yaml v1.0.2 github.com/rivo/uniseg v0.2.0 github.com/segmentio/encoding v0.3.5 github.com/shirou/gopsutil/v3 v3.22.3 diff --git a/pkg/go.sum b/pkg/go.sum index 0ab7c4208212..40e319b70177 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -1471,8 +1471,8 @@ github.com/prometheus/prometheus v0.37.0/go.mod h1:egARUgz+K93zwqsVIAneFlLZefyGO github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pulumi/pulumi-java/pkg v0.6.0 h1:haiSQJlhrQIBBcR0r0aQCIF8i69e4znzRnHpaNQUchE= github.com/pulumi/pulumi-java/pkg v0.6.0/go.mod h1:xSK2B792P8zjwYZTHYapMM1RJdue2BpRFQNYObWO0C8= -github.com/pulumi/pulumi-yaml v1.0.1 h1:P+txHPqaRd1b8Pf8vLJ1fQZTjmwFp1FDo9mCmuxI+6A= -github.com/pulumi/pulumi-yaml v1.0.1/go.mod h1:vxV5TdH3Xk5HRHNftcDXSbsZFJcJY9ME6k4zD+xw9OY= +github.com/pulumi/pulumi-yaml v1.0.2 h1:8fuoFNJlYJm1ni1Fff8QsIvef3E+ilroTPDrYKyy088= +github.com/pulumi/pulumi-yaml v1.0.2/go.mod h1:FKly+y0x5onXHEZALNnFglr6ZZnro4Y/jlN4sYLKYeM= github.com/pulumi/ssh-agent v0.5.1 h1:7DT4FcZNHWBAp9BFI+k0+HeBYGWbJvilJ29ra/4FlRM= github.com/pulumi/ssh-agent v0.5.1/go.mod h1:e6cyz/FUcE3PcJZ0tiuygkRsnHnCZcSQoQU+APbnrVA= github.com/rakyll/embedmd v0.0.0-20171029212350-c8060a0752a2/go.mod h1:7jOTMgqac46PZcF54q6l2hkLEG8op93fZu61KmxWDV4= diff --git a/tests/integration/gather_plugin/dotnet/MyStack.cs b/tests/integration/gather_plugin/dotnet/MyStack.cs index d5de0fca8acf..a0f9927b2259 100644 --- a/tests/integration/gather_plugin/dotnet/MyStack.cs +++ b/tests/integration/gather_plugin/dotnet/MyStack.cs @@ -7,10 +7,10 @@ public MyStack() // Create an AWS resource (S3 Bucket) var r = new Random( "default", 10, new ComponentResourceOptions{ - PluginDownloadURL = "get.com", + PluginDownloadURL = "get.example.test", }); var provider = new Provider("explicit", new CustomResourceOptions{ - PluginDownloadURL = "get.pulumi/test/providers", + PluginDownloadURL = "get.pulumi.test/providers", }); var e = new Random("explicit", 8, new ComponentResourceOptions{ Provider = provider, diff --git a/tests/integration/gather_plugin/go/main.go b/tests/integration/gather_plugin/go/main.go index a2d6a629771b..f52902f69c8e 100644 --- a/tests/integration/gather_plugin/go/main.go +++ b/tests/integration/gather_plugin/go/main.go @@ -14,13 +14,13 @@ func main() { pulumi.Run(func(ctx *pulumi.Context) error { r, err := NewRandom(ctx, "default", &RandomArgs{ Length: pulumi.Int(10), - }, pulumi.PluginDownloadURL("get.com")) + }, pulumi.PluginDownloadURL("get.example.test")) if err != nil { return err } provider, err := NewProvider(ctx, "explicit", - pulumi.PluginDownloadURL("get.pulumi/test/providers")) + pulumi.PluginDownloadURL("get.pulumi.test/providers")) e, err := NewRandom(ctx, "explicit", &RandomArgs{ Length: pulumi.Int(8), }, pulumi.Provider(provider)) diff --git a/tests/integration/gather_plugin/nodejs/index.ts b/tests/integration/gather_plugin/nodejs/index.ts index caaaa3112534..07ffb569b832 100644 --- a/tests/integration/gather_plugin/nodejs/index.ts +++ b/tests/integration/gather_plugin/nodejs/index.ts @@ -17,12 +17,12 @@ class RandomProvider extends pulumi.ProviderResource { } const r = new Random("default", 10, { - pluginDownloadURL: "get.com", + pluginDownloadURL: "get.example.test", }); export const defaultProvider = r.result; const provider = new RandomProvider("explicit", { - pluginDownloadURL: "get.pulumi/test/providers", + pluginDownloadURL: "get.pulumi.test/providers", }); new Random("explicit", 8, { provider: provider }); diff --git a/tests/integration/gather_plugin/python/__main__.py b/tests/integration/gather_plugin/python/__main__.py index 7ad2b3012dca..41c0e1439f4e 100644 --- a/tests/integration/gather_plugin/python/__main__.py +++ b/tests/integration/gather_plugin/python/__main__.py @@ -18,8 +18,8 @@ class RandomProvider(Provider): def __init__(self, opts: Optional[ResourceOptions]=None): Provider.__init__(self, "testprovider", "provider", None, opts) -example_url = ResourceOptions(plugin_download_url="get.com") -provider_url = ResourceOptions(plugin_download_url="get.pulumi/test/providers") +example_url = ResourceOptions(plugin_download_url="get.example.test") +provider_url = ResourceOptions(plugin_download_url="get.pulumi.test/providers") # Create resource with specified PluginDownloadURL r = Random("default", length=10, opts=example_url) diff --git a/tests/integration/integration_dotnet_smoke_test.go b/tests/integration/integration_dotnet_smoke_test.go index 341ce754fccb..59f5e35d41f2 100644 --- a/tests/integration/integration_dotnet_smoke_test.go +++ b/tests/integration/integration_dotnet_smoke_test.go @@ -1,4 +1,17 @@ -// Copyright 2016-2020, Pulumi Corporation. All rights reserved. +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build dotnet || all package ints diff --git a/tests/integration/integration_dotnet_test.go b/tests/integration/integration_dotnet_test.go index 0ffacd528005..680a94c4c6f7 100644 --- a/tests/integration/integration_dotnet_test.go +++ b/tests/integration/integration_dotnet_test.go @@ -1,4 +1,17 @@ -// Copyright 2016-2020, Pulumi Corporation. All rights reserved. +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build (dotnet || all) && !smoke package ints diff --git a/tests/integration/integration_go_smoke_test.go b/tests/integration/integration_go_smoke_test.go index 43835a54a666..a4a7bed3012e 100644 --- a/tests/integration/integration_go_smoke_test.go +++ b/tests/integration/integration_go_smoke_test.go @@ -1,4 +1,17 @@ -// Copyright 2016-2021, Pulumi Corporation. All rights reserved. +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build go || all package ints @@ -162,14 +175,3 @@ func optsForConstructGo(t *testing.T, dir string, expectedResourceCount int, loc }, } } - -// TestRefreshGo simply tests that we can build and run an empty Go project with the `refresh` option set. -func TestRefreshGo(t *testing.T) { - integration.ProgramTest(t, &integration.ProgramTestOptions{ - Dir: filepath.Join("refresh", "go"), - Dependencies: []string{ - "github.com/pulumi/pulumi/sdk/v3", - }, - Quick: true, - }) -} diff --git a/tests/integration/integration_go_test.go b/tests/integration/integration_go_test.go index ad8d36050098..d3090005c1a6 100644 --- a/tests/integration/integration_go_test.go +++ b/tests/integration/integration_go_test.go @@ -1,4 +1,17 @@ -// Copyright 2016-2021, Pulumi Corporation. All rights reserved. +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build (go || all) && !smoke package ints @@ -787,3 +800,14 @@ func TestProjectMainGo(t *testing.T) { } integration.ProgramTest(t, &test) } + +// TestRefreshGo simply tests that we can build and run an empty Go project with the `refresh` option set. +func TestRefreshGo(t *testing.T) { + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join("refresh", "go"), + Dependencies: []string{ + "github.com/pulumi/pulumi/sdk/v3", + }, + Quick: true, + }) +} diff --git a/tests/integration/integration_nodejs_smoke_test.go b/tests/integration/integration_nodejs_smoke_test.go index a42324289e4e..c7b63f3d519c 100644 --- a/tests/integration/integration_nodejs_smoke_test.go +++ b/tests/integration/integration_nodejs_smoke_test.go @@ -1,4 +1,17 @@ -// Copyright 2016-2022, Pulumi Corporation. All rights reserved. +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build nodejs || all package ints diff --git a/tests/integration/integration_nodejs_test.go b/tests/integration/integration_nodejs_test.go index a927169d6102..559077da88e5 100644 --- a/tests/integration/integration_nodejs_test.go +++ b/tests/integration/integration_nodejs_test.go @@ -1,4 +1,17 @@ -// Copyright 2016-2022, Pulumi Corporation. All rights reserved. +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build (nodejs || all) && !smoke package ints diff --git a/tests/integration/integration_python_smoke_test.go b/tests/integration/integration_python_smoke_test.go index 67708ff9c77b..e8e9483648cf 100644 --- a/tests/integration/integration_python_smoke_test.go +++ b/tests/integration/integration_python_smoke_test.go @@ -1,4 +1,17 @@ -// Copyright 2016-2021, Pulumi Corporation. All rights reserved. +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build python || all package ints diff --git a/tests/integration/integration_python_test.go b/tests/integration/integration_python_test.go index db4de89efed0..0567df22e700 100644 --- a/tests/integration/integration_python_test.go +++ b/tests/integration/integration_python_test.go @@ -1,4 +1,17 @@ -// Copyright 2016-2021, Pulumi Corporation. All rights reserved. +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //go:build (python || all) && !smoke package ints diff --git a/tests/integration/integration_smoke_test.go b/tests/integration/integration_smoke_test.go new file mode 100644 index 000000000000..75d33381cdeb --- /dev/null +++ b/tests/integration/integration_smoke_test.go @@ -0,0 +1,199 @@ +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build all + +package ints + +import ( + "bytes" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/pulumi/pulumi/pkg/v3/testing/integration" + "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" + ptesting "github.com/pulumi/pulumi/sdk/v3/go/common/testing" + "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" +) + +// TestConfigSave ensures that config commands in the Pulumi CLI work as expected. +func TestConfigSave(t *testing.T) { + t.Parallel() + e := ptesting.NewEnvironment(t) + defer func() { + if !t.Failed() { + e.DeleteEnvironment() + } + }() + + // Initialize an empty stack. + path := filepath.Join(e.RootPath, "Pulumi.yaml") + project := workspace.Project{ + Name: "testing-config", + Runtime: workspace.NewProjectRuntimeInfo("nodejs", nil), + } + + err := project.Save(path) + assert.NoError(t, err) + e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) + e.RunCommand("pulumi", "stack", "init", "testing-2") + e.RunCommand("pulumi", "stack", "init", "testing-1") + + // Now configure and save a few different things: + e.RunCommand("pulumi", "config", "set", "configA", "value1") + e.RunCommand("pulumi", "config", "set", "configB", "value2", "--stack", "testing-2") + + e.RunCommand("pulumi", "stack", "select", "testing-2") + + e.RunCommand("pulumi", "config", "set", "configD", "value4") + e.RunCommand("pulumi", "config", "set", "configC", "value3", "--stack", "testing-1") + + // Now read back the config using the CLI: + { + stdout, _ := e.RunCommand("pulumi", "config", "get", "configB") + assert.Equal(t, "value2\n", stdout) + } + { + // the config in a different stack, so this should error. + stdout, stderr := e.RunCommandExpectError("pulumi", "config", "get", "configA") + assert.Equal(t, "", stdout) + assert.NotEqual(t, "", stderr) + } + { + // but selecting the stack should let you see it + stdout, _ := e.RunCommand("pulumi", "config", "get", "configA", "--stack", "testing-1") + assert.Equal(t, "value1\n", stdout) + } + + // Finally, check that the stack file contains what we expected. + validate := func(k string, v string, cfg config.Map) { + key, err := config.ParseKey("testing-config:config:" + k) + assert.NoError(t, err) + d, ok := cfg[key] + assert.True(t, ok, "config key %v should be set", k) + dv, err := d.Value(nil) + assert.NoError(t, err) + assert.Equal(t, v, dv) + } + + testStack1, err := workspace.LoadProjectStack(&project, filepath.Join(e.CWD, "Pulumi.testing-1.yaml")) + assert.NoError(t, err) + testStack2, err := workspace.LoadProjectStack(&project, filepath.Join(e.CWD, "Pulumi.testing-2.yaml")) + assert.NoError(t, err) + + assert.Equal(t, 2, len(testStack1.Config)) + assert.Equal(t, 2, len(testStack2.Config)) + + validate("configA", "value1", testStack1.Config) + validate("configC", "value3", testStack1.Config) + + validate("configB", "value2", testStack2.Config) + validate("configD", "value4", testStack2.Config) + + e.RunCommand("pulumi", "stack", "rm", "--yes") +} + +func TestRotatePassphrase(t *testing.T) { + t.Parallel() + + e := ptesting.NewEnvironment(t) + defer func() { + if !t.Failed() { + e.DeleteEnvironment() + } + }() + + e.ImportDirectory("rotate_passphrase") + e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) + + e.RunCommand("pulumi", "stack", "init", "dev") + e.RunCommand("pulumi", "up", "--skip-preview", "--yes") + + e.RunCommand("pulumi", "config", "set", "--secret", "foo", "bar") + + e.SetEnvVars("PULUMI_TEST_PASSPHRASE=true") + e.Stdin = strings.NewReader("qwerty\nqwerty\n") + e.RunCommand("pulumi", "stack", "change-secrets-provider", "passphrase") + + e.Stdin, e.Passphrase = nil, "qwerty" + e.RunCommand("pulumi", "config", "get", "foo") +} + +//nolint:paralleltest // uses parallel programtest +func TestJSONOutputWithStreamingPreview(t *testing.T) { + stdout := &bytes.Buffer{} + + // Test with env var for streaming preview (should *not* print previewSummary). + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join("stack_outputs", "nodejs"), + Dependencies: []string{"@pulumi/pulumi"}, + Stdout: stdout, + Verbose: true, + JSONOutput: true, + Env: []string{"PULUMI_ENABLE_STREAMING_JSON_PREVIEW=1"}, + ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) { + output := stdout.String() + + // Check that the previewSummary is *not* present. + assert.NotRegexp(t, previewSummaryRegex, output) + + // Check that each event present in the event stream is also in stdout. + for _, evt := range stack.Events { + assertOutputContainsEvent(t, evt, output) + } + }, + }) +} + +func TestPassphrasePrompting(t *testing.T) { + t.Parallel() + + e := ptesting.NewEnvironment(t) + defer func() { + if !t.Failed() { + e.DeleteEnvironment() + } + }() + + e.NoPassphrase = true + // Setting PULUMI_TEST_PASSPHRASE allows prompting (reading from stdin) + // even though the test won't be interactive. + e.SetEnvVars("PULUMI_TEST_PASSPHRASE=true") + + e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) + + e.Stdin = strings.NewReader("qwerty\nqwerty\n") + e.RunCommand("pulumi", "new", "go", + "--name", "pphraseprompt", + "--description", "A project that tests passphrase prompts", + "--stack", "dev", + "--secrets-provider", "passphrase", + "--yes", + "--force") + + e.Stdin = strings.NewReader("qwerty\n") + e.RunCommand("pulumi", "up", "--stack", "dev", "--skip-preview", "--yes") + + e.Stdin = strings.NewReader("qwerty\n") + e.RunCommand("pulumi", "stack", "export", "--stack", "dev", "--file", "stack.json") + + e.Stdin = strings.NewReader("qwerty\n") + e.RunCommand("pulumi", "stack", "import", "--stack", "dev", "--file", "stack.json") + + e.Stdin = strings.NewReader("qwerty\n") + e.RunCommand("pulumi", "destroy", "--stack", "dev", "--skip-preview", "--yes") +} diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index 996d287a6cbc..b0c8958714f2 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -1,71 +1,38 @@ -// Copyright 2016-2022, Pulumi Corporation. All rights reserved. +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build all package ints import ( - "bufio" "bytes" - "context" "encoding/json" - "errors" "fmt" "os" - "os/exec" "path/filepath" - "regexp" - "runtime" - "strings" "testing" - "time" - - "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" "github.com/stretchr/testify/assert" - "google.golang.org/grpc" "github.com/pulumi/pulumi/pkg/v3/resource/deploy/providers" "github.com/pulumi/pulumi/pkg/v3/testing/integration" - "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" + "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" ptesting "github.com/pulumi/pulumi/sdk/v3/go/common/testing" - "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" - "github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil" - "github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil" "github.com/pulumi/pulumi/sdk/v3/go/common/workspace" - pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go" ) -const WindowsOS = "windows" - -// assertPerfBenchmark implements the integration.TestStatsReporter interface, and reports test -// failures when a scenario exceeds the provided threshold. -type assertPerfBenchmark struct { - T *testing.T - MaxPreviewDuration time.Duration - MaxUpdateDuration time.Duration -} - -func (t assertPerfBenchmark) ReportCommand(stats integration.TestCommandStats) { - var maxDuration *time.Duration - if strings.HasPrefix(stats.StepName, "pulumi-preview") { - maxDuration = &t.MaxPreviewDuration - } - if strings.HasPrefix(stats.StepName, "pulumi-update") { - maxDuration = &t.MaxUpdateDuration - } - - if maxDuration != nil && *maxDuration != 0 { - if stats.ElapsedSeconds < maxDuration.Seconds() { - t.T.Logf( - "Test step %q was under threshold. %.2fs (max %.2fs)", - stats.StepName, stats.ElapsedSeconds, maxDuration.Seconds()) - } else { - t.T.Errorf( - "Test step %q took longer than expected. %.2fs vs. max %.2fs", - stats.StepName, stats.ElapsedSeconds, maxDuration.Seconds()) - } - } -} - // TestStackTagValidation verifies various error scenarios related to stack names and tags. func TestStackTagValidation(t *testing.T) { t.Parallel() @@ -150,83 +117,6 @@ func TestStackInitValidation(t *testing.T) { }) } -// TestConfigSave ensures that config commands in the Pulumi CLI work as expected. -func TestConfigSave(t *testing.T) { - t.Parallel() - e := ptesting.NewEnvironment(t) - defer func() { - if !t.Failed() { - e.DeleteEnvironment() - } - }() - - // Initialize an empty stack. - path := filepath.Join(e.RootPath, "Pulumi.yaml") - project := workspace.Project{ - Name: "testing-config", - Runtime: workspace.NewProjectRuntimeInfo("nodejs", nil), - } - - err := project.Save(path) - assert.NoError(t, err) - e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) - e.RunCommand("pulumi", "stack", "init", "testing-2") - e.RunCommand("pulumi", "stack", "init", "testing-1") - - // Now configure and save a few different things: - e.RunCommand("pulumi", "config", "set", "configA", "value1") - e.RunCommand("pulumi", "config", "set", "configB", "value2", "--stack", "testing-2") - - e.RunCommand("pulumi", "stack", "select", "testing-2") - - e.RunCommand("pulumi", "config", "set", "configD", "value4") - e.RunCommand("pulumi", "config", "set", "configC", "value3", "--stack", "testing-1") - - // Now read back the config using the CLI: - { - stdout, _ := e.RunCommand("pulumi", "config", "get", "configB") - assert.Equal(t, "value2\n", stdout) - } - { - // the config in a different stack, so this should error. - stdout, stderr := e.RunCommandExpectError("pulumi", "config", "get", "configA") - assert.Equal(t, "", stdout) - assert.NotEqual(t, "", stderr) - } - { - // but selecting the stack should let you see it - stdout, _ := e.RunCommand("pulumi", "config", "get", "configA", "--stack", "testing-1") - assert.Equal(t, "value1\n", stdout) - } - - // Finally, check that the stack file contains what we expected. - validate := func(k string, v string, cfg config.Map) { - key, err := config.ParseKey("testing-config:config:" + k) - assert.NoError(t, err) - d, ok := cfg[key] - assert.True(t, ok, "config key %v should be set", k) - dv, err := d.Value(nil) - assert.NoError(t, err) - assert.Equal(t, v, dv) - } - - testStack1, err := workspace.LoadProjectStack(&project, filepath.Join(e.CWD, "Pulumi.testing-1.yaml")) - assert.NoError(t, err) - testStack2, err := workspace.LoadProjectStack(&project, filepath.Join(e.CWD, "Pulumi.testing-2.yaml")) - assert.NoError(t, err) - - assert.Equal(t, 2, len(testStack1.Config)) - assert.Equal(t, 2, len(testStack2.Config)) - - validate("configA", "value1", testStack1.Config) - validate("configC", "value3", testStack1.Config) - - validate("configB", "value2", testStack2.Config) - validate("configD", "value4", testStack2.Config) - - e.RunCommand("pulumi", "stack", "rm", "--yes") -} - // TestConfigPaths ensures that config commands with paths work as expected. func TestConfigPaths(t *testing.T) { t.Parallel() @@ -622,351 +512,6 @@ func TestConfigPaths(t *testing.T) { e.RunCommand("pulumi", "stack", "rm", "--yes") } -//nolint:deadcode -func testComponentSlowLocalProvider(t *testing.T) integration.LocalDependency { - return integration.LocalDependency{ - Package: "testcomponent", - Path: filepath.Join("construct_component_slow", "testcomponent"), - } -} - -// nolint: unused,deadcode -func testComponentProviderSchema(t *testing.T, path string) { - t.Parallel() - - runComponentSetup(t, "component_provider_schema") - - tests := []struct { - name string - env []string - version int32 - expected string - expectedError string - }{ - { - name: "Default", - expected: "{}", - }, - { - name: "Schema", - env: []string{"INCLUDE_SCHEMA=true"}, - expected: `{"hello": "world"}`, - }, - { - name: "Invalid Version", - version: 15, - expectedError: "unsupported schema version 15", - }, - } - for _, test := range tests { - test := test - t.Run(test.name, func(t *testing.T) { - t.Parallel() - // Start the plugin binary. - cmd := exec.Command(path, "ignored") - cmd.Env = append(os.Environ(), test.env...) - stdout, err := cmd.StdoutPipe() - assert.NoError(t, err) - err = cmd.Start() - assert.NoError(t, err) - defer func() { - // Ignore the error as it may fail with access denied on Windows. - cmd.Process.Kill() // nolint: errcheck - }() - - // Read the port from standard output. - reader := bufio.NewReader(stdout) - bytes, err := reader.ReadBytes('\n') - assert.NoError(t, err) - port := strings.TrimSpace(string(bytes)) - - // Create a connection to the server. - conn, err := grpc.Dial("127.0.0.1:"+port, grpc.WithInsecure(), rpcutil.GrpcChannelOptions()) - assert.NoError(t, err) - client := pulumirpc.NewResourceProviderClient(conn) - - // Call GetSchema and verify the results. - resp, err := client.GetSchema(context.Background(), &pulumirpc.GetSchemaRequest{Version: test.version}) - if test.expectedError != "" { - assert.Error(t, err) - assert.Contains(t, err.Error(), test.expectedError) - } else { - assert.Equal(t, test.expected, resp.GetSchema()) - } - }) - } -} - -// Test remote component inputs properly handle unknowns. -// nolint: unused,deadcode -func testConstructUnknown(t *testing.T, lang string, dependencies ...string) { - t.Parallel() - - const testDir = "construct_component_unknown" - runComponentSetup(t, testDir) - - tests := []struct { - componentDir string - }{ - { - componentDir: "testcomponent", - }, - { - componentDir: "testcomponent-python", - }, - { - componentDir: "testcomponent-go", - }, - } - for _, test := range tests { - test := test - t.Run(test.componentDir, func(t *testing.T) { - localProviders := - []integration.LocalDependency{ - {Package: "testprovider", Path: buildTestProvider(t, filepath.Join("..", "testprovider"))}, - {Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir)}, - } - integration.ProgramTest(t, &integration.ProgramTestOptions{ - Dir: filepath.Join(testDir, lang), - Dependencies: dependencies, - LocalProviders: localProviders, - SkipRefresh: true, - SkipPreview: false, - SkipUpdate: true, - SkipExportImport: true, - SkipEmptyPreviewUpdate: true, - Quick: false, - }) - }) - } -} - -// Test methods properly handle unknowns. -// nolint: unused,deadcode -func testConstructMethodsUnknown(t *testing.T, lang string, dependencies ...string) { - t.Parallel() - - const testDir = "construct_component_methods_unknown" - runComponentSetup(t, testDir) - tests := []struct { - componentDir string - }{ - { - componentDir: "testcomponent", - }, - { - componentDir: "testcomponent-python", - }, - { - componentDir: "testcomponent-go", - }, - } - for _, test := range tests { - test := test - - t.Run(test.componentDir, func(t *testing.T) { - localProviders := - []integration.LocalDependency{ - {Package: "testprovider", Path: buildTestProvider(t, filepath.Join("..", "testprovider"))}, - {Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir)}, - } - integration.ProgramTest(t, &integration.ProgramTestOptions{ - Dir: filepath.Join(testDir, lang), - Dependencies: dependencies, - LocalProviders: localProviders, - SkipRefresh: true, - SkipPreview: false, - SkipUpdate: true, - SkipExportImport: true, - SkipEmptyPreviewUpdate: true, - Quick: false, - }) - }) - } -} - -func buildTestProvider(t *testing.T, providerDir string) string { - fn := func() { - providerName := "pulumi-resource-testprovider" - if runtime.GOOS == "windows" { - providerName += ".exe" - } - - _, err := os.Stat(filepath.Join(providerDir, providerName)) - if err == nil { - return - } else if errors.Is(err, os.ErrNotExist) { - // Not built yet, continue. - } else { - t.Fatalf("Unexpected error building test provider: %v", err) - } - - cmd := exec.Command("go", "build", "-o", providerName) - cmd.Dir = providerDir - output, err := cmd.CombinedOutput() - if err != nil { - contract.AssertNoErrorf(err, "failed to run setup script: %v", string(output)) - } - } - lockfile := filepath.Join(providerDir, ".lock") - timeout := 10 * time.Minute - synchronouslyDo(t, lockfile, timeout, fn) - - // Allows us to drop this in in places where providerDir was used: - return providerDir -} - -func runComponentSetup(t *testing.T, testDir string) { - ptesting.YarnInstallMutex.Lock() - defer ptesting.YarnInstallMutex.Unlock() - - setupFilename, err := filepath.Abs("component_setup.sh") - contract.AssertNoError(err) - // even for Windows, we want forward slashes as bash treats backslashes as escape sequences. - setupFilename = filepath.ToSlash(setupFilename) - fn := func() { - cmd := exec.Command("bash", setupFilename) - cmd.Dir = testDir - output, err := cmd.CombinedOutput() - if err != nil { - contract.AssertNoErrorf(err, "failed to run setup script: %v", string(output)) - } - } - lockfile := filepath.Join(testDir, ".lock") - timeout := 10 * time.Minute - synchronouslyDo(t, lockfile, timeout, fn) -} - -func synchronouslyDo(t *testing.T, lockfile string, timeout time.Duration, fn func()) { - mutex := fsutil.NewFileMutex(lockfile) - defer func() { - assert.NoError(t, mutex.Unlock()) - }() - - lockWait := make(chan struct{}, 1) - go func() { - for { - if err := mutex.Lock(); err != nil { - time.Sleep(1 * time.Second) - continue - } else { - break - } - } - - fn() - lockWait <- struct{}{} - }() - - select { - case <-time.After(timeout): - t.Fatalf("timed out waiting for lock on %s", lockfile) - case <-lockWait: - // waited for fn, success. - } -} - -// Test methods that create resources. -// nolint: unused,deadcode -func testConstructMethodsResources(t *testing.T, lang string, dependencies ...string) { - t.Parallel() - - const testDir = "construct_component_methods_resources" - runComponentSetup(t, testDir) - - tests := []struct { - componentDir string - }{ - { - componentDir: "testcomponent", - }, - { - componentDir: "testcomponent-python", - }, - { - componentDir: "testcomponent-go", - }, - } - for _, test := range tests { - test := test - t.Run(test.componentDir, func(t *testing.T) { - localProviders := - []integration.LocalDependency{ - {Package: "testprovider", Path: buildTestProvider(t, filepath.Join("..", "testprovider"))}, - {Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir)}, - } - integration.ProgramTest(t, &integration.ProgramTestOptions{ - Dir: filepath.Join(testDir, lang), - Dependencies: dependencies, - LocalProviders: localProviders, - Quick: true, - ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { - assert.NotNil(t, stackInfo.Deployment) - assert.Equal(t, 6, len(stackInfo.Deployment.Resources)) - var hasExpectedResource bool - var result string - for _, res := range stackInfo.Deployment.Resources { - if res.URN.Name().String() == "myrandom" { - hasExpectedResource = true - result = res.Outputs["result"].(string) - assert.Equal(t, float64(10), res.Inputs["length"]) - assert.Equal(t, 10, len(result)) - } - } - assert.True(t, hasExpectedResource) - assert.Equal(t, result, stackInfo.Outputs["result"]) - }, - }) - }) - } -} - -// Test failures returned from methods are observed. -// nolint: unused,deadcode -func testConstructMethodsErrors(t *testing.T, lang string, dependencies ...string) { - t.Parallel() - - const testDir = "construct_component_methods_errors" - runComponentSetup(t, testDir) - - tests := []struct { - componentDir string - }{ - { - componentDir: "testcomponent", - }, - { - componentDir: "testcomponent-python", - }, - { - componentDir: "testcomponent-go", - }, - } - for _, test := range tests { - test := test - t.Run(test.componentDir, func(t *testing.T) { - stderr := &bytes.Buffer{} - expectedError := "the failure reason (the failure property)" - - localProvider := integration.LocalDependency{ - Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir), - } - integration.ProgramTest(t, &integration.ProgramTestOptions{ - Dir: filepath.Join(testDir, lang), - Dependencies: dependencies, - LocalProviders: []integration.LocalDependency{localProvider}, - Quick: true, - Stderr: stderr, - ExpectFailure: true, - ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { - output := stderr.String() - assert.Contains(t, output, expectedError) - }, - }) - }) - } -} - //nolint:paralleltest // uses parallel programtest func TestDestroyStackRef(t *testing.T) { e := ptesting.NewEnvironment(t) @@ -989,44 +534,6 @@ func TestDestroyStackRef(t *testing.T) { e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "-s", "dev") } -func TestRotatePassphrase(t *testing.T) { - t.Parallel() - - e := ptesting.NewEnvironment(t) - defer func() { - if !t.Failed() { - e.DeleteEnvironment() - } - }() - - e.ImportDirectory("rotate_passphrase") - e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) - - e.RunCommand("pulumi", "stack", "init", "dev") - e.RunCommand("pulumi", "up", "--skip-preview", "--yes") - - e.RunCommand("pulumi", "config", "set", "--secret", "foo", "bar") - - e.SetEnvVars("PULUMI_TEST_PASSPHRASE=true") - e.Stdin = strings.NewReader("qwerty\nqwerty\n") - e.RunCommand("pulumi", "stack", "change-secrets-provider", "passphrase") - - e.Stdin, e.Passphrase = nil, "qwerty" - e.RunCommand("pulumi", "config", "get", "foo") -} - -var previewSummaryRegex = regexp.MustCompile( - `{\s+"steps": \[[\s\S]+],\s+"duration": \d+,\s+"changeSummary": {[\s\S]+}\s+}`) - -func assertOutputContainsEvent(t *testing.T, evt apitype.EngineEvent, output string) { - evtJSON := bytes.Buffer{} - encoder := json.NewEncoder(&evtJSON) - encoder.SetEscapeHTML(false) - err := encoder.Encode(evt) - assert.NoError(t, err) - assert.Contains(t, output, evtJSON.String()) -} - //nolint:paralleltest // uses parallel programtest func TestJSONOutput(t *testing.T) { stdout := &bytes.Buffer{} @@ -1052,97 +559,6 @@ func TestJSONOutput(t *testing.T) { }) } -//nolint:paralleltest // uses parallel programtest -func TestJSONOutputWithStreamingPreview(t *testing.T) { - stdout := &bytes.Buffer{} - - // Test with env var for streaming preview (should *not* print previewSummary). - integration.ProgramTest(t, &integration.ProgramTestOptions{ - Dir: filepath.Join("stack_outputs", "nodejs"), - Dependencies: []string{"@pulumi/pulumi"}, - Stdout: stdout, - Verbose: true, - JSONOutput: true, - Env: []string{"PULUMI_ENABLE_STREAMING_JSON_PREVIEW=1"}, - ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) { - output := stdout.String() - - // Check that the previewSummary is *not* present. - assert.NotRegexp(t, previewSummaryRegex, output) - - // Check that each event present in the event stream is also in stdout. - for _, evt := range stack.Events { - assertOutputContainsEvent(t, evt, output) - } - }, - }) -} - -func TestExcludeProtected(t *testing.T) { - t.Parallel() - e := ptesting.NewEnvironment(t) - defer func() { - if !t.Failed() { - e.DeleteEnvironment() - } - }() - - e.ImportDirectory("exclude_protected") - - e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) - - e.RunCommand("pulumi", "stack", "init", "dev") - - e.RunCommand("yarn", "link", "@pulumi/pulumi") - e.RunCommand("yarn", "install") - - e.RunCommand("pulumi", "up", "--skip-preview", "--yes") - - stdout, _ := e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "--exclude-protected") - assert.Contains(t, stdout, "All unprotected resources were destroyed. There are still 7 protected resources") - // We run the command again, but this time there are not unprotected resources to destroy. - stdout, _ = e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "--exclude-protected") - assert.Contains(t, stdout, "There were no unprotected resources to destroy. There are still 7") -} - -// nolint: unused,deadcode -func testConstructOutputValues(t *testing.T, lang string, dependencies ...string) { - t.Parallel() - - const testDir = "construct_component_output_values" - runComponentSetup(t, testDir) - - tests := []struct { - componentDir string - }{ - { - componentDir: "testcomponent", - }, - { - componentDir: "testcomponent-python", - }, - { - componentDir: "testcomponent-go", - }, - } - for _, test := range tests { - test := test - t.Run(test.componentDir, func(t *testing.T) { - localProviders := - []integration.LocalDependency{ - {Package: "testprovider", Path: buildTestProvider(t, filepath.Join("..", "testprovider"))}, - {Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir)}, - } - integration.ProgramTest(t, &integration.ProgramTestOptions{ - Dir: filepath.Join(testDir, lang), - Dependencies: dependencies, - LocalProviders: localProviders, - Quick: true, - }) - }) - } -} - func TestProviderDownloadURL(t *testing.T) { t.Parallel() @@ -1157,11 +573,11 @@ func TestProviderDownloadURL(t *testing.T) { for _, resource := range data.Resources { switch { case providers.IsDefaultProvider(resource.URN): - assert.Equalf(t, "get.com", resource.Inputs[urlKey], "Inputs") - assert.Equalf(t, "get.com", resource.Outputs[urlKey], "Outputs") + assert.Equalf(t, "get.example.test", resource.Inputs[urlKey], "Inputs") + assert.Equalf(t, "get.example.test", resource.Outputs[urlKey], "Outputs") case providers.IsProviderType(resource.Type): - assert.Equalf(t, "get.pulumi/test/providers", resource.Inputs[urlKey], "Inputs") - assert.Equal(t, "get.pulumi/test/providers", resource.Outputs[urlKey], "Outputs") + assert.Equalf(t, "get.pulumi.test/providers", resource.Inputs[urlKey], "Inputs") + assert.Equal(t, "get.pulumi.test/providers", resource.Outputs[urlKey], "Outputs") default: _, hasURL := resource.Inputs[urlKey] assert.False(t, hasURL) @@ -1203,29 +619,8 @@ func TestProviderDownloadURL(t *testing.T) { } } -// printfTestValidation is used by the TestPrintfXYZ test cases in the language-specific test -// files. It validates that there are a precise count of expected stdout/stderr lines in the test output. -// -//nolint:deadcode // The linter doesn't see the uses since the consumers are conditionally compiled tests. -func printfTestValidation(t *testing.T, stack integration.RuntimeValidationStackInfo) { - var foundStdout int - var foundStderr int - for _, ev := range stack.Events { - if de := ev.DiagnosticEvent; de != nil { - if strings.HasPrefix(de.Message, fmt.Sprintf("Line %d", foundStdout)) { - foundStdout++ - } else if strings.HasPrefix(de.Message, fmt.Sprintf("Errln %d", foundStderr+10)) { - foundStderr++ - } - } - } - assert.Equal(t, 11, foundStdout) - assert.Equal(t, 11, foundStderr) -} - -func TestPassphrasePrompting(t *testing.T) { +func TestExcludeProtected(t *testing.T) { t.Parallel() - e := ptesting.NewEnvironment(t) defer func() { if !t.Failed() { @@ -1233,31 +628,20 @@ func TestPassphrasePrompting(t *testing.T) { } }() - e.NoPassphrase = true - // Setting PULUMI_TEST_PASSPHRASE allows prompting (reading from stdin) - // even though the test won't be interactive. - e.SetEnvVars("PULUMI_TEST_PASSPHRASE=true") + e.ImportDirectory("exclude_protected") e.RunCommand("pulumi", "login", "--cloud-url", e.LocalURL()) - e.Stdin = strings.NewReader("qwerty\nqwerty\n") - e.RunCommand("pulumi", "new", "go", - "--name", "pphraseprompt", - "--description", "A project that tests passphrase prompts", - "--stack", "dev", - "--secrets-provider", "passphrase", - "--yes", - "--force") - - e.Stdin = strings.NewReader("qwerty\n") - e.RunCommand("pulumi", "up", "--stack", "dev", "--skip-preview", "--yes") + e.RunCommand("pulumi", "stack", "init", "dev") - e.Stdin = strings.NewReader("qwerty\n") - e.RunCommand("pulumi", "stack", "export", "--stack", "dev", "--file", "stack.json") + e.RunCommand("yarn", "link", "@pulumi/pulumi") + e.RunCommand("yarn", "install") - e.Stdin = strings.NewReader("qwerty\n") - e.RunCommand("pulumi", "stack", "import", "--stack", "dev", "--file", "stack.json") + e.RunCommand("pulumi", "up", "--skip-preview", "--yes") - e.Stdin = strings.NewReader("qwerty\n") - e.RunCommand("pulumi", "destroy", "--stack", "dev", "--skip-preview", "--yes") + stdout, _ := e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "--exclude-protected") + assert.Contains(t, stdout, "All unprotected resources were destroyed. There are still 7 protected resources") + // We run the command again, but this time there are not unprotected resources to destroy. + stdout, _ = e.RunCommand("pulumi", "destroy", "--skip-preview", "--yes", "--exclude-protected") + assert.Contains(t, stdout, "There were no unprotected resources to destroy. There are still 7") } diff --git a/tests/integration/integration_util_test.go b/tests/integration/integration_util_test.go new file mode 100644 index 000000000000..5f8f6768f984 --- /dev/null +++ b/tests/integration/integration_util_test.go @@ -0,0 +1,483 @@ +// Copyright 2016-2022, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The linter doesn't see the uses since the consumers are conditionally compiled tests. +// +// nolint:unused,deadcode,varcheck +package ints + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + "time" + + "github.com/pulumi/pulumi/pkg/v3/testing/integration" + "github.com/pulumi/pulumi/sdk/v3/go/common/apitype" + ptesting "github.com/pulumi/pulumi/sdk/v3/go/common/testing" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/fsutil" + "github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil" + pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go" + "github.com/stretchr/testify/assert" + "google.golang.org/grpc" +) + +const WindowsOS = "windows" + +// assertPerfBenchmark implements the integration.TestStatsReporter interface, and reports test +// failures when a scenario exceeds the provided threshold. +type assertPerfBenchmark struct { + T *testing.T + MaxPreviewDuration time.Duration + MaxUpdateDuration time.Duration +} + +func (t assertPerfBenchmark) ReportCommand(stats integration.TestCommandStats) { + var maxDuration *time.Duration + if strings.HasPrefix(stats.StepName, "pulumi-preview") { + maxDuration = &t.MaxPreviewDuration + } + if strings.HasPrefix(stats.StepName, "pulumi-update") { + maxDuration = &t.MaxUpdateDuration + } + + if maxDuration != nil && *maxDuration != 0 { + if stats.ElapsedSeconds < maxDuration.Seconds() { + t.T.Logf( + "Test step %q was under threshold. %.2fs (max %.2fs)", + stats.StepName, stats.ElapsedSeconds, maxDuration.Seconds()) + } else { + t.T.Errorf( + "Test step %q took longer than expected. %.2fs vs. max %.2fs", + stats.StepName, stats.ElapsedSeconds, maxDuration.Seconds()) + } + } +} + +func testComponentSlowLocalProvider(t *testing.T) integration.LocalDependency { + return integration.LocalDependency{ + Package: "testcomponent", + Path: filepath.Join("construct_component_slow", "testcomponent"), + } +} + +func testComponentProviderSchema(t *testing.T, path string) { + t.Parallel() + + runComponentSetup(t, "component_provider_schema") + + tests := []struct { + name string + env []string + version int32 + expected string + expectedError string + }{ + { + name: "Default", + expected: "{}", + }, + { + name: "Schema", + env: []string{"INCLUDE_SCHEMA=true"}, + expected: `{"hello": "world"}`, + }, + { + name: "Invalid Version", + version: 15, + expectedError: "unsupported schema version 15", + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + // Start the plugin binary. + cmd := exec.Command(path, "ignored") + cmd.Env = append(os.Environ(), test.env...) + stdout, err := cmd.StdoutPipe() + assert.NoError(t, err) + err = cmd.Start() + assert.NoError(t, err) + defer func() { + // Ignore the error as it may fail with access denied on Windows. + cmd.Process.Kill() // nolint: errcheck + }() + + // Read the port from standard output. + reader := bufio.NewReader(stdout) + bytes, err := reader.ReadBytes('\n') + assert.NoError(t, err) + port := strings.TrimSpace(string(bytes)) + + // Create a connection to the server. + conn, err := grpc.Dial("127.0.0.1:"+port, grpc.WithInsecure(), rpcutil.GrpcChannelOptions()) + assert.NoError(t, err) + client := pulumirpc.NewResourceProviderClient(conn) + + // Call GetSchema and verify the results. + resp, err := client.GetSchema(context.Background(), &pulumirpc.GetSchemaRequest{Version: test.version}) + if test.expectedError != "" { + assert.Error(t, err) + assert.Contains(t, err.Error(), test.expectedError) + } else { + assert.Equal(t, test.expected, resp.GetSchema()) + } + }) + } +} + +// Test remote component inputs properly handle unknowns. +func testConstructUnknown(t *testing.T, lang string, dependencies ...string) { + t.Parallel() + + const testDir = "construct_component_unknown" + runComponentSetup(t, testDir) + + tests := []struct { + componentDir string + }{ + { + componentDir: "testcomponent", + }, + { + componentDir: "testcomponent-python", + }, + { + componentDir: "testcomponent-go", + }, + } + for _, test := range tests { + test := test + t.Run(test.componentDir, func(t *testing.T) { + localProviders := + []integration.LocalDependency{ + {Package: "testprovider", Path: buildTestProvider(t, filepath.Join("..", "testprovider"))}, + {Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir)}, + } + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join(testDir, lang), + Dependencies: dependencies, + LocalProviders: localProviders, + SkipRefresh: true, + SkipPreview: false, + SkipUpdate: true, + SkipExportImport: true, + SkipEmptyPreviewUpdate: true, + Quick: false, + }) + }) + } +} + +// Test methods properly handle unknowns. +func testConstructMethodsUnknown(t *testing.T, lang string, dependencies ...string) { + t.Parallel() + + const testDir = "construct_component_methods_unknown" + runComponentSetup(t, testDir) + tests := []struct { + componentDir string + }{ + { + componentDir: "testcomponent", + }, + { + componentDir: "testcomponent-python", + }, + { + componentDir: "testcomponent-go", + }, + } + for _, test := range tests { + test := test + + t.Run(test.componentDir, func(t *testing.T) { + localProviders := + []integration.LocalDependency{ + {Package: "testprovider", Path: buildTestProvider(t, filepath.Join("..", "testprovider"))}, + {Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir)}, + } + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join(testDir, lang), + Dependencies: dependencies, + LocalProviders: localProviders, + SkipRefresh: true, + SkipPreview: false, + SkipUpdate: true, + SkipExportImport: true, + SkipEmptyPreviewUpdate: true, + Quick: false, + }) + }) + } +} + +func buildTestProvider(t *testing.T, providerDir string) string { + fn := func() { + providerName := "pulumi-resource-testprovider" + if runtime.GOOS == "windows" { + providerName += ".exe" + } + + _, err := os.Stat(filepath.Join(providerDir, providerName)) + if err == nil { + return + } else if errors.Is(err, os.ErrNotExist) { + // Not built yet, continue. + } else { + t.Fatalf("Unexpected error building test provider: %v", err) + } + + cmd := exec.Command("go", "build", "-o", providerName) + cmd.Dir = providerDir + output, err := cmd.CombinedOutput() + if err != nil { + contract.AssertNoErrorf(err, "failed to run setup script: %v", string(output)) + } + } + lockfile := filepath.Join(providerDir, ".lock") + timeout := 10 * time.Minute + synchronouslyDo(t, lockfile, timeout, fn) + + // Allows us to drop this in in places where providerDir was used: + return providerDir +} + +func runComponentSetup(t *testing.T, testDir string) { + ptesting.YarnInstallMutex.Lock() + defer ptesting.YarnInstallMutex.Unlock() + + setupFilename, err := filepath.Abs("component_setup.sh") + contract.AssertNoError(err) + // even for Windows, we want forward slashes as bash treats backslashes as escape sequences. + setupFilename = filepath.ToSlash(setupFilename) + fn := func() { + cmd := exec.Command("bash", setupFilename) + cmd.Dir = testDir + output, err := cmd.CombinedOutput() + if err != nil { + contract.AssertNoErrorf(err, "failed to run setup script: %v", string(output)) + } + } + lockfile := filepath.Join(testDir, ".lock") + timeout := 10 * time.Minute + synchronouslyDo(t, lockfile, timeout, fn) +} + +func synchronouslyDo(t *testing.T, lockfile string, timeout time.Duration, fn func()) { + mutex := fsutil.NewFileMutex(lockfile) + defer func() { + assert.NoError(t, mutex.Unlock()) + }() + + lockWait := make(chan struct{}, 1) + go func() { + for { + if err := mutex.Lock(); err != nil { + time.Sleep(1 * time.Second) + continue + } else { + break + } + } + + fn() + lockWait <- struct{}{} + }() + + select { + case <-time.After(timeout): + t.Fatalf("timed out waiting for lock on %s", lockfile) + case <-lockWait: + // waited for fn, success. + } +} + +// Test methods that create resources. +func testConstructMethodsResources(t *testing.T, lang string, dependencies ...string) { + t.Parallel() + + const testDir = "construct_component_methods_resources" + runComponentSetup(t, testDir) + + tests := []struct { + componentDir string + }{ + { + componentDir: "testcomponent", + }, + { + componentDir: "testcomponent-python", + }, + { + componentDir: "testcomponent-go", + }, + } + for _, test := range tests { + test := test + t.Run(test.componentDir, func(t *testing.T) { + localProviders := + []integration.LocalDependency{ + {Package: "testprovider", Path: buildTestProvider(t, filepath.Join("..", "testprovider"))}, + {Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir)}, + } + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join(testDir, lang), + Dependencies: dependencies, + LocalProviders: localProviders, + Quick: true, + ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { + assert.NotNil(t, stackInfo.Deployment) + assert.Equal(t, 6, len(stackInfo.Deployment.Resources)) + var hasExpectedResource bool + var result string + for _, res := range stackInfo.Deployment.Resources { + if res.URN.Name().String() == "myrandom" { + hasExpectedResource = true + result = res.Outputs["result"].(string) + assert.Equal(t, float64(10), res.Inputs["length"]) + assert.Equal(t, 10, len(result)) + } + } + assert.True(t, hasExpectedResource) + assert.Equal(t, result, stackInfo.Outputs["result"]) + }, + }) + }) + } +} + +// Test failures returned from methods are observed. +func testConstructMethodsErrors(t *testing.T, lang string, dependencies ...string) { + t.Parallel() + + const testDir = "construct_component_methods_errors" + runComponentSetup(t, testDir) + + tests := []struct { + componentDir string + }{ + { + componentDir: "testcomponent", + }, + { + componentDir: "testcomponent-python", + }, + { + componentDir: "testcomponent-go", + }, + } + for _, test := range tests { + test := test + t.Run(test.componentDir, func(t *testing.T) { + stderr := &bytes.Buffer{} + expectedError := "the failure reason (the failure property)" + + localProvider := integration.LocalDependency{ + Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir), + } + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join(testDir, lang), + Dependencies: dependencies, + LocalProviders: []integration.LocalDependency{localProvider}, + Quick: true, + Stderr: stderr, + ExpectFailure: true, + ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) { + output := stderr.String() + assert.Contains(t, output, expectedError) + }, + }) + }) + } +} + +func testConstructOutputValues(t *testing.T, lang string, dependencies ...string) { + t.Parallel() + + const testDir = "construct_component_output_values" + runComponentSetup(t, testDir) + + tests := []struct { + componentDir string + }{ + { + componentDir: "testcomponent", + }, + { + componentDir: "testcomponent-python", + }, + { + componentDir: "testcomponent-go", + }, + } + for _, test := range tests { + test := test + t.Run(test.componentDir, func(t *testing.T) { + localProviders := + []integration.LocalDependency{ + {Package: "testprovider", Path: buildTestProvider(t, filepath.Join("..", "testprovider"))}, + {Package: "testcomponent", Path: filepath.Join(testDir, test.componentDir)}, + } + integration.ProgramTest(t, &integration.ProgramTestOptions{ + Dir: filepath.Join(testDir, lang), + Dependencies: dependencies, + LocalProviders: localProviders, + Quick: true, + }) + }) + } +} + +var previewSummaryRegex = regexp.MustCompile( + `{\s+"steps": \[[\s\S]+],\s+"duration": \d+,\s+"changeSummary": {[\s\S]+}\s+}`) + +func assertOutputContainsEvent(t *testing.T, evt apitype.EngineEvent, output string) { + evtJSON := bytes.Buffer{} + encoder := json.NewEncoder(&evtJSON) + encoder.SetEscapeHTML(false) + err := encoder.Encode(evt) + assert.NoError(t, err) + assert.Contains(t, output, evtJSON.String()) +} + +// printfTestValidation is used by the TestPrintfXYZ test cases in the language-specific test +// files. It validates that there are a precise count of expected stdout/stderr lines in the test output. +func printfTestValidation(t *testing.T, stack integration.RuntimeValidationStackInfo) { + var foundStdout int + var foundStderr int + for _, ev := range stack.Events { + if de := ev.DiagnosticEvent; de != nil { + if strings.HasPrefix(de.Message, fmt.Sprintf("Line %d", foundStdout)) { + foundStdout++ + } else if strings.HasPrefix(de.Message, fmt.Sprintf("Errln %d", foundStderr+10)) { + foundStderr++ + } + } + } + assert.Equal(t, 11, foundStdout) + assert.Equal(t, 11, foundStderr) +}