From 33c305e4fd9b94f4b4a93aea2b158a71fec818b2 Mon Sep 17 00:00:00 2001 From: Aaron Friel Date: Fri, 25 Nov 2022 13:48:05 -0800 Subject: [PATCH 1/4] ci: Make Windows smoke tests advisory only --- .github/workflows/ci-run-test.yml | 7 +++++++ .github/workflows/ci.yml | 2 ++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/ci-run-test.yml b/.github/workflows/ci-run-test.yml index f882ef78d56c..914a961928f3 100644 --- a/.github/workflows/ci-run-test.yml +++ b/.github/workflows/ci-run-test.yml @@ -50,6 +50,11 @@ on: "nodejs": "14.x", "python": "3.9.x" } + continue-on-error: + description: "Whether to continue running the job if the step fails" + required: false + default: false + type: boolean defaults: @@ -83,6 +88,8 @@ jobs: runs-on: ${{ inputs.platform }} + timeout-minutes: 60 + continue-on-error: ${{ inputs.continue-on-error }} steps: - name: "Windows cache workaround" # https://github.com/actions/cache/issues/752#issuecomment-1222415717 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4914670397b7..68530abe8e14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,6 +293,7 @@ jobs: name: Smoke Test${{ matrix.platform && '' }} needs: [matrix, build-binaries, build-sdks] if: ${{ needs.matrix.outputs.smoke-test-matrix != '{}' }} + # alow jobs to fail if the platform contains windows strategy: fail-fast: ${{ contains(needs.matrix.outputs.smoke-test-matrix, 'macos') }} matrix: ${{ fromJson(needs.matrix.outputs.smoke-test-matrix) }} @@ -309,6 +310,7 @@ jobs: # require-build: false # TODO, remove ${{ matrix.require-build || false }} version-set: ${{ toJson(matrix.version-set) }} + continue-on-error: ${{ contains(matrix.platform, 'windows') }} secrets: inherit test-collect-reports: From 55ed8c00afc243a34bfcc10f2d49c44557716be8 Mon Sep 17 00:00:00 2001 From: Aaron Friel Date: Sun, 4 Dec 2022 10:52:30 -0800 Subject: [PATCH 2/4] ci: use a larger runner size for Windows smoke tests --- scripts/get-job-matrix.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/get-job-matrix.py b/scripts/get-job-matrix.py index b955505ee8c6..c56411c2beec 100755 --- a/scripts/get-job-matrix.py +++ b/scripts/get-job-matrix.py @@ -391,6 +391,9 @@ def get_matrix( test_suites += run_gotestsum_ci_matrix_single_package(item, pkg_tests, tags) + if kind == JobKind.SMOKE_TEST: + platforms = list(map(lambda p: "windows-8core-2022" if p == "windows-latest" else p, platforms)) + return { "test-suite": test_suites, "platform": platforms, From 8457dfb0a0ce3ac41ac03df731472c0eb7c40a9c Mon Sep 17 00:00:00 2001 From: Aaron Friel Date: Mon, 5 Dec 2022 14:37:30 -0800 Subject: [PATCH 3/4] ci: Remove page file tweak for Windows --- .github/workflows/ci-run-test.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ci-run-test.yml b/.github/workflows/ci-run-test.yml index 914a961928f3..05ca346097df 100644 --- a/.github/workflows/ci-run-test.yml +++ b/.github/workflows/ci-run-test.yml @@ -132,14 +132,6 @@ jobs: if: ${{ inputs.enable-coverage && runner.os != 'Windows' }} run: | echo "PULUMI_TEST_COVERAGE_PATH=$(pwd)/coverage" >> "$GITHUB_ENV" - # See: https://github.com/actions/virtual-environments/issues/2642#issuecomment-774988591 - - name: Configure Windows pagefile - uses: aaronfriel/action-configure-pagefile@v2.0-beta.1 - if: ${{ runner.os == 'Windows' }} - with: - minimum-size: 4GB - maximum-size: 4GB - disk-root: "D:" - name: Configure Go Cache Key env: CACHE_KEY: "${{ fromJson(inputs.version-set).go }}-${{ runner.os }}-${{ runner.arch }}" From 592dc9267d05bf3dcf415474cb35858c0aa585b5 Mon Sep 17 00:00:00 2001 From: Kyle Pitzen Date: Tue, 6 Dec 2022 10:11:28 -0500 Subject: [PATCH 4/4] fix(sdk/python): Allow for duplicate output values in python programs I used the node SDK as inspiration for this change - there were a few things we were doing differently in python (specifically handling all cases in the outer layer of `massage` rather than allowing for more fine-grained control in a `massage_complex` function like in node. We also take the stack concept from the node SDK to resolve the immediate issue of duplicate outputs. --- ...for-duplicate-output-values-in-python.yaml | 4 ++ sdk/python/.gitignore | 1 + sdk/python/lib/pulumi/runtime/stack.py | 67 +++++++++++-------- .../test/langhost/stack_output/__main__.py | 5 ++ .../stack_output/test_stack_output.py | 2 + 5 files changed, 52 insertions(+), 27 deletions(-) create mode 100644 changelog/pending/20221206--sdk-python--allows-for-duplicate-output-values-in-python.yaml diff --git a/changelog/pending/20221206--sdk-python--allows-for-duplicate-output-values-in-python.yaml b/changelog/pending/20221206--sdk-python--allows-for-duplicate-output-values-in-python.yaml new file mode 100644 index 000000000000..a48c6a732b65 --- /dev/null +++ b/changelog/pending/20221206--sdk-python--allows-for-duplicate-output-values-in-python.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: sdk/python + description: Allows for duplicate output values in python diff --git a/sdk/python/.gitignore b/sdk/python/.gitignore index e10ec23834df..f073ece241e7 100644 --- a/sdk/python/.gitignore +++ b/sdk/python/.gitignore @@ -6,3 +6,4 @@ .venv/ venv/ .coverage +build/ diff --git a/sdk/python/lib/pulumi/runtime/stack.py b/sdk/python/lib/pulumi/runtime/stack.py index b78924185348..2ed495dbcc36 100644 --- a/sdk/python/lib/pulumi/runtime/stack.py +++ b/sdk/python/lib/pulumi/runtime/stack.py @@ -174,54 +174,67 @@ def massage(attr: Any, seen: List[Any]): if is_primitive(attr): return attr + if isinstance(attr, Output): + return attr.apply(lambda v: massage(v, seen)) + + if isawaitable(attr): + return Output.from_input(attr).apply(lambda v: massage(v, seen)) + # from this point on, we have complex objects. If we see them again, we don't want to emit them # again fully or else we'd loop infinitely. if reference_contains(attr, seen): # Note: for Resources we hit again, emit their urn so cycles can be easily understood in # the popo objects. if isinstance(attr, Resource): - return attr.urn - + return massage(attr.urn, seen) # otherwise just emit as nothing to stop the looping. return None - seen.append(attr) - - # first check if the value is an actual dictionary. If so, massage the values of it to deeply - # make sure this is a popo. - if isinstance(attr, dict): - result = {} - # Don't use attr.items() here, as it will error in the case of outputs with an `items` property. - for key in attr: - # ignore private keys - if not key.startswith("_"): - result[key] = massage(attr[key], seen) + try: + seen.append(attr) + return massage_complex(attr, seen) + finally: + popped = seen.pop() + if popped is not attr: + raise Exception("Invariant broken when processing stack outputs") - return result - if isinstance(attr, Output): - return attr.apply(lambda v: massage(v, seen)) +def massage_complex(attr: Any, seen: List[Any]) -> Any: + def is_public_key(key: str) -> bool: + return not key.startswith("_") - if isawaitable(attr): - return Output.from_input(attr).apply(lambda v: massage(v, seen)) + def serialize_all_keys(include: Callable[[str], bool]): + plain_object: Dict[str, Any] = {} + for key in attr.__dict__.keys(): + if include(key): + plain_object[key] = massage(attr.__dict__[key], seen) + return plain_object if isinstance(attr, Resource): - result = massage(attr.__dict__, seen) + serialized_attr = serialize_all_keys(is_public_key) # In preview only, we mark the result with "@isPulumiResource" to indicate that it is derived # from a resource. This allows the engine to perform resource-specific filtering of unknowns # from output diffs during a preview. This filtering is not necessary during an update because # all property values are known. - if is_dry_run(): - result["@isPulumiResource"] = True - return result + return ( + serialized_attr + if not is_dry_run() + else {**serialized_attr, "@isPulumiResource": True} + ) - if hasattr(attr, "__dict__"): - # recurse on the dictionary itself. It will be handled above. - return massage(attr.__dict__, seen) + # first check if the value is an actual dictionary. If so, massage the values of it to deeply + # make sure this is a popo. + if isinstance(attr, dict): + # Don't use attr.items() here, as it will error in the case of outputs with an `items` property. + return { + key: massage(attr[key], seen) for key in attr if not key.startswith("_") + } + + if hasattr(attr, "__iter__"): + return [massage(item, seen) for item in attr] - # finally, recurse through iterables, converting into a list of massaged values. - return [massage(a, seen) for a in attr] + return serialize_all_keys(is_public_key) def reference_contains(val1: Any, seen: List[Any]) -> bool: diff --git a/sdk/python/lib/test/langhost/stack_output/__main__.py b/sdk/python/lib/test/langhost/stack_output/__main__.py index 8c14150b7f09..1489f2a21e69 100644 --- a/sdk/python/lib/test/langhost/stack_output/__main__.py +++ b/sdk/python/lib/test/langhost/stack_output/__main__.py @@ -18,6 +18,9 @@ def __init__(self): self.num = 1 self._private = 2 + +my_test_class_instance = TestClass() + recursive = {"a": 1} recursive["b"] = 2 recursive["c"] = recursive @@ -34,3 +37,5 @@ def __init__(self): pulumi.export("output", pulumi.Output.from_input(1)) pulumi.export("class", TestClass()) pulumi.export("recursive", recursive) +pulumi.export("duplicate_output_0", my_test_class_instance) +pulumi.export("duplicate_output_1", my_test_class_instance) diff --git a/sdk/python/lib/test/langhost/stack_output/test_stack_output.py b/sdk/python/lib/test/langhost/stack_output/test_stack_output.py index ccd4b37e1d51..5810f69fcddf 100644 --- a/sdk/python/lib/test/langhost/stack_output/test_stack_output.py +++ b/sdk/python/lib/test/langhost/stack_output/test_stack_output.py @@ -39,4 +39,6 @@ def register_resource_outputs(self, _ctx, _dry_run, _urn, ty, _name, _resource, "output": 1.0, "class": {"num": 1.0}, "recursive": {"a": 1.0, "b": 2.0}, + "duplicate_output_0": {'num': 1.0}, + "duplicate_output_1": {'num': 1.0}, }, outputs)