Skip to content

Commit

Permalink
Run NodeJS conformance tests in parallel (#16078)
Browse files Browse the repository at this point in the history 
Had to make a fix in the test runner to not do sdk-gen in parallel, but
otherwise this was a fairly simple change to the node tests.
  • Loading branch information
@Frassle
Frassle committed Apr 28, 2024
1 parent 59efa58 commit 0063635
Show file tree
Hide file tree
Showing 2 changed files with 98 additions and 77 deletions.
161 changes: 90 additions & 71 deletions cmd/pulumi-test-language/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"slices"
"strconv"
"strings"
"sync"

mapset "github.com/deckarep/golang-set/v2"

Expand Down Expand Up @@ -100,6 +101,8 @@ type languageTestServer struct {
done chan error
addr string

sdkLock sync.Mutex

// Used by _bad snapshot_ tests to disable snapshot writing.
DisableSnapshotWriting bool
}
Expand Down Expand Up @@ -568,88 +571,104 @@ func (eng *languageTestServer) RunLanguageTest(

sdkName := fmt.Sprintf("%s-%s", pkg, spec.Version)
sdkTempDir := filepath.Join(token.TemporaryDirectory, "sdks", sdkName)
_, err = os.Stat(sdkTempDir)
if err == nil {
// If the directory already exists then we don't need to regenerate the SDK
sdkArtifact, err := languageClient.Pack(sdkTempDir, artifactsDir)
if err != nil {
return nil, fmt.Errorf("sdk packing for %s: %w", pkg, err)
// Multiple tests might try to generate the same SDK at the same time so we need to be atomic here. There's two
// ways to do that. 1 is to generate to a temporary directory and then atomic rename it but Go say it doesn't
// support that, so option 2 we just lock around this section.
//
// TODO[pulumi/issues/16079]: This could probably be a per-sdk lock to be more fine grained and allow more
// parallelism.
response, err := func() (*testingrpc.RunLanguageTestResponse, error) {
eng.sdkLock.Lock()
defer eng.sdkLock.Unlock()

_, err = os.Stat(sdkTempDir)
if err == nil {
// If the directory already exists then we don't need to regenerate the SDK
sdkArtifact, err := languageClient.Pack(sdkTempDir, artifactsDir)
if err != nil {
return nil, fmt.Errorf("sdk packing for %s: %w", pkg, err)
}
localDependencies[pkg] = sdkArtifact
return nil, nil
}
localDependencies[pkg] = sdkArtifact
continue
}

err = os.MkdirAll(sdkTempDir, 0o755)
if err != nil {
return nil, fmt.Errorf("create temp sdks dir: %w", err)
}
err = os.MkdirAll(sdkTempDir, 0o755)
if err != nil {
return nil, fmt.Errorf("create temp sdks dir: %w", err)
}

schemaBytes, err = boundSpec.MarshalJSON()
if err != nil {
return nil, fmt.Errorf("marshal schema for provider %s: %w", pkg, err)
}
schemaBytes, err = boundSpec.MarshalJSON()
if err != nil {
return nil, fmt.Errorf("marshal schema for provider %s: %w", pkg, err)
}

diags, err = languageClient.GeneratePackage(
sdkTempDir, string(schemaBytes), nil, grpcServer.Addr(), localDependencies)
if err != nil {
return makeTestResponse(fmt.Sprintf("generate package %s: %v", pkg, err)), nil
}
// TODO: Might be good to test warning diagnostics here
if diags.HasErrors() {
return makeTestResponse(fmt.Sprintf("generate package %s: %v", pkg, diags)), nil
}
diags, err = languageClient.GeneratePackage(
sdkTempDir, string(schemaBytes), nil, grpcServer.Addr(), localDependencies)
if err != nil {
return makeTestResponse(fmt.Sprintf("generate package %s: %v", pkg, err)), nil
}
// TODO: Might be good to test warning diagnostics here
if diags.HasErrors() {
return makeTestResponse(fmt.Sprintf("generate package %s: %v", pkg, diags)), nil
}

snapshotDir := filepath.Join(token.SnapshotDirectory, "sdks", sdkName)
sdkSnapshotDir, err := editSnapshot(sdkTempDir, snapshotEdits)
if err != nil {
return nil, fmt.Errorf("sdk snapshot creation for %s: %w", pkg, err)
}
validations, err := doSnapshot(eng.DisableSnapshotWriting, sdkSnapshotDir, snapshotDir)
// If we made a snapshot edit we can clean it up now
if sdkSnapshotDir != sdkTempDir {
err := os.RemoveAll(sdkSnapshotDir)
snapshotDir := filepath.Join(token.SnapshotDirectory, "sdks", sdkName)
sdkSnapshotDir, err := editSnapshot(sdkTempDir, snapshotEdits)
if err != nil {
return nil, fmt.Errorf("remove snapshot dir: %w", err)
return nil, fmt.Errorf("sdk snapshot creation for %s: %w", pkg, err)
}
validations, err := doSnapshot(eng.DisableSnapshotWriting, sdkSnapshotDir, snapshotDir)
// If we made a snapshot edit we can clean it up now
if sdkSnapshotDir != sdkTempDir {
err := os.RemoveAll(sdkSnapshotDir)
if err != nil {
return nil, fmt.Errorf("remove snapshot dir: %w", err)
}
}
if err != nil {
return nil, fmt.Errorf("sdk snapshot validation for %s: %w", pkg, err)
}
if len(validations) > 0 {
return makeTestResponse(
fmt.Sprintf("sdk snapshot validation for %s failed:\n%s",
pkg, strings.Join(validations, "\n"))), nil
}
}
if err != nil {
return nil, fmt.Errorf("sdk snapshot validation for %s: %w", pkg, err)
}
if len(validations) > 0 {
return makeTestResponse(
fmt.Sprintf("sdk snapshot validation for %s failed:\n%s",
pkg, strings.Join(validations, "\n"))), nil
}

// Pack the SDK and add it to the artifact dependencies, we do this in the temporary directory so that
// any intermediate build files don't end up getting captured in the snapshot folder.
sdkArtifact, err := languageClient.Pack(sdkTempDir, artifactsDir)
if err != nil {
return nil, fmt.Errorf("sdk packing for %s: %w", pkg, err)
}
localDependencies[pkg] = sdkArtifact
// Pack the SDK and add it to the artifact dependencies, we do this in the temporary directory so that
// any intermediate build files don't end up getting captured in the snapshot folder.
sdkArtifact, err := languageClient.Pack(sdkTempDir, artifactsDir)
if err != nil {
return nil, fmt.Errorf("sdk packing for %s: %w", pkg, err)
}
localDependencies[pkg] = sdkArtifact

// Check that packing the SDK didn't mutate any files, but it may have added ignorable build files.
// Again we need to make a snapshot edit for this.
sdkSnapshotDir, err = editSnapshot(sdkTempDir, snapshotEdits)
if err != nil {
return nil, fmt.Errorf("sdk snapshot creation for %s: %w", pkg, err)
}
validations, err = compareDirectories(sdkSnapshotDir, snapshotDir, true /* allowNewFiles */)
// If we made a snapshot edit we can clean it up now
if sdkSnapshotDir != sdkTempDir {
err := os.RemoveAll(sdkSnapshotDir)
// Check that packing the SDK didn't mutate any files, but it may have added ignorable build files.
// Again we need to make a snapshot edit for this.
sdkSnapshotDir, err = editSnapshot(sdkTempDir, snapshotEdits)
if err != nil {
return nil, fmt.Errorf("remove snapshot dir: %w", err)
return nil, fmt.Errorf("sdk snapshot creation for %s: %w", pkg, err)
}
}
if err != nil {
return nil, fmt.Errorf("sdk post pack change validation for %s: %w", pkg, err)
}
if len(validations) > 0 {
return makeTestResponse(
fmt.Sprintf("sdk post pack change validation for %s failed:\n%s",
pkg, strings.Join(validations, "\n"))), nil
validations, err = compareDirectories(sdkSnapshotDir, snapshotDir, true /* allowNewFiles */)
// If we made a snapshot edit we can clean it up now
if sdkSnapshotDir != sdkTempDir {
err := os.RemoveAll(sdkSnapshotDir)
if err != nil {
return nil, fmt.Errorf("remove snapshot dir: %w", err)
}
}
if err != nil {
return nil, fmt.Errorf("sdk post pack change validation for %s: %w", pkg, err)
}
if len(validations) > 0 {
return makeTestResponse(
fmt.Sprintf("sdk post pack change validation for %s failed:\n%s",
pkg, strings.Join(validations, "\n"))), nil
}

return nil, nil
}()
if response != nil || err != nil {
return response, err
}
}

Expand Down
14 changes: 8 additions & 6 deletions sdk/nodejs/cmd/pulumi-language-nodejs/language_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,8 @@ func TestLanguage(t *testing.T) {
require.NoError(t, err)

// We should run the nodejs tests twice. Once with tsc and once with ts-node.
//nolint:paralleltest // These aren't yet safe to run in parallel
for _, forceTsc := range []bool{false, true} {
forceTsc := forceTsc
cancel := make(chan bool)

// Run the language plugin
Expand Down Expand Up @@ -200,11 +200,11 @@ func TestLanguage(t *testing.T) {
})
require.NoError(t, err)

// TODO(https://github.com/pulumi/pulumi/issues/13945): enable parallel tests
//nolint:paralleltest // These aren't yet safe to run in parallel
for _, tt := range tests.Tests {
tt := tt
t.Run(tt, func(t *testing.T) {
t.Run(fmt.Sprintf("forceTsc=%v-/%s", forceTsc, tt), func(t *testing.T) {
t.Parallel()

result, err := engine.RunLanguageTest(context.Background(), &testingrpc.RunLanguageTestRequest{
Token: prepare.Token,
Test: tt,
Expand All @@ -220,7 +220,9 @@ func TestLanguage(t *testing.T) {
})
}

close(cancel)
assert.NoError(t, <-handle.Done)
t.Cleanup(func() {
close(cancel)
assert.NoError(t, <-handle.Done)
})
}
}

0 comments on commit 0063635

Please sign in to comment.