feat: Introduce spec JSON schema (#14296)

Closes #14027

Blocked by:
* cloudquery/codegen#39
* invopop/jsonschema#109 – merged to `cloudquery/jsonschema@cqmain`
* invopop/jsonschema#110 – merged to `cloudquery/jsonschema@cqmain`

I propose reviewing the annotations along with tests, as the JSON schemas generated are just too long to grasp visually.

.gitattributes

@@ -1 +1,2 @@
 website/pages/docs/plugins/sources/**/tables.md linguist-generated
+**/schema.json linguist-generated

plugins/source/aws/Makefile

@@ -46,6 +46,10 @@ gen-docs: build
 	sed -i.bak -e 's_(\(.*\).md)_(\1)_' ../../../website/tables/aws/*.md
 	rm -rf ../../../website/tables/aws/*.bak
 
+.PHONY: gen-spec-schema
+gen-spec-schema:
+	go run client/spec/gen/main.go
+
 # All gen targets
 .PHONY: gen
-gen: gen-mocks gen-docs
+gen: gen-spec-schema gen-mocks gen-docs

plugins/source/aws/client/account.go

@@ -12,6 +12,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/aws/smithy-go"
 	"github.com/cloudquery/cloudquery/plugins/source/aws/client/services"
+	"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
 	"github.com/rs/zerolog"
 )
 
@@ -21,7 +22,7 @@ type svcsDetail struct {
 	svcs      Services
 }
 
-func (c *Client) setupAWSAccount(ctx context.Context, logger zerolog.Logger, awsPluginSpec *Spec, adminAccountSts AssumeRoleAPIClient, account Account) (*svcsDetail, error) {
+func (c *Client) setupAWSAccount(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec, adminAccountSts AssumeRoleAPIClient, account spec.Account) (*svcsDetail, error) {
 	if account.AccountName == "" {
 		account.AccountName = account.ID
 	}
@@ -46,7 +47,7 @@ func (c *Client) setupAWSAccount(ctx context.Context, logger zerolog.Logger, aws
 	awsCfg, err := ConfigureAwsSDK(ctx, logger, awsPluginSpec, account, adminAccountSts)
 	if err != nil {
 		warningMsg := logger.Warn().Str("account", account.AccountName).Err(err)
-		if account.source == "org" {
+		if account.Source == spec.AccountSourceOrg {
 			warningMsg.Msg("Unable to assume role in account")
 			return nil, nil
 		}

plugins/source/aws/client/aws_sdk.go

@@ -9,29 +9,24 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
+	"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
 	"github.com/rs/zerolog"
 )
 
-func ConfigureAwsSDK(ctx context.Context, logger zerolog.Logger, awsPluginSpec *Spec, account Account, stsClient AssumeRoleAPIClient) (aws.Config, error) {
+func ConfigureAwsSDK(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec, account spec.Account, stsClient AssumeRoleAPIClient) (aws.Config, error) {
 	var err error
 	var awsCfg aws.Config
 
-	maxAttempts := 10
-	if awsPluginSpec.MaxRetries != nil {
-		maxAttempts = *awsPluginSpec.MaxRetries
-	}
-	maxBackoff := 30
-	if awsPluginSpec.MaxBackoff != nil {
-		maxBackoff = *awsPluginSpec.MaxBackoff
-	}
+	// This sets MaxRetries & MaxBackoff, too
+	awsPluginSpec.SetDefaults()
 
 	configFns := []func(*config.LoadOptions) error{
 		config.WithDefaultRegion(defaultRegion),
 		// https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/retries-timeouts/
 		config.WithRetryer(func() aws.Retryer {
 			return retry.NewStandard(func(so *retry.StandardOptions) {
-				so.MaxAttempts = maxAttempts
-				so.MaxBackoff = time.Duration(maxBackoff) * time.Second
+				so.MaxAttempts = *awsPluginSpec.MaxRetries
+				so.MaxBackoff = time.Duration(*awsPluginSpec.MaxBackoff) * time.Second
 				so.RateLimiter = &NoRateLimiter{}
 			})
 		}),

plugins/source/aws/client/client.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	wafv2types "github.com/aws/aws-sdk-go-v2/service/wafv2/types"
 	"github.com/aws/smithy-go/logging"
+	"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
 	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/cloudquery/plugin-sdk/v4/state"
 	"github.com/rs/zerolog"
@@ -32,7 +33,7 @@ type Client struct {
 	LanguageCode         string
 	Backend              state.Client
 	specificRegions      bool
-	Spec                 *Spec
+	Spec                 *spec.Spec
 	accountMutex         map[string]*sync.Mutex
 }
 
@@ -88,13 +89,13 @@ func (s *ServicesManager) InitServicesForPartitionAccount(partition, accountId s
 	s.services[partition][accountId].Regions = funk.UniqString(append(s.services[partition][accountId].Regions, svcs.Regions...))
 }
 
-func NewAwsClient(logger zerolog.Logger, spec *Spec) Client {
+func NewAwsClient(logger zerolog.Logger, s *spec.Spec) Client {
 	return Client{
 		ServicesManager: &ServicesManager{
 			services: ServicesPartitionAccountMap{},
 		},
 		logger:       logger,
-		Spec:         spec,
+		Spec:         s,
 		accountMutex: map[string]*sync.Mutex{},
 	}
 }
@@ -200,14 +201,14 @@ func (c *Client) withLanguageCode(code string) *Client {
 }
 
 // Configure is the entrypoint into configuring the AWS plugin. It is called by the plugin initialization in resources/plugin/aws.go
-func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.ClientMeta, error) {
-	if err := spec.Validate(); err != nil {
+func Configure(ctx context.Context, logger zerolog.Logger, s spec.Spec) (schema.ClientMeta, error) {
+	if err := s.Validate(); err != nil {
 		return nil, fmt.Errorf("spec validation failed: %w", err)
 	}
-	spec.SetDefaults()
+	s.SetDefaults()
 
-	if spec.TableOptions != nil {
-		structVal := reflect.ValueOf(*spec.TableOptions)
+	if s.TableOptions != nil {
+		structVal := reflect.ValueOf(*s.TableOptions)
 		fieldNum := structVal.NumField()
 		for i := 0; i < fieldNum; i++ {
 			field := structVal.Field(i)
@@ -218,7 +219,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.Cl
 		}
 	}
 
-	client := NewAwsClient(logger, &spec)
+	client := NewAwsClient(logger, &s)
 
 	var adminAccountSts AssumeRoleAPIClient
 
@@ -231,11 +232,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec Spec) (schema.Cl
 		}
 	}
 	if len(client.Spec.Accounts) == 0 {
-		client.Spec.Accounts = []Account{
-			{
-				ID: defaultVar,
-			},
-		}
+		client.Spec.Accounts = []spec.Account{{ID: defaultVar}}
 	}
 
 	initLock := sync.Mutex{}

plugins/source/aws/client/organizations.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/cloudquery/cloudquery/plugins/source/aws/client/services"
+	"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
 	"github.com/thoas/go-funk"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -15,10 +16,10 @@ import (
 )
 
 // Parses org configuration and grabs the appropriate accounts
-func loadOrgAccounts(ctx context.Context, logger zerolog.Logger, awsPluginSpec *Spec) ([]Account, AssumeRoleAPIClient, error) {
+func loadOrgAccounts(ctx context.Context, logger zerolog.Logger, awsPluginSpec *spec.Spec) ([]spec.Account, AssumeRoleAPIClient, error) {
 	// If user doesn't specify any configs for admin account instantiate default values
 	if awsPluginSpec.Organization.AdminAccount == nil {
-		awsPluginSpec.Organization.AdminAccount = &Account{
+		awsPluginSpec.Organization.AdminAccount = &spec.Account{
 			AccountName:  "Default-Admin-Account",
 			LocalProfile: "",
 		}
@@ -46,7 +47,7 @@ func loadOrgAccounts(ctx context.Context, logger zerolog.Logger, awsPluginSpec *
 }
 
 // Load accounts from the appropriate endpoint as well as normalizing response
-func loadAccounts(ctx context.Context, awsPluginSpec *Spec, accountsApi services.OrganizationsClient, region string) ([]Account, error) {
+func loadAccounts(ctx context.Context, awsPluginSpec *spec.Spec, accountsApi services.OrganizationsClient, region string) ([]spec.Account, error) {
 	var rawAccounts []orgTypes.Account
 	var err error
 	if len(awsPluginSpec.Organization.OrganizationUnits) > 0 {
@@ -56,10 +57,10 @@ func loadAccounts(ctx context.Context, awsPluginSpec *Spec, accountsApi services
 	}
 
 	if err != nil {
-		return []Account{}, err
+		return []spec.Account{}, err
 	}
 	seen := map[string]struct{}{}
-	accounts := make([]Account, 0)
+	accounts := make([]spec.Account, 0)
 	for _, account := range rawAccounts {
 		// Only load Active accounts
 		if account.Status != orgTypes.AccountStatusActive || account.Id == nil {
@@ -83,21 +84,21 @@ func loadAccounts(ctx context.Context, awsPluginSpec *Spec, accountsApi services
 			roleArn.Partition = parsed.Partition
 		}
 
-		accounts = append(accounts, Account{
+		accounts = append(accounts, spec.Account{
 			ID:              *account.Id,
 			RoleARN:         roleArn.String(),
 			RoleSessionName: awsPluginSpec.Organization.ChildAccountRoleSessionName,
 			ExternalID:      awsPluginSpec.Organization.ChildAccountExternalID,
 			LocalProfile:    awsPluginSpec.Organization.AdminAccount.LocalProfile,
 			Regions:         awsPluginSpec.Organization.ChildAccountRegions,
-			source:          "org",
+			Source:          spec.AccountSourceOrg,
 		})
 	}
 	return accounts, err
 }
 
 // Get Accounts for specific Organizational Units
-func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient, awsOrg *AwsOrg, region string) ([]orgTypes.Account, error) {
+func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient, awsOrg *spec.Org, region string) ([]orgTypes.Account, error) {
 	q := awsOrg.OrganizationUnits
 	var ou string
 	var rawAccounts []orgTypes.Account
@@ -158,7 +159,7 @@ func getOUAccounts(ctx context.Context, accountsApi services.OrganizationsClient
 }
 
 // Get All accounts in a specific organization
-func getAllAccounts(ctx context.Context, accountsApi services.OrganizationsClient, org *AwsOrg, region string) ([]orgTypes.Account, error) {
+func getAllAccounts(ctx context.Context, accountsApi services.OrganizationsClient, org *spec.Org, region string) ([]orgTypes.Account, error) {
 	var rawAccounts []orgTypes.Account
 	accountsPaginator := organizations.NewListAccountsPaginator(accountsApi, &organizations.ListAccountsInput{})
 	for accountsPaginator.HasMorePages() {

plugins/source/aws/client/organizations_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/cloudquery/cloudquery/plugins/source/aws/client/mocks"
+	"github.com/cloudquery/cloudquery/plugins/source/aws/client/spec"
 	"github.com/golang/mock/gomock"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -111,99 +112,99 @@ func Test_loadAccounts(t *testing.T) {
 	ctx := context.Background()
 	tests := []struct {
 		name    string
-		spec    *Spec
+		spec    *spec.Spec
 		want    []string
 		wantErr error
 	}{
 		{
 			name: "all_accounts",
-			spec: &Spec{
-				Organization: &AwsOrg{
-					AdminAccount: &Account{},
+			spec: &spec.Spec{
+				Organization: &spec.Org{
+					AdminAccount: &spec.Account{},
 				},
 			},
 			want: []string{"id-child1-account", "id-child2-account", "id-parent1-account", "id-parent2-account", "id-top-level-account"},
 		},
 		{
 			name: "all_accounts_with_skip_member_accounts",
-			spec: &Spec{
-				Organization: &AwsOrg{
-					AdminAccount:       &Account{},
+			spec: &spec.Spec{
+				Organization: &spec.Org{
+					AdminAccount:       &spec.Account{},
 					SkipMemberAccounts: []string{"id-child2-account", "id-parent1-account", "id-parent2-account", "id-top-level-account"},
 				},
 			},
 			want: []string{"id-child1-account"},
 		},
 		{
 			name: "org_root",
-			spec: &Spec{
-				Organization: &AwsOrg{
+			spec: &spec.Spec{
+				Organization: &spec.Org{
 					OrganizationUnits: []string{"root"},
-					AdminAccount:      &Account{},
+					AdminAccount:      &spec.Account{},
 				},
 			},
 			want: []string{"id-top-level-account", "id-child1-account", "id-parent1-account", "id-child2-account", "id-parent2-account"},
 		},
 		{
 			name: "ou_parent1",
-			spec: &Spec{
-				Organization: &AwsOrg{
+			spec: &spec.Spec{
+				Organization: &spec.Org{
 					OrganizationUnits: []string{"ou-parent1"},
-					AdminAccount:      &Account{},
+					AdminAccount:      &spec.Account{},
 				},
 			},
 			want: []string{"id-parent1-account", "id-child1-account"},
 		},
 		{
 			name: "ou_parent1_and_parent2",
-			spec: &Spec{
-				Organization: &AwsOrg{
+			spec: &spec.Spec{
+				Organization: &spec.Org{
 					OrganizationUnits: []string{"ou-parent1", "ou-parent2"},
-					AdminAccount:      &Account{},
+					AdminAccount:      &spec.Account{},
 				},
 			},
 			want: []string{"id-parent1-account", "id-child1-account", "id-parent2-account", "id-child2-account"},
 		},
 		{
 			name: "ou_parent1_skip_child1",
-			spec: &Spec{
-				Organization: &AwsOrg{
+			spec: &spec.Spec{
+				Organization: &spec.Org{
 					OrganizationUnits:  []string{"ou-parent1"},
 					SkipMemberAccounts: []string{"id-child1-account"},
-					AdminAccount:       &Account{},
+					AdminAccount:       &spec.Account{},
 				},
 			},
 			want: []string{"id-parent1-account"},
 		},
 		{
 			name: "ou_root_skip_parent1",
-			spec: &Spec{
-				Organization: &AwsOrg{
+			spec: &spec.Spec{
+				Organization: &spec.Org{
 					OrganizationUnits:       []string{"root"},
 					SkipOrganizationalUnits: []string{"ou-parent1"},
-					AdminAccount:            &Account{},
+					AdminAccount:            &spec.Account{},
 				},
 			},
 			want: []string{"id-top-level-account", "id-parent2-account", "id-child2-account"},
 		},
 		{
 			name: "ou_root_skip_parent1",
-			spec: &Spec{
-				Organization: &AwsOrg{
+			spec: &spec.Spec{
+				Organization: &spec.Org{
 					OrganizationUnits:       []string{"root"},
 					SkipOrganizationalUnits: []string{"ou-parent1"},
-					AdminAccount:            &Account{},
+					AdminAccount:            &spec.Account{},
 				},
 			},
 			want: []string{"id-top-level-account", "id-parent2-account", "id-child2-account"},
 		},
 		{
 			name: "ou_root_and_parent1",
-			spec: &Spec{
-				Organization: &AwsOrg{
+			spec: &spec.Spec{
+				Organization: &spec.Org{
 					OrganizationUnits:       []string{"root", "ou-parent1"},
 					SkipOrganizationalUnits: []string{},
-					AdminAccount:            &Account{},
+					AdminAccount:            &spec.Account{},
 				},
 			},
 			want: []string{"id-top-level-account", "id-parent1-account", "id-child1-account", "id-parent2-account", "id-child2-account"},