diff --git a/clients/githubrepo/client.go b/clients/githubrepo/client.go index d4706ae492e..c7fef13b6de 100644 --- a/clients/githubrepo/client.go +++ b/clients/githubrepo/client.go @@ -180,7 +180,7 @@ func (client *Client) ListStatuses(ref string) ([]clients.Status, error) { return client.statuses.listStatuses(ref) } -//ListProgrammingLanguages implements RepoClient.ListProgrammingLanguages. +// ListProgrammingLanguages implements RepoClient.ListProgrammingLanguages. func (client *Client) ListProgrammingLanguages() ([]clients.Language, error) { return client.languages.listProgrammingLanguages() } diff --git a/clients/localdir/client.go b/clients/localdir/client.go index 7b58ea8d6fd..e8887bf5d2b 100644 --- a/clients/localdir/client.go +++ b/clients/localdir/client.go @@ -219,7 +219,7 @@ func (client *localDirClient) Close() error { } // ListProgrammingLanguages implements RepoClient.ListProgrammingLanguages. -// TODO: add ListProgrammingLanguages support for local directories +// TODO: add ListProgrammingLanguages support for local directories. func (client *localDirClient) ListProgrammingLanguages() ([]clients.Language, error) { return nil, fmt.Errorf("ListProgrammingLanguages: %w", clients.ErrUnsupportedFeature) } diff --git a/cron/internal/config/config.go b/cron/internal/config/config.go index 30b13ee6f68..4f8ec90b8bf 100644 --- a/cron/internal/config/config.go +++ b/cron/internal/config/config.go @@ -47,6 +47,7 @@ const ( blacklistedChecks string = "SCORECARD_BLACKLISTED_CHECKS" bigqueryTable string = "SCORECARD_BIGQUERY_TABLE" resultDataBucketURL string = "SCORECARD_DATA_BUCKET_URL" + apiResultsBucketURL string = "SCORECARD_API_RESULTS_BUCKET_URL" // Raw results. rawBigqueryTable string = "RAW_SCORECARD_BIGQUERY_TABLE" rawResultDataBucketURL string = "RAW_SCORECARD_DATA_BUCKET_URL" @@ -78,6 +79,7 @@ type config struct { // Raw results. RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"` RawBigQueryTable string `yaml:"raw-bigquery-table"` + ExportResultsBucketURL string `yaml:"export-results-bucket-url"` } func getParsedConfigFromFile(byteValue []byte) (config, error) { @@ -120,7 +122,6 @@ func getIntConfigValue(envVar string, byteValue []byte, fieldName, configName st return 0, fmt.Errorf("error getting config value %s: %w", configName, err) } - // nolint: exhaustive switch value.Kind() { case reflect.String: //nolint:wrapcheck @@ -137,7 +138,7 @@ func getFloat64ConfigValue(envVar string, byteValue []byte, fieldName, configNam if err != nil { return 0, fmt.Errorf("error getting config value %s: %w", configName, err) } - // nolint: exhaustive + switch value.Kind() { case reflect.String: //nolint: wrapcheck, gomnd @@ -232,3 +233,9 @@ func GetBlacklistedChecks() ([]string, error) { func GetMetricExporter() (string, error) { return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter") } + +// GetBQExportResultsBucketURL returns the bucket URL for storing cron job results. +func GetBQExportResultsBucketURL() (string, error) { + return getStringConfigValue(apiResultsBucketURL, configYAML, + "ExportResultsBucketURL", "export-results-bucket-url") +} diff --git a/cron/internal/config/config.yaml b/cron/internal/config/config.yaml index 02f8d399cdf..205ee9477f1 100644 --- a/cron/internal/config/config.yaml +++ b/cron/internal/config/config.yaml @@ -29,3 +29,5 @@ result-data-bucket-url: gs://ossf-scorecard-data2 # Raw results. raw-result-data-bucket-url: gs://ossf-scorecard-rawdata raw-bigquery-table: scorecard-rawdata +# export-bucket +export-results-bucket-url: gs://ossf-scorecard-cron-releasetest-results diff --git a/cron/internal/config/config_test.go b/cron/internal/config/config_test.go index aa473f58bca..669eccd30f2 100644 --- a/cron/internal/config/config_test.go +++ b/cron/internal/config/config_test.go @@ -37,8 +37,9 @@ const ( prodShardSize int = 10 prodMetricExporter string = "stackdriver" // Raw results. - prodRawBucket = "gs://ossf-scorecard-rawdata" - prodRawBigQueryTable = "scorecard-rawdata" + prodRawBucket = "gs://ossf-scorecard-rawdata" + prodRawBigQueryTable = "scorecard-rawdata" + prodBigQueryExportsBucketURL = "gs://ossf-scorecard-cron-releasetest-results" ) func getByteValueFromFile(filename string) ([]byte, error) { @@ -74,6 +75,7 @@ func TestYAMLParsing(t *testing.T) { MetricExporter: prodMetricExporter, RawResultDataBucketURL: prodRawBucket, RawBigQueryTable: prodRawBigQueryTable, + ExportResultsBucketURL: prodBigQueryExportsBucketURL, }, }, @@ -344,3 +346,18 @@ func TestGetMetricExporter(t *testing.T) { } }) } + +//nolint:paralleltest // Since os.Setenv is used. +func TestGetBigQueryExportsBucketURL(t *testing.T) { + t.Run("GetBigQueryExportsBucketURL", func(t *testing.T) { + bigqueryExportsBucketURL := apiResultsBucketURL + os.Unsetenv(bigqueryExportsBucketURL) + bucket, err := GetBQExportResultsBucketURL() + if err != nil { + t.Errorf("failed to get production bucket URL from config: %v", err) + } + if bucket != prodBigQueryExportsBucketURL { + t.Errorf("test failed: expected - %s, got = %s", prodBigQueryExportsBucketURL, bucket) + } + }) +} diff --git a/cron/internal/controller/main.go b/cron/internal/controller/main.go index 7b5ff9d3d83..d80f2ef7425 100644 --- a/cron/internal/controller/main.go +++ b/cron/internal/controller/main.go @@ -133,7 +133,7 @@ func main() { ShardLoc: new(string), CommitSha: new(string), } - *metadata.NumShard = (shardNum + 1) + *metadata.NumShard = shardNum + 1 *metadata.ShardLoc = bucket + "/" + data.GetBlobFilename("", t) *metadata.CommitSha = version.GetVersionInfo().GitCommit metadataJSON, err := protojson.Marshal(&metadata) diff --git a/cron/internal/worker/main.go b/cron/internal/worker/main.go index e539a8c20a8..689bb9347d1 100644 --- a/cron/internal/worker/main.go +++ b/cron/internal/worker/main.go @@ -48,7 +48,7 @@ var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to tru // nolint: gocognit func processRequest(ctx context.Context, batchRequest *data.ScorecardBatchRequest, - blacklistedChecks []string, bucketURL, rawBucketURL string, + blacklistedChecks []string, bucketURL, rawBucketURL, exportBucketURL string, checkDocs docs.Doc, repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient, ciiClient clients.CIIBestPracticesClient, @@ -101,6 +101,7 @@ func processRequest(ctx context.Context, for _, check := range blacklistedChecks { delete(checksToRun, check) } + result, err := pkg.RunScorecards(ctx, repo, commitSHA, checksToRun, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient) if errors.Is(err, sce.ErrRepoUnreachable) { @@ -128,11 +129,41 @@ func processRequest(ctx context.Context, if err := format.AsJSON2(&result, true /*showDetails*/, log.InfoLevel, checkDocs, &buffer2); err != nil { return fmt.Errorf("error during result.AsJSON2: %w", err) } + // these are for exporting results to GCS for API consumption + var exportBuffer bytes.Buffer + var exportRawBuffer bytes.Buffer + + if err := format.AsJSON2(&result, true /*showDetails*/, log.InfoLevel, checkDocs, &exportBuffer); err != nil { + return fmt.Errorf("error during result.AsJSON2 for export: %w", err) + } + if err := format.AsRawJSON(&result, &exportRawBuffer); err != nil { + return fmt.Errorf("error during result.AsRawJSON for export: %w", err) + } + exportPath := fmt.Sprintf("%s/result.json", repo.URI()) + exportCommitSHAPath := fmt.Sprintf("%s/%s/result.json", repo.URI(), result.Repo.CommitSHA) + exportRawPath := fmt.Sprintf("%s/raw.json", repo.URI()) + exportRawCommitSHAPath := fmt.Sprintf("%s/%s/raw.json", repo.URI(), result.Repo.CommitSHA) // Raw result. if err := format.AsRawJSON(&result, &rawBuffer); err != nil { return fmt.Errorf("error during result.AsRawJSON: %w", err) } + + // These are results without the commit SHA which represents the latest commit. + if err := data.WriteToBlobStore(ctx, exportBucketURL, exportPath, exportBuffer.Bytes()); err != nil { + return fmt.Errorf("error during writing to exportBucketURL: %w", err) + } + // Export result based on commitSHA. + if err := data.WriteToBlobStore(ctx, exportBucketURL, exportCommitSHAPath, exportBuffer.Bytes()); err != nil { + return fmt.Errorf("error during exportBucketURL with commit SHA: %w", err) + } + // Export raw result. + if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawPath, exportRawBuffer.Bytes()); err != nil { + return fmt.Errorf("error during writing to exportBucketURL for raw results: %w", err) + } + if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawCommitSHAPath, exportRawBuffer.Bytes()); err != nil { + return fmt.Errorf("error during exportBucketURL for raw results with commit SHA: %w", err) + } } if err := data.WriteToBlobStore(ctx, bucketURL, filename, buffer2.Bytes()); err != nil { @@ -207,6 +238,11 @@ func main() { panic(err) } + exportBucketURL, err := config.GetBQExportResultsBucketURL() + if err != nil { + panic(err) + } + logger := log.NewLogger(log.InfoLevel) repoClient := githubrepo.CreateGithubRepoClient(ctx, logger) ciiClient := clients.BlobCIIBestPracticesClient(ciiDataBucketURL) @@ -242,7 +278,7 @@ func main() { break } if err := processRequest(ctx, req, blacklistedChecks, - bucketURL, rawBucketURL, checkDocs, + bucketURL, rawBucketURL, exportBucketURL, checkDocs, repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil { // TODO(log): Previously Warn. Consider logging an error here. logger.Info(fmt.Sprintf("error processing request: %v", err)) diff --git a/cron/k8s/worker.release.yaml b/cron/k8s/worker.release.yaml index a3b3c69de9e..4812411b86f 100644 --- a/cron/k8s/worker.release.yaml +++ b/cron/k8s/worker.release.yaml @@ -42,6 +42,8 @@ spec: value: "printer" - name: GITHUB_AUTH_SERVER value: "10.4.4.210:80" + - name: "SCORECARD_API_RESULTS_BUCKET_URL" + value: "gs://ossf-scorecard-cron-releasetest-results" resources: requests: memory: 5Gi