✨ Export Scorecards results for API (#2081)

* 🌱 Export Scorecards results for API - Exporting the Scorecard results for the scorecard API. - The code exports as result.json without the commit SHA and also with the commit SHA. * Some cleanup and tweaks. * Some cleanup and tweaks.
ossf · Jul 23, 2022 · e23ee84 · e23ee84
1 parent 30e3f64
commit e23ee84
Show file tree

Hide file tree

Showing 8 changed files with 73 additions and 9 deletions.
diff --git a/clients/githubrepo/client.go b/clients/githubrepo/client.go
@@ -180,7 +180,7 @@ func (client *Client) ListStatuses(ref string) ([]clients.Status, error) {
 	return client.statuses.listStatuses(ref)
 }
 
-//ListProgrammingLanguages implements RepoClient.ListProgrammingLanguages.
+// ListProgrammingLanguages implements RepoClient.ListProgrammingLanguages.
 func (client *Client) ListProgrammingLanguages() ([]clients.Language, error) {
 	return client.languages.listProgrammingLanguages()
 }

diff --git a/clients/localdir/client.go b/clients/localdir/client.go
@@ -219,7 +219,7 @@ func (client *localDirClient) Close() error {
 }
 
 // ListProgrammingLanguages implements RepoClient.ListProgrammingLanguages.
-// TODO: add ListProgrammingLanguages support for local directories
+// TODO: add ListProgrammingLanguages support for local directories.
 func (client *localDirClient) ListProgrammingLanguages() ([]clients.Language, error) {
 	return nil, fmt.Errorf("ListProgrammingLanguages: %w", clients.ErrUnsupportedFeature)
 }

diff --git a/cron/internal/config/config.go b/cron/internal/config/config.go
@@ -47,6 +47,7 @@ const (
 	blacklistedChecks      string = "SCORECARD_BLACKLISTED_CHECKS"
 	bigqueryTable          string = "SCORECARD_BIGQUERY_TABLE"
 	resultDataBucketURL    string = "SCORECARD_DATA_BUCKET_URL"
+	apiResultsBucketURL    string = "SCORECARD_API_RESULTS_BUCKET_URL"
 	// Raw results.
 	rawBigqueryTable       string = "RAW_SCORECARD_BIGQUERY_TABLE"
 	rawResultDataBucketURL string = "RAW_SCORECARD_DATA_BUCKET_URL"
@@ -78,6 +79,7 @@ type config struct {
 	// Raw results.
 	RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"`
 	RawBigQueryTable       string `yaml:"raw-bigquery-table"`
+	ExportResultsBucketURL string `yaml:"export-results-bucket-url"`
 }
 
 func getParsedConfigFromFile(byteValue []byte) (config, error) {
@@ -120,7 +122,6 @@ func getIntConfigValue(envVar string, byteValue []byte, fieldName, configName st
 		return 0, fmt.Errorf("error getting config value %s: %w", configName, err)
 	}
 
-	// nolint: exhaustive
 	switch value.Kind() {
 	case reflect.String:
 		//nolint:wrapcheck
@@ -137,7 +138,7 @@ func getFloat64ConfigValue(envVar string, byteValue []byte, fieldName, configNam
 	if err != nil {
 		return 0, fmt.Errorf("error getting config value %s: %w", configName, err)
 	}
-	// nolint: exhaustive
+
 	switch value.Kind() {
 	case reflect.String:
 		//nolint: wrapcheck, gomnd
@@ -232,3 +233,9 @@ func GetBlacklistedChecks() ([]string, error) {
 func GetMetricExporter() (string, error) {
 	return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter")
 }
+
+// GetBQExportResultsBucketURL returns the bucket URL for storing cron job results.
+func GetBQExportResultsBucketURL() (string, error) {
+	return getStringConfigValue(apiResultsBucketURL, configYAML,
+		"ExportResultsBucketURL", "export-results-bucket-url")
+}
diff --git a/cron/internal/config/config.yaml b/cron/internal/config/config.yaml
@@ -29,3 +29,5 @@ result-data-bucket-url: gs://ossf-scorecard-data2
 # Raw results.
 raw-result-data-bucket-url: gs://ossf-scorecard-rawdata
 raw-bigquery-table: scorecard-rawdata
+# export-bucket
+export-results-bucket-url: gs://ossf-scorecard-cron-releasetest-results
diff --git a/cron/internal/config/config_test.go b/cron/internal/config/config_test.go
@@ -37,8 +37,9 @@ const (
 	prodShardSize           int    = 10
 	prodMetricExporter      string = "stackdriver"
 	// Raw results.
-	prodRawBucket        = "gs://ossf-scorecard-rawdata"
-	prodRawBigQueryTable = "scorecard-rawdata"
+	prodRawBucket                = "gs://ossf-scorecard-rawdata"
+	prodRawBigQueryTable         = "scorecard-rawdata"
+	prodBigQueryExportsBucketURL = "gs://ossf-scorecard-cron-releasetest-results"
 )
 
 func getByteValueFromFile(filename string) ([]byte, error) {
@@ -74,6 +75,7 @@ func TestYAMLParsing(t *testing.T) {
 				MetricExporter:         prodMetricExporter,
 				RawResultDataBucketURL: prodRawBucket,
 				RawBigQueryTable:       prodRawBigQueryTable,
+				ExportResultsBucketURL: prodBigQueryExportsBucketURL,
 			},
 		},
 
@@ -344,3 +346,18 @@ func TestGetMetricExporter(t *testing.T) {
 		}
 	})
 }
+
+//nolint:paralleltest // Since os.Setenv is used.
+func TestGetBigQueryExportsBucketURL(t *testing.T) {
+	t.Run("GetBigQueryExportsBucketURL", func(t *testing.T) {
+		bigqueryExportsBucketURL := apiResultsBucketURL
+		os.Unsetenv(bigqueryExportsBucketURL)
+		bucket, err := GetBQExportResultsBucketURL()
+		if err != nil {
+			t.Errorf("failed to get production bucket URL from config: %v", err)
+		}
+		if bucket != prodBigQueryExportsBucketURL {
+			t.Errorf("test failed: expected - %s, got = %s", prodBigQueryExportsBucketURL, bucket)
+		}
+	})
+}
diff --git a/cron/internal/controller/main.go b/cron/internal/controller/main.go
@@ -133,7 +133,7 @@ func main() {
 		ShardLoc:  new(string),
 		CommitSha: new(string),
 	}
-	*metadata.NumShard = (shardNum + 1)
+	*metadata.NumShard = shardNum + 1
 	*metadata.ShardLoc = bucket + "/" + data.GetBlobFilename("", t)
 	*metadata.CommitSha = version.GetVersionInfo().GitCommit
 	metadataJSON, err := protojson.Marshal(&metadata)

diff --git a/cron/internal/worker/main.go b/cron/internal/worker/main.go
@@ -48,7 +48,7 @@ var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to tru
 // nolint: gocognit
 func processRequest(ctx context.Context,
 	batchRequest *data.ScorecardBatchRequest,
-	blacklistedChecks []string, bucketURL, rawBucketURL string,
+	blacklistedChecks []string, bucketURL, rawBucketURL, exportBucketURL string,
 	checkDocs docs.Doc,
 	repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient,
 	ciiClient clients.CIIBestPracticesClient,
@@ -101,6 +101,7 @@ func processRequest(ctx context.Context,
 		for _, check := range blacklistedChecks {
 			delete(checksToRun, check)
 		}
+
 		result, err := pkg.RunScorecards(ctx, repo, commitSHA, checksToRun,
 			repoClient, ossFuzzRepoClient, ciiClient, vulnsClient)
 		if errors.Is(err, sce.ErrRepoUnreachable) {
@@ -128,11 +129,41 @@ func processRequest(ctx context.Context,
 		if err := format.AsJSON2(&result, true /*showDetails*/, log.InfoLevel, checkDocs, &buffer2); err != nil {
 			return fmt.Errorf("error during result.AsJSON2: %w", err)
 		}
+		// these are for exporting results to GCS for API consumption
+		var exportBuffer bytes.Buffer
+		var exportRawBuffer bytes.Buffer
+
+		if err := format.AsJSON2(&result, true /*showDetails*/, log.InfoLevel, checkDocs, &exportBuffer); err != nil {
+			return fmt.Errorf("error during result.AsJSON2 for export: %w", err)
+		}
+		if err := format.AsRawJSON(&result, &exportRawBuffer); err != nil {
+			return fmt.Errorf("error during result.AsRawJSON for export: %w", err)
+		}
+		exportPath := fmt.Sprintf("%s/result.json", repo.URI())
+		exportCommitSHAPath := fmt.Sprintf("%s/%s/result.json", repo.URI(), result.Repo.CommitSHA)
+		exportRawPath := fmt.Sprintf("%s/raw.json", repo.URI())
+		exportRawCommitSHAPath := fmt.Sprintf("%s/%s/raw.json", repo.URI(), result.Repo.CommitSHA)
 
 		// Raw result.
 		if err := format.AsRawJSON(&result, &rawBuffer); err != nil {
 			return fmt.Errorf("error during result.AsRawJSON: %w", err)
 		}
+
+		// These are results without the commit SHA which represents the latest commit.
+		if err := data.WriteToBlobStore(ctx, exportBucketURL, exportPath, exportBuffer.Bytes()); err != nil {
+			return fmt.Errorf("error during writing to exportBucketURL: %w", err)
+		}
+		// Export result based on commitSHA.
+		if err := data.WriteToBlobStore(ctx, exportBucketURL, exportCommitSHAPath, exportBuffer.Bytes()); err != nil {
+			return fmt.Errorf("error during exportBucketURL with commit SHA: %w", err)
+		}
+		// Export raw result.
+		if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawPath, exportRawBuffer.Bytes()); err != nil {
+			return fmt.Errorf("error during writing to exportBucketURL for raw results: %w", err)
+		}
+		if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawCommitSHAPath, exportRawBuffer.Bytes()); err != nil {
+			return fmt.Errorf("error during exportBucketURL for raw results with commit SHA: %w", err)
+		}
 	}
 
 	if err := data.WriteToBlobStore(ctx, bucketURL, filename, buffer2.Bytes()); err != nil {
@@ -207,6 +238,11 @@ func main() {
 		panic(err)
 	}
 
+	exportBucketURL, err := config.GetBQExportResultsBucketURL()
+	if err != nil {
+		panic(err)
+	}
+
 	logger := log.NewLogger(log.InfoLevel)
 	repoClient := githubrepo.CreateGithubRepoClient(ctx, logger)
 	ciiClient := clients.BlobCIIBestPracticesClient(ciiDataBucketURL)
@@ -242,7 +278,7 @@ func main() {
 			break
 		}
 		if err := processRequest(ctx, req, blacklistedChecks,
-			bucketURL, rawBucketURL, checkDocs,
+			bucketURL, rawBucketURL, exportBucketURL, checkDocs,
 			repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil {
 			// TODO(log): Previously Warn. Consider logging an error here.
 			logger.Info(fmt.Sprintf("error processing request: %v", err))

diff --git a/cron/k8s/worker.release.yaml b/cron/k8s/worker.release.yaml
@@ -42,6 +42,8 @@ spec:
           value: "printer"
         - name: GITHUB_AUTH_SERVER
           value: "10.4.4.210:80"
+        - name: "SCORECARD_API_RESULTS_BUCKET_URL"
+          value: "gs://ossf-scorecard-cron-releasetest-results"
         resources:
           requests:
             memory: 5Gi