From 28e68ff39243b90df000593f1edc6dbd1c8ca9e9 Mon Sep 17 00:00:00 2001 From: naveensrinivasan <172697+naveensrinivasan@users.noreply.github.com> Date: Thu, 21 Jul 2022 11:52:57 -0500 Subject: [PATCH] Some cleanup and tweaks. --- cron/internal/config/config.go | 30 +++++++++++++------------- cron/internal/config/config.yaml | 2 +- cron/internal/config/config_test.go | 33 +++++++++++++++-------------- cron/internal/worker/main.go | 28 ++++++++++++++++++++---- cron/k8s/worker.release.yaml | 2 ++ 5 files changed, 59 insertions(+), 36 deletions(-) diff --git a/cron/internal/config/config.go b/cron/internal/config/config.go index 9bd4558d0512..5dbbcbff559e 100644 --- a/cron/internal/config/config.go +++ b/cron/internal/config/config.go @@ -34,20 +34,20 @@ const ( // ShardNumFilename is the name of the file that stores the number of shards. ShardNumFilename string = ".shard_num" // TransferStatusFilename file identifies if shard transfer to BigQuery is completed. - TransferStatusFilename string = ".transfer_complete" - projectID string = "SCORECARD_PROJECT_ID" - requestTopicURL string = "SCORECARD_REQUEST_TOPIC_URL" - requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL" - bigqueryDataset string = "SCORECARD_BIGQUERY_DATASET" - completionThreshold string = "SCORECARD_COMPLETION_THRESHOLD" - shardSize string = "SCORECARD_SHARD_SIZE" - webhookURL string = "SCORECARD_WEBHOOK_URL" - metricExporter string = "SCORECARD_METRIC_EXPORTER" - ciiDataBucketURL string = "SCORECARD_CII_DATA_BUCKET_URL" - blacklistedChecks string = "SCORECARD_BLACKLISTED_CHECKS" - bigqueryTable string = "SCORECARD_BIGQUERY_TABLE" - resultDataBucketURL string = "SCORECARD_DATA_BUCKET_URL" - bqExportResultsBucketURL string = "SCORECARD_BQ_EXPORT_RESULTS_BUCKET_URL" + TransferStatusFilename string = ".transfer_complete" + projectID string = "SCORECARD_PROJECT_ID" + requestTopicURL string = "SCORECARD_REQUEST_TOPIC_URL" + requestSubscriptionURL string = "SCORECARD_REQUEST_SUBSCRIPTION_URL" + bigqueryDataset string = "SCORECARD_BIGQUERY_DATASET" + completionThreshold string = "SCORECARD_COMPLETION_THRESHOLD" + shardSize string = "SCORECARD_SHARD_SIZE" + webhookURL string = "SCORECARD_WEBHOOK_URL" + metricExporter string = "SCORECARD_METRIC_EXPORTER" + ciiDataBucketURL string = "SCORECARD_CII_DATA_BUCKET_URL" + blacklistedChecks string = "SCORECARD_BLACKLISTED_CHECKS" + bigqueryTable string = "SCORECARD_BIGQUERY_TABLE" + resultDataBucketURL string = "SCORECARD_DATA_BUCKET_URL" + apiResultsBucketURL string = "SCORECARD_API_RESULTS_BUCKET_URL" // Raw results. rawBigqueryTable string = "RAW_SCORECARD_BIGQUERY_TABLE" rawResultDataBucketURL string = "RAW_SCORECARD_DATA_BUCKET_URL" @@ -236,6 +236,6 @@ func GetMetricExporter() (string, error) { // GetBQExportResultsBucketURL returns the bucket URL for storing cron job results. func GetBQExportResultsBucketURL() (string, error) { - return getStringConfigValue(bqExportResultsBucketURL, configYAML, + return getStringConfigValue(apiResultsBucketURL, configYAML, "BigQueryExportResultsBucketURL", "bq-export-results-bucket-url") } diff --git a/cron/internal/config/config.yaml b/cron/internal/config/config.yaml index 504697897986..4392e9fc16a6 100644 --- a/cron/internal/config/config.yaml +++ b/cron/internal/config/config.yaml @@ -30,4 +30,4 @@ result-data-bucket-url: gs://ossf-scorecard-data2 raw-result-data-bucket-url: gs://ossf-scorecard-rawdata raw-bigquery-table: scorecard-rawdata # export-bucket -bigquery-export-results-bucket-url: gs://ossf-scorecard-bq-export-results +bigquery-export-results-bucket-url: gs://ossf-scorecard-cron-results diff --git a/cron/internal/config/config_test.go b/cron/internal/config/config_test.go index a6dd7e2cebe0..a2cdf52e783f 100644 --- a/cron/internal/config/config_test.go +++ b/cron/internal/config/config_test.go @@ -39,7 +39,7 @@ const ( // Raw results. prodRawBucket = "gs://ossf-scorecard-rawdata" prodRawBigQueryTable = "scorecard-rawdata" - prodBigQueryExportsBucketURL = "gs://ossf-scorecard-bq-export-results" + prodBigQueryExportsBucketURL = "gs://ossf-scorecard-cron-results" ) func getByteValueFromFile(filename string) ([]byte, error) { @@ -61,20 +61,21 @@ func TestYAMLParsing(t *testing.T) { name: "validate", filename: "config.yaml", expectedConfig: config{ - ProjectID: prodProjectID, - ResultDataBucketURL: prodBucket, - RequestTopicURL: prodTopic, - RequestSubscriptionURL: prodSubscription, - BigQueryDataset: prodBigQueryDataset, - BigQueryTable: prodBigQueryTable, - CompletionThreshold: prodCompletionThreshold, - WebhookURL: prodWebhookURL, - CIIDataBucketURL: prodCIIDataBucket, - BlacklistedChecks: prodBlacklistedChecks, - ShardSize: prodShardSize, - MetricExporter: prodMetricExporter, - RawResultDataBucketURL: prodRawBucket, - RawBigQueryTable: prodRawBigQueryTable, + ProjectID: prodProjectID, + ResultDataBucketURL: prodBucket, + RequestTopicURL: prodTopic, + RequestSubscriptionURL: prodSubscription, + BigQueryDataset: prodBigQueryDataset, + BigQueryTable: prodBigQueryTable, + CompletionThreshold: prodCompletionThreshold, + WebhookURL: prodWebhookURL, + CIIDataBucketURL: prodCIIDataBucket, + BlacklistedChecks: prodBlacklistedChecks, + ShardSize: prodShardSize, + MetricExporter: prodMetricExporter, + RawResultDataBucketURL: prodRawBucket, + RawBigQueryTable: prodRawBigQueryTable, + BigQueryExportResultsBucketURL: prodBigQueryExportsBucketURL, }, }, @@ -349,7 +350,7 @@ func TestGetMetricExporter(t *testing.T) { //nolint:paralleltest // Since os.Setenv is used. func TestGetBigQueryExportsBucketURL(t *testing.T) { t.Run("GetBigQueryExportsBucketURL", func(t *testing.T) { - bigqueryExportsBucketURL := bqExportResultsBucketURL + bigqueryExportsBucketURL := apiResultsBucketURL os.Unsetenv(bigqueryExportsBucketURL) bucket, err := GetBQExportResultsBucketURL() if err != nil { diff --git a/cron/internal/worker/main.go b/cron/internal/worker/main.go index bcad520d739c..689bb9347d1c 100644 --- a/cron/internal/worker/main.go +++ b/cron/internal/worker/main.go @@ -129,20 +129,40 @@ func processRequest(ctx context.Context, if err := format.AsJSON2(&result, true /*showDetails*/, log.InfoLevel, checkDocs, &buffer2); err != nil { return fmt.Errorf("error during result.AsJSON2: %w", err) } + // these are for exporting results to GCS for API consumption + var exportBuffer bytes.Buffer + var exportRawBuffer bytes.Buffer + + if err := format.AsJSON2(&result, true /*showDetails*/, log.InfoLevel, checkDocs, &exportBuffer); err != nil { + return fmt.Errorf("error during result.AsJSON2 for export: %w", err) + } + if err := format.AsRawJSON(&result, &exportRawBuffer); err != nil { + return fmt.Errorf("error during result.AsRawJSON for export: %w", err) + } exportPath := fmt.Sprintf("%s/result.json", repo.URI()) exportCommitSHAPath := fmt.Sprintf("%s/%s/result.json", repo.URI(), result.Repo.CommitSHA) + exportRawPath := fmt.Sprintf("%s/raw.json", repo.URI()) + exportRawCommitSHAPath := fmt.Sprintf("%s/%s/raw.json", repo.URI(), result.Repo.CommitSHA) // Raw result. if err := format.AsRawJSON(&result, &rawBuffer); err != nil { return fmt.Errorf("error during result.AsRawJSON: %w", err) } - if err := data.WriteToBlobStore(ctx, exportBucketURL, exportPath, buffer2.Bytes()); err != nil { - return fmt.Errorf("error during WriteToBlobStore2: %w", err) + // These are results without the commit SHA which represents the latest commit. + if err := data.WriteToBlobStore(ctx, exportBucketURL, exportPath, exportBuffer.Bytes()); err != nil { + return fmt.Errorf("error during writing to exportBucketURL: %w", err) } // Export result based on commitSHA. - if err := data.WriteToBlobStore(ctx, exportBucketURL, exportCommitSHAPath, buffer2.Bytes()); err != nil { - return fmt.Errorf("error during WriteToBlobStore2: %w", err) + if err := data.WriteToBlobStore(ctx, exportBucketURL, exportCommitSHAPath, exportBuffer.Bytes()); err != nil { + return fmt.Errorf("error during exportBucketURL with commit SHA: %w", err) + } + // Export raw result. + if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawPath, exportRawBuffer.Bytes()); err != nil { + return fmt.Errorf("error during writing to exportBucketURL for raw results: %w", err) + } + if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawCommitSHAPath, exportRawBuffer.Bytes()); err != nil { + return fmt.Errorf("error during exportBucketURL for raw results with commit SHA: %w", err) } } diff --git a/cron/k8s/worker.release.yaml b/cron/k8s/worker.release.yaml index a3b3c69de9ef..f7c8b29d5ffb 100644 --- a/cron/k8s/worker.release.yaml +++ b/cron/k8s/worker.release.yaml @@ -42,6 +42,8 @@ spec: value: "printer" - name: GITHUB_AUTH_SERVER value: "10.4.4.210:80" + - name: "SCORECARD_API_RESULTS_BUCKET_URL" + value: "gs://ossf-scorecard-cron-results" resources: requests: memory: 5Gi