diff --git a/dependencydiff/dependencydiff.go b/dependencydiff/dependencydiff.go new file mode 100644 index 00000000000..21459d8ec50 --- /dev/null +++ b/dependencydiff/dependencydiff.go @@ -0,0 +1,160 @@ +// Copyright 2022 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dependencydiff + +import ( + "context" + "fmt" + "path" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/checks" + "github.com/ossf/scorecard/v4/clients" + "github.com/ossf/scorecard/v4/clients/githubrepo" + sce "github.com/ossf/scorecard/v4/errors" + "github.com/ossf/scorecard/v4/log" + "github.com/ossf/scorecard/v4/pkg" + "github.com/ossf/scorecard/v4/policy" +) + +// Depdiff is the exported name for dependency-diff. +const Depdiff = "Dependency-diff" + +type dependencydiffContext struct { + logger *log.Logger + ownerName, repoName, baseSHA, headSHA string + ctx context.Context + ghRepo clients.Repo + ghRepoClient clients.RepoClient + ossFuzzClient clients.RepoClient + vulnsClient clients.VulnerabilitiesClient + ciiClient clients.CIIBestPracticesClient + changeTypesToCheck map[pkg.ChangeType]bool + checkNamesToRun []string + dependencydiffs []dependency + results []pkg.DependencyCheckResult +} + +// GetDependencyDiffResults gets dependency changes between two given code commits BASE and HEAD +// along with the Scorecard check results of the dependencies, and returns a slice of DependencyCheckResult. +// TO use this API, an access token must be set following https://github.com/ossf/scorecard#authentication. +func GetDependencyDiffResults( + ctx context.Context, ownerName, repoName, baseSHA, headSHA string, scorecardChecksNames []string, + changeTypesToCheck map[pkg.ChangeType]bool) ([]pkg.DependencyCheckResult, error) { + // Fetch the raw dependency diffs. + dCtx := dependencydiffContext{ + logger: log.NewLogger(log.InfoLevel), + ownerName: ownerName, + repoName: repoName, + baseSHA: baseSHA, + headSHA: headSHA, + ctx: ctx, + changeTypesToCheck: changeTypesToCheck, + checkNamesToRun: scorecardChecksNames, + } + err := fetchRawDependencyDiffData(&dCtx) + if err != nil { + return nil, fmt.Errorf("error in fetchRawDependencyDiffData: %w", err) + } + + // Initialize the repo and client(s) corresponding to the checks to run. + err = initRepoAndClientByChecks(&dCtx) + if err != nil { + return nil, fmt.Errorf("error in initRepoAndClientByChecks: %w", err) + } + err = getScorecardCheckResults(&dCtx) + if err != nil { + return nil, fmt.Errorf("error getting scorecard check results: %w", err) + } + return dCtx.results, nil +} + +func initRepoAndClientByChecks(dCtx *dependencydiffContext) error { + repo, repoClient, ossFuzzClient, ciiClient, vulnsClient, err := checker.GetClients( + dCtx.ctx, path.Join(dCtx.ownerName, dCtx.repoName), "", dCtx.logger, + ) + if err != nil { + return fmt.Errorf("error creating the github repo: %w", err) + } + // If the caller doesn't specify the checks to run, run all checks and return all the clients. + if dCtx.checkNamesToRun == nil || len(dCtx.checkNamesToRun) == 0 { + dCtx.ghRepo, dCtx.ghRepoClient, dCtx.ossFuzzClient, dCtx.ciiClient, dCtx.vulnsClient = + repo, repoClient, ossFuzzClient, ciiClient, vulnsClient + return nil + } + dCtx.ghRepo = repo + dCtx.ghRepoClient = githubrepo.CreateGithubRepoClient(dCtx.ctx, dCtx.logger) + for _, cn := range dCtx.checkNamesToRun { + switch cn { + case checks.CheckFuzzing: + dCtx.ossFuzzClient = ossFuzzClient + case checks.CheckCIIBestPractices: + dCtx.ciiClient = ciiClient + case checks.CheckVulnerabilities: + dCtx.vulnsClient = vulnsClient + } + } + return nil +} + +func getScorecardCheckResults(dCtx *dependencydiffContext) error { + // Initialize the checks to run from the caller's input. + checksToRun, err := policy.GetEnabled(nil, dCtx.checkNamesToRun, nil) + if err != nil { + return fmt.Errorf("error init scorecard checks: %w", err) + } + for _, d := range dCtx.dependencydiffs { + depCheckResult := pkg.DependencyCheckResult{ + PackageURL: d.PackageURL, + SourceRepository: d.SourceRepository, + ChangeType: d.ChangeType, + ManifestPath: d.ManifestPath, + Ecosystem: d.Ecosystem, + Version: d.Version, + Name: d.Name, + } + // For now we skip those without source repo urls. + // TODO (#2063): use the BigQuery dataset to supplement null source repo URLs to fetch the Scorecard results for them. + if d.SourceRepository != nil && *d.SourceRepository != "" { + if d.ChangeType != nil && (dCtx.changeTypesToCheck[*d.ChangeType] || dCtx.changeTypesToCheck == nil) { + // Run scorecard on those types of dependencies that the caller would like to check. + // If the input map changeTypesToCheck is empty, by default, we run checks for all valid types. + // TODO (#2064): use the Scorecare REST API to retrieve the Scorecard result statelessly. + scorecardResult, err := pkg.RunScorecards( + dCtx.ctx, + dCtx.ghRepo, + // TODO (#2065): In future versions, ideally, this should be + // the commitSHA corresponding to d.Version instead of HEAD. + clients.HeadSHA, + checksToRun, + dCtx.ghRepoClient, + dCtx.ossFuzzClient, + dCtx.ciiClient, + dCtx.vulnsClient, + ) + // If the run fails, we leave the current dependency scorecard result empty and record the error + // rather than letting the entire API return nil since we still expect results for other dependencies. + if err != nil { + depCheckResult.ScorecardResultsWithError.Error = sce.WithMessage(sce.ErrScorecardInternal, + fmt.Sprintf("error running the scorecard checks: %v", err)) + } else { // Otherwise, we record the scorecard check results for this dependency. + depCheckResult.ScorecardResultsWithError.ScorecardResults = &scorecardResult + } + } + } + dCtx.results = append(dCtx.results, depCheckResult) + } + return nil +} diff --git a/dependencydiff/dependencydiff_test.go b/dependencydiff/dependencydiff_test.go new file mode 100644 index 00000000000..c0447519d45 --- /dev/null +++ b/dependencydiff/dependencydiff_test.go @@ -0,0 +1,166 @@ +// Copyright 2022 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dependencydiff + +import ( + "context" + "path" + "testing" + + "github.com/ossf/scorecard/v4/clients" + "github.com/ossf/scorecard/v4/log" +) + +// Test_fetchRawDependencyDiffData is a test function for fetchRawDependencyDiffData. +func Test_fetchRawDependencyDiffData(t *testing.T) { + t.Parallel() + //nolint + tests := []struct { + name string + dCtx dependencydiffContext + wantEmpty bool + wantErr bool + }{ + { + name: "error response", + dCtx: dependencydiffContext{ + logger: log.NewLogger(log.InfoLevel), + ctx: context.Background(), + ownerName: "no_such_owner", + repoName: "repo_not_exist", + baseSHA: "base", + headSHA: clients.HeadSHA, + }, + wantEmpty: true, + wantErr: true, + }, + // Considering of the token usage, normal responses are tested in the e2e test. + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := fetchRawDependencyDiffData(&tt.dCtx) + if (err != nil) != tt.wantErr { + t.Errorf("fetchRawDependencyDiffData() error = {%v}, want error: %v", err, tt.wantErr) + return + } + lenResults := len(tt.dCtx.dependencydiffs) + if (lenResults == 0) != tt.wantEmpty { + t.Errorf("want empty results: %v, got len of results:%d", tt.wantEmpty, lenResults) + return + } + + }) + } +} + +func Test_initRepoAndClientByChecks(t *testing.T) { + t.Parallel() + //nolint + tests := []struct { + name string + dCtx dependencydiffContext + wantGhRepo, wantRepoClient, wantFuzzClient bool + wantVulnClient, wantCIIClient bool + wantErr bool + }{ + { + name: "error creating repo", + dCtx: dependencydiffContext{ + logger: log.NewLogger(log.InfoLevel), + ctx: context.Background(), + ownerName: path.Join("host_not_exist.com", "owner_not_exist"), + repoName: "repo_not_exist", + checkNamesToRun: []string{}, + }, + wantGhRepo: false, + wantRepoClient: false, + wantFuzzClient: false, + wantVulnClient: false, + wantCIIClient: false, + wantErr: true, + }, + // Same as the above, putting the normal responses to the e2e test. + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := initRepoAndClientByChecks(&tt.dCtx) + if (err != nil) != tt.wantErr { + t.Errorf("initRepoAndClientByChecks() error = {%v}, want error: %v", err, tt.wantErr) + return + } + if (tt.dCtx.ghRepo != nil) != tt.wantGhRepo { + t.Errorf("init repo error, wantGhRepo: %v, got %v", tt.wantGhRepo, tt.dCtx.ghRepo) + return + } + if (tt.dCtx.ghRepoClient != nil) != tt.wantRepoClient { + t.Errorf("init repo error, wantRepoClient: %v, got %v", tt.wantRepoClient, tt.dCtx.ghRepoClient) + return + } + if (tt.dCtx.ossFuzzClient != nil) != tt.wantFuzzClient { + t.Errorf("init repo error, wantFuzzClient: %v, got %v", tt.wantFuzzClient, tt.dCtx.ossFuzzClient) + return + } + if (tt.dCtx.vulnsClient != nil) != tt.wantVulnClient { + t.Errorf("init repo error, wantVulnClient: %v, got %v", tt.wantVulnClient, tt.dCtx.vulnsClient) + return + } + if (tt.dCtx.ciiClient != nil) != tt.wantCIIClient { + t.Errorf("init repo error, wantCIIClient: %v, got %v", tt.wantCIIClient, tt.dCtx.ciiClient) + return + } + }) + } +} + +func Test_getScorecardCheckResults(t *testing.T) { + t.Parallel() + //nolint + tests := []struct { + name string + dCtx dependencydiffContext + wantErr bool + }{ + { + name: "empty response", + dCtx: dependencydiffContext{ + ctx: context.Background(), + logger: log.NewLogger(log.InfoLevel), + ownerName: "owner_not_exist", + repoName: "repo_not_exist", + }, + wantErr: false, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := initRepoAndClientByChecks(&tt.dCtx) + if err != nil { + t.Errorf("init repo and client error") + return + } + err = getScorecardCheckResults(&tt.dCtx) + if (err != nil) != tt.wantErr { + t.Errorf("getScorecardCheckResults() error = {%v}, want error: %v", err, tt.wantErr) + return + } + }) + } +} diff --git a/dependencydiff/raw_dependencies.go b/dependencydiff/raw_dependencies.go new file mode 100644 index 00000000000..519e560217c --- /dev/null +++ b/dependencydiff/raw_dependencies.go @@ -0,0 +1,72 @@ +// Copyright 2022 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dependencydiff + +import ( + "fmt" + "net/http" + "path" + + "github.com/google/go-github/v38/github" + + "github.com/ossf/scorecard/v4/clients/githubrepo/roundtripper" + "github.com/ossf/scorecard/v4/pkg" +) + +// dependency is a raw dependency fetched from the GitHub Dependency Review API. +// Fields of a dependnecy correspondings to those of pkg.DependencyCheckResult. +type dependency struct { + // Package URL is a short link for a package. + PackageURL *string `json:"package_url"` + + // SourceRepository is the source repository URL of the dependency. + SourceRepository *string `json:"source_repository_url"` + + // ChangeType indicates whether the dependency is added, updated, or removed. + ChangeType *pkg.ChangeType `json:"change_type"` + + // ManifestPath is the path of the manifest file of the dependency, such as go.mod for Go. + ManifestPath *string `json:"manifest"` + + // Ecosystem is the name of the package management system, such as NPM, GO, PYPI. + Ecosystem *string `json:"ecosystem"` + + // Version is the package version of the dependency. + Version *string `json:"version"` + + // Name is the name of the dependency. + Name string `json:"name"` +} + +// fetchRawDependencyDiffData fetches the dependency-diffs between the two code commits +// using the GitHub Dependency Review API, and returns a slice of DependencyCheckResult. +func fetchRawDependencyDiffData(dCtx *dependencydiffContext) error { + ghrt := roundtripper.NewTransport(dCtx.ctx, dCtx.logger) + ghClient := github.NewClient(&http.Client{Transport: ghrt}) + req, err := ghClient.NewRequest( + "GET", + path.Join("repos", dCtx.ownerName, dCtx.repoName, + "dependency-graph", "compare", dCtx.baseSHA+"..."+dCtx.headSHA), + nil, + ) + if err != nil { + return fmt.Errorf("request for dependency-diff failed with %w", err) + } + _, err = ghClient.Do(dCtx.ctx, req, &dCtx.dependencydiffs) + if err != nil { + return fmt.Errorf("error parsing the dependency-diff reponse: %w", err) + } + return nil +} diff --git a/e2e/dependencydiff_test.go b/e2e/dependencydiff_test.go new file mode 100644 index 00000000000..d0246293537 --- /dev/null +++ b/e2e/dependencydiff_test.go @@ -0,0 +1,94 @@ +// Copyright 2021 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/ossf/scorecard/v4/checks" + "github.com/ossf/scorecard/v4/dependencydiff" + "github.com/ossf/scorecard/v4/pkg" +) + +const ( + OWNER = "ossf-tests" + REPO = "scorecard-depdiff" + BASE = "fd2a82b3b735fffbc2d782ed5f50301b879ecc51" + HEAD = "1989568f93e484f6a86f8b276b170e3d6962ce12" +) + +var _ = Describe("E2E TEST:"+dependencydiff.Depdiff, func() { + Context("E2E TEST:Validating use of the dependency-diff API", func() { + It("Should return a slice of dependency-diff checking results", func() { + ctx := context.Background() + ownerName, repoName := OWNER, REPO + baseSHA, headSHA := BASE, HEAD + scorecardChecksNames := []string{ + checks.CheckBranchProtection, + } + changeTypesToCheck := map[pkg.ChangeType]bool{ + pkg.Removed: true, // Only checking those removed ones will make this test faster. + } + results, err := dependencydiff.GetDependencyDiffResults( + ctx, + ownerName, repoName, baseSHA, headSHA, + scorecardChecksNames, + changeTypesToCheck, + ) + Expect(err).Should(BeNil()) + Expect(len(results) > 0).Should(BeTrue()) + }) + It("Should return a valid empty result", func() { + ctx := context.Background() + ownerName, repoName := OWNER, REPO + baseSHA, headSHA := BASE, BASE + + scorecardChecksNames := []string{ + checks.CheckBranchProtection, + } + changeTypesToCheck := map[pkg.ChangeType]bool{ + pkg.Removed: true, + } + results, err := dependencydiff.GetDependencyDiffResults( + ctx, + ownerName, repoName, baseSHA, headSHA, + scorecardChecksNames, + changeTypesToCheck, + ) + Expect(err).Should(BeNil()) + Expect(len(results) == 0).Should(BeTrue()) + }) + It("Should initialize clients corresponding to the checks to run and do not crash", func() { + ctx := context.Background() + ownerName, repoName := OWNER, REPO + baseSHA, headSHA := BASE, HEAD + + scorecardChecksNames := []string{} + changeTypesToCheck := map[pkg.ChangeType]bool{ + pkg.Removed: true, + } + _, err := dependencydiff.GetDependencyDiffResults( + ctx, + ownerName, repoName, baseSHA, headSHA, + scorecardChecksNames, + changeTypesToCheck, + ) + Expect(err).Should(BeNil()) + }) + }) +}) diff --git a/pkg/check-depdiff/dependencies.go b/pkg/dependencydiff_result.go similarity index 62% rename from pkg/check-depdiff/dependencies.go rename to pkg/dependencydiff_result.go index bf386d592c8..5208d9812ce 100644 --- a/pkg/check-depdiff/dependencies.go +++ b/pkg/dependencydiff_result.go @@ -12,17 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package depdiff +package pkg -import "github.com/ossf/scorecard/v4/pkg" +import ( + "encoding/json" + "fmt" + "io" + + sce "github.com/ossf/scorecard/v4/errors" +) // ChangeType is the change type (added, updated, removed) of a dependency. type ChangeType string const ( - // Added suggests the dependency is a new one. + // Added suggests the dependency is a newly added one. Added ChangeType = "added" - // Updated suggests the dependency is bumped from an old version. + // Updated suggests the dependency is updated from an old version. Updated ChangeType = "updated" // Removed suggests the dependency is removed. Removed ChangeType = "removed" @@ -38,18 +44,27 @@ func (ct *ChangeType) IsValid() bool { } } +// ScorecardResultsWithError is used for the dependency-diff module to record scorecard results and their errors. +type ScorecardResultsWithError struct { + // ScorecardResults is the scorecard result for the dependency repo. + ScorecardResults *ScorecardResult `json:"scorecardResults"` + + // Error is an error returned when running the scorecard checks. A nil Error indicates the run succeeded. + Error error `json:"scorecardRunTimeError"` +} + // DependencyCheckResult is the dependency structure used in the returned results. type DependencyCheckResult struct { + // ChangeType indicates whether the dependency is added, updated, or removed. + ChangeType *ChangeType `json:"changeType"` + // Package URL is a short link for a package. PackageURL *string `json:"packageUrl"` // SourceRepository is the source repository URL of the dependency. SourceRepository *string `json:"sourceRepository"` - // ChangeType indicates whether the dependency is added, updated, or removed. - ChangeType *ChangeType `json:"changeType"` - - // ManifestPath is the name of the manifest file of the dependency, such as go.mod for Go. + // ManifestPath is the path of the manifest file of the dependency, such as go.mod for Go. ManifestPath *string `json:"manifestPath"` // Ecosystem is the name of the package management system, such as NPM, GO, PYPI. @@ -58,9 +73,17 @@ type DependencyCheckResult struct { // Version is the package version of the dependency. Version *string `json:"version"` - // ScorecardResults is the scorecard result for the dependency repo. - ScorecardResults *pkg.ScorecardResult `json:"scorecardResults"` + // ScorecardResultsWithError is the scorecard checking results of the dependency. + ScorecardResultsWithError ScorecardResultsWithError `json:"scorecardResultsWithError"` // Name is the name of the dependency. Name string `json:"name"` } + +// AsJSON for DependencyCheckResult exports the DependencyCheckResult as a JSON object. +func (dr *DependencyCheckResult) AsJSON(writer io.Writer) error { + if err := json.NewEncoder(writer).Encode(*dr); err != nil { + return sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("encoder.Encode: %v", err)) + } + return nil +}