✨ Add probe metadata about supported ecosystems (#3797)

* 🌱 Add probe metadata about supported ecosystems

Signed-off-by: Adam Korczynski <adam@adalogics.com>

* Add metadata for the rest of the probes

Signed-off-by: Adam Korczynski <adam@adalogics.com>

* fix wrong formatting

Signed-off-by: Adam Korczynski <adam@adalogics.com>

* remove oss-fuzz, osv, cii_blob, cii_http clients

Signed-off-by: Adam Korczynski <adam@adalogics.com>

* add github and gitlab clients for 2 probes

Signed-off-by: Adam Korczynski <adam@adalogics.com>

---------

Signed-off-by: Adam Korczynski <adam@adalogics.com>

clients/languages.go

@@ -44,6 +44,9 @@ const (
 	// C#: https://docs.microsoft.com/en-us/dotnet/csharp/
 	CSharp LanguageName = "c#"
 
+	// ObjectiveC: the objective c language.
+	ObjectiveC LanguageName = "objectivec"
+
 	// Ruby: https://www.ruby-lang.org/
 	Ruby LanguageName = "ruby"
 
@@ -77,6 +80,9 @@ const (
 	// Other indicates other languages not listed by the GitHub API.
 	Other LanguageName = "other"
 
+	// All indicates all programming languages.
+	All LanguageName = "all"
+
 	// Add more languages here if needed,
 	// please use lowercases for the LanguageName value.
 )

finding/probe/probe.go

@@ -21,6 +21,8 @@ import (
 	"strings"
 
 	"gopkg.in/yaml.v3"
+
+	"github.com/ossf/scorecard/v4/clients"
 )
 
 var errInvalid = errors.New("invalid")
@@ -57,11 +59,23 @@ type yamlRemediation struct {
 	Effort   RemediationEffort `yaml:"effort"`
 }
 
+type yamlEcosystem struct {
+	Languages []string `yaml:"languages"`
+	Clients   []string `yaml:"clients"`
+}
+
+var supportedClients = map[string]bool{
+	"github":   true,
+	"gitlab":   true,
+	"localdir": true,
+}
+
 type yamlProbe struct {
 	ID             string          `yaml:"id"`
 	Short          string          `yaml:"short"`
 	Motivation     string          `yaml:"motivation"`
 	Implementation string          `yaml:"implementation"`
+	Ecosystem      yamlEcosystem   `yaml:"ecosystem"`
 	Remediation    yamlRemediation `yaml:"remediation"`
 }
 
@@ -114,6 +128,9 @@ func validate(r *yamlProbe, probeID string) error {
 	if err := validateRemediation(r.Remediation); err != nil {
 		return err
 	}
+	if err := validateEcosystem(r.Ecosystem); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -134,6 +151,43 @@ func validateRemediation(r yamlRemediation) error {
 	}
 }
 
+func validateEcosystem(r yamlEcosystem) error {
+	if err := validateSupportedLanguages(r); err != nil {
+		return err
+	}
+	if err := validateSupportedClients(r); err != nil {
+		return err
+	}
+	return nil
+}
+
+func validateSupportedLanguages(r yamlEcosystem) error {
+	for _, lang := range r.Languages {
+		switch clients.LanguageName(lang) {
+		case clients.Go, clients.Python, clients.JavaScript,
+			clients.Cpp, clients.C, clients.TypeScript,
+			clients.Java, clients.CSharp, clients.Ruby,
+			clients.PHP, clients.StarLark, clients.Scala,
+			clients.Kotlin, clients.Swift, clients.Rust,
+			clients.Haskell, clients.All, clients.Dockerfile,
+			clients.ObjectiveC:
+			continue
+		default:
+			return fmt.Errorf("%w: %v", errInvalid, fmt.Sprintf("language '%v'", r))
+		}
+	}
+	return nil
+}
+
+func validateSupportedClients(r yamlEcosystem) error {
+	for _, lang := range r.Clients {
+		if _, ok := supportedClients[lang]; !ok {
+			return fmt.Errorf("%w: %v", errInvalid, fmt.Sprintf("client '%v'", r))
+		}
+	}
+	return nil
+}
+
 func parseFromYAML(content []byte) (*yamlProbe, error) {
 	r := yamlProbe{}
 

finding/probe/probe_test.go

@@ -82,6 +82,18 @@ func Test_FromBytes(t *testing.T) {
 			path: "testdata/invalid-effort.yml",
 			err:  errInvalid,
 		},
+		{
+			name: "invalid language",
+			id:   "invalid-language",
+			path: "testdata/invalid-language.yml",
+			err:  errInvalid,
+		},
+		{
+			name: "invalid client",
+			id:   "invalid-client",
+			path: "testdata/invalid-client.yml",
+			err:  errInvalid,
+		},
 	}
 	for _, tt := range tests {
 		tt := tt // Re-initializing variable so it is not changed while executing the closure below

finding/probe/testdata/all-fields.yml

@@ -14,3 +14,11 @@ remediation:
   markdown:
     - step1
     - step2 [google.com](https://www.google.com/something)
+ecosystem:
+  languages:
+    - c
+    - c++
+  clients:
+    - github
+    - gitlab
+    - localdir

finding/probe/testdata/invalid-client.yml

@@ -0,0 +1,23 @@
+id: invalid-client
+short: short description
+motivation: >
+  mot1
+  mot2
+implementation: >
+  impl1
+  impl2
+remediation:
+  effort: Low
+  text:
+    - step1
+    - step2 https://www.google.com/something
+  markdown:
+    - step1
+    - step2 [google.com](https://www.google.com/something)
+ecosystem:
+  languages:
+    - c
+  clients:
+    - githubb
+    - gitlab
+    - localdir

finding/probe/testdata/invalid-language.yml

@@ -0,0 +1,23 @@
+id: invalid-language
+short: short description
+motivation: >
+  mot1
+  mot2
+implementation: >
+  impl1
+  impl2
+remediation:
+  effort: Low
+  text:
+    - step1
+    - step2 https://www.google.com/something
+  markdown:
+    - step1
+    - step2 [google.com](https://www.google.com/something)
+ecosystem:
+  languages:
+    - fortran
+  clients:
+    - github
+    - gitlab
+    - localdir

probes/blocksDeleteOnBranches/def.yml

@@ -24,4 +24,10 @@ remediation:
   effort: Low
   text:
     - Disallow deletion of branches in your project to remove negative outcomes.
-    - GitHub and GitLab by default disable deleting a protected branch. 
+    - GitHub and GitLab by default disable deleting a protected branch. 
+ecosystem:
+  languages:
+    - all
+  clients:
+    - github
+    - gitlab

probes/blocksForcePushOnBranches/def.yml

@@ -30,4 +30,10 @@ remediation:
   markdown:
     - Disallow force pushes branches in your project to remove negative outcomes.
     - For GitHub-hosted projects, force pushes are disabled by default. To make sure it has not been enabled, see ["Allow force pushes"](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches#allow-force-pushes).
-    - For GitLab-hosted projects, follow the ["Protected branches"](https://docs.gitlab.com/ee/user/project/protected_branches.html) documentation to see who can force push to the project.
+    - For GitLab-hosted projects, follow the ["Protected branches"](https://docs.gitlab.com/ee/user/project/protected_branches.html) documentation to see who can force push to the project.
+ecosystem:
+  languages:
+    - all
+  clients:
+    - github
+    - gitlab

probes/branchProtectionAppliesToAdmins/def.yml

@@ -31,4 +31,10 @@ remediation:
     - The remediation effort can be Low to High depending on other branch protection settings.
     - Enforce protection rules for admins on all branches.
     - For GitHub-hosted projects, see the ["Do not allow bypassing the above settings"](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches#do-not-allow-bypassing-the-above-settings) section in the GitHub docs.
-    - For GitLab-hosted projects, see the ["Protected branches"](https://docs.gitlab.com/ee/user/project/protected_branches.html) documentation.
+    - For GitLab-hosted projects, see the ["Protected branches"](https://docs.gitlab.com/ee/user/project/protected_branches.html) documentation.
+ecosystem:
+  languages:
+    - all
+  clients:
+    - github
+    - gitlab

probes/contributorsFromOrgOrCompany/def.yml

@@ -29,4 +29,10 @@ remediation:
   effort: High
   text:
     - Encourage community-driven contributions to your project.
-    - Ask contributors to join their respective organizations, if they have not already. Otherwise, there is no remediation for this probe; it simply provides insight into how many organizations have contributed so that you can make a trust-based decision based on that information.
+    - Ask contributors to join their respective organizations, if they have not already. Otherwise, there is no remediation for this probe; it simply provides insight into how many organizations have contributed so that you can make a trust-based decision based on that information.
+ecosystem:
+  languages:
+    - all
+  clients:
+    - github
+    - gitlab

probes/dismissesStaleReviews/def.yml

@@ -25,4 +25,10 @@ remediation:
   text:
     - Configure your repository so that the stale status of PRs is dismissed when users make new commits.
     - For GitHub-hosted projects, see ["Require pull request reviews before merging"](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches#require-pull-request-reviews-before-merging).
-    - For GitLab-hosted projects, see ["Remove all approvals when commits are added to the source branch"](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/settings.html#remove-all-approvals-when-commits-are-added-to-the-source-branch).
+    - For GitLab-hosted projects, see ["Remove all approvals when commits are added to the source branch"](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/settings.html#remove-all-approvals-when-commits-are-added-to-the-source-branch).
+ecosystem:
+  languages:
+    - all
+  clients:
+    - github
+    - gitlab

probes/freeOfAnyBinaryArtifacts/def.yml

@@ -26,3 +26,9 @@ remediation:
   text:
     - Remove the generated executable artifacts from the repository.
     - Build from source.
+ecosystem:
+  languages:
+    - all
+  clients:
+    - github
+    - gitlab

probes/freeOfUnverifiedBinaryArtifacts/def.yml

@@ -26,3 +26,9 @@ remediation:
   text:
     - Remove the generated executable artifacts from the repository.
     - Build from source.
+ecosystem:
+  languages:
+    - all
+  clients:
+    - github
+    - gitlab

probes/fuzzedWithCLibFuzzer/def.yml

@@ -30,3 +30,10 @@ remediation:
   markdown:
     - Follow the steps in [https://llvm.org/docs/LibFuzzer.html](https://llvm.org/docs/LibFuzzer.html) to enable fuzzing on your project.
     - Over time, try to add fuzzing for more functionalities of your project.
+ecosystem:
+  languages:
+    - c
+    - c++
+  clients:
+    - github
+    - gitlab

probes/fuzzedWithClusterFuzzLite/def.yml

@@ -29,4 +29,9 @@ remediation:
     - Over time, try to add fuzzing for more functionalities of your project.
   markdown:
     - Follow the steps in [https://github.com/google/clusterfuzzlite](https://github.com/google/clusterfuzzlite) to integrate fuzzing as part of CI.
-    - Over time, try to add fuzzing for more functionalities of your project.
+    - Over time, try to add fuzzing for more functionalities of your project.
+ecosystem:
+  languages:
+    - all
+  clients:
+    - github

probes/fuzzedWithCppLibFuzzer/def.yml

@@ -30,3 +30,10 @@ remediation:
   markdown:
     - Follow the steps in [https://llvm.org/docs/LibFuzzer.html](https://llvm.org/docs/LibFuzzer.html) to enable fuzzing on your project.
     - Over time, try to add fuzzing for more functionalities of your project.
+ecosystem:
+  languages:
+    - c
+    - c++
+  clients:
+    - github
+    - gitlab

probes/fuzzedWithGoNative/def.yml

@@ -29,4 +29,10 @@ remediation:
     - Over time, try to add fuzzing for more functionalities of your project.
   markdown:
     - Follow the steps in [https://go.dev/doc/fuzz/](https://go.dev/doc/fuzz/) to enable fuzzing on your project.
-    - Over time, try to add fuzzing for more functionalities of your project.
+    - Over time, try to add fuzzing for more functionalities of your project.
+ecosystem:
+  languages:
+    - go
+  clients:
+    - github
+    - gitlab

probes/fuzzedWithJavaJazzerFuzzer/def.yml

@@ -30,3 +30,9 @@ remediation:
   markdown:
     - Follow the steps in [https://github.com/CodeIntelligenceTesting/jazzer](https://github.com/CodeIntelligenceTesting/jazzer) to enable fuzzing on your project.
     - Over time, try to add fuzzing for more functionalities of your project.
+ecosystem:
+  languages:
+    - java
+  clients:
+    - github
+    - gitlab

probes/fuzzedWithOSSFuzz/def.yml

@@ -29,4 +29,17 @@ remediation:
     - Over time, try to add fuzzing for more functionalities of your project.
   markdown:
     - Follow the steps in [https://github.com/google/oss-fuzz](https://github.com/google/oss-fuzz) to integrate fuzzing for your project.
-    - Over time, try to add fuzzing for more functionalities of your project.
+    - Over time, try to add fuzzing for more functionalities of your project.
+ecosystem:
+  languages:
+    - c
+    - c++
+    - go
+    - java
+    - javascript
+    - python
+    - rust
+    - typescript
+  clients:
+    - github
+    - gitlab

probes/fuzzedWithPropertyBasedHaskell/def.yml

@@ -39,4 +39,10 @@ remediation:
     - '[validity](https://github.com/NorfairKing/validity)'
     - '[smallcheck](https://hackage.haskell.org/package/smallcheck)'
     - '[hspec](https://hspec.github.io/)'
-    - '[tasty](https://hackage.haskell.org/package/tasty)'
+    - '[tasty](https://hackage.haskell.org/package/tasty)'
+ecosystem:
+  languages:
+    - haskell
+  clients:
+    - github
+    - gitlab

probes/fuzzedWithPropertyBasedJavascript/def.yml

@@ -27,4 +27,10 @@ remediation:
   text:
     - 'Use fast-check: https://github.com/dubzzz/fast-check'
   markdown:
-    - 'Use [fast-check](https://github.com/dubzzz/fast-check)'
+    - 'Use [fast-check](https://github.com/dubzzz/fast-check)'
+ecosystem:
+  languages:
+    - javascript
+  clients:
+    - github
+    - gitlab

probes/fuzzedWithPropertyBasedTypescript/def.yml

@@ -27,4 +27,10 @@ remediation:
   text:
     - 'Use fast-check: https://github.com/dubzzz/fast-check'
   markdown:
-    - 'Use [fast-check](https://github.com/dubzzz/fast-check)'
+    - 'Use [fast-check](https://github.com/dubzzz/fast-check)'
+ecosystem:
+  languages:
+    - typescript
+  clients:
+    - github
+    - gitlab