Merge pull request #3631 from dependabot/go_modules_ignore_conditions…

…_take_two

Go modules support for ignore conditions

bin/docker-dev-shell

@@ -179,6 +179,7 @@ docker run --rm -ti \
   -v "$(pwd)/go_modules/.rubocop.yml:$CODE_DIR/go_modules/.rubocop.yml" \
   -v "$(pwd)/go_modules/Gemfile:$CODE_DIR/go_modules/Gemfile" \
   -v "$(pwd)/go_modules/dependabot-go_modules.gemspec:$CODE_DIR/go_modules/dependabot-go_modules.gemspec" \
+  -v "$(pwd)/go_modules/helpers:$CODE_DIR/go_modules/helpers" \
   -v "$(pwd)/go_modules/lib:$CODE_DIR/go_modules/lib" \
   -v "$(pwd)/go_modules/spec:$CODE_DIR/go_modules/spec" \
   -v "$(pwd)/go_modules/script:$CODE_DIR/go_modules/script" \

go_modules/helpers/main.go

@@ -33,10 +33,10 @@ func main() {
 		funcErr error
 	)
 	switch helperParams.Function {
-	case "getUpdatedVersion":
+	case "getVersions":
 		var args updatechecker.Args
 		parseArgs(helperParams.Args, &args)
-		funcOut, funcErr = updatechecker.GetUpdatedVersion(&args)
+		funcOut, funcErr = updatechecker.GetVersions(&args)
 	case "updateDependencyFile":
 		var args updater.Args
 		parseArgs(helperParams.Args, &args)

go_modules/helpers/updatechecker/main.go

@@ -4,44 +4,30 @@ import (
 	"context"
 	"errors"
 	"io/ioutil"
-	"regexp"
 
 	"github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modfetch"
 	"github.com/dependabot/gomodules-extracted/cmd/go/_internal_/modload"
 	"golang.org/x/mod/modfile"
 	"golang.org/x/mod/semver"
 )
 
-var (
-	pseudoVersionRegexp = regexp.MustCompile(`\b\d{14}-[0-9a-f]{12}$`)
-)
-
 type Dependency struct {
-	Name     string `json:"name"`
-	Version  string `json:"version"`
-	Indirect bool   `json:"indirect"`
-}
-
-type IgnoreRange struct {
-	MinVersionInclusive string `json:"min_version_inclusive"`
-	MaxVersionExclusive string `json:"max_version_exclusive"`
+	Name    string `json:"name"`
+	Version string `json:"version"`
 }
 
 type Args struct {
-	Dependency   *Dependency    `json:"dependency"`
-	IgnoreRanges []*IgnoreRange `json:"ignore_ranges"`
+	Dependency *Dependency `json:"dependency"`
 }
 
-func GetUpdatedVersion(args *Args) (interface{}, error) {
+// GetVersions returns a list of versions for the given dependency that
+// are within the same major version.
+func GetVersions(args *Args) (interface{}, error) {
 	if args.Dependency == nil {
 		return nil, errors.New("Expected args.dependency to not be nil")
 	}
 
 	currentVersion := args.Dependency.Version
-	currentPrerelease := semver.Prerelease(currentVersion)
-	if pseudoVersionRegexp.MatchString(currentPrerelease) {
-		return currentVersion, nil
-	}
 
 	modload.LoadModFile(context.Background())
 
@@ -57,32 +43,25 @@ func GetUpdatedVersion(args *Args) (interface{}, error) {
 	}
 
 	currentMajor := semver.Major(currentVersion)
-	latestVersion := args.Dependency.Version
+
+	var candidateVersions []string
 
 Outer:
 	for _, v := range versions {
 		if semver.Major(v) != currentMajor {
 			continue
 		}
 
-		if semver.Compare(v, latestVersion) < 1 {
-			continue
-		}
-
-		if currentPrerelease == "" && semver.Prerelease(v) != "" {
-			continue
-		}
-
 		for _, exclude := range excludes {
 			if v == exclude {
 				continue Outer
 			}
 		}
 
-		latestVersion = v
+		candidateVersions = append(candidateVersions, v)
 	}
 
-	return latestVersion, nil
+	return candidateVersions, nil
 }
 
 func goModExcludes(dependency string) ([]string, error) {

go_modules/lib/dependabot/go_modules/update_checker.rb

@@ -5,20 +5,12 @@
 require "dependabot/shared_helpers"
 require "dependabot/errors"
 require "dependabot/go_modules/native_helpers"
-require "dependabot/go_modules/resolvability_errors"
 require "dependabot/go_modules/version"
 
 module Dependabot
   module GoModules
     class UpdateChecker < Dependabot::UpdateCheckers::Base
-      RESOLVABILITY_ERROR_REGEXES = [
-        # Package url/proxy doesn't include any redirect meta tags
-        /no go-import meta tags/,
-        # Package url 404s
-        /404 Not Found/,
-        /Repository not found/,
-        /unrecognized import path/
-      ].freeze
+      require_relative "update_checker/latest_version_finder"
 
       def latest_resolvable_version
         # We don't yet support updating indirect dependencies for go_modules
@@ -33,7 +25,13 @@ def latest_resolvable_version
         end
 
         @latest_resolvable_version ||=
-          version_class.new(find_latest_resolvable_version.gsub(/^v/, ""))
+          LatestVersionFinder.new(
+            dependency: dependency,
+            dependency_files: dependency_files,
+            credentials: credentials,
+            ignored_versions: ignored_versions,
+            raise_on_ignored: raise_on_ignored,
+          ).latest_version
       end
 
       # This is currently used to short-circuit latest_resolvable_version,
@@ -56,51 +54,6 @@ def updated_requirements
 
       private
 
-      def find_latest_resolvable_version
-        SharedHelpers.in_a_temporary_directory do
-          SharedHelpers.with_git_configured(credentials: credentials) do
-            File.write("go.mod", go_mod.content)
-
-            # Turn off the module proxy for now, as it's causing issues with
-            # private git dependencies
-            env = { "GOPRIVATE" => "*" }
-
-            SharedHelpers.run_helper_subprocess(
-              command: NativeHelpers.helper_path,
-              env: env,
-              function: "getUpdatedVersion",
-              args: {
-                dependency: {
-                  name: dependency.name,
-                  version: "v" + dependency.version,
-                  indirect: dependency.requirements.empty?
-                }
-              }
-            )
-          end
-        end
-      rescue SharedHelpers::HelperSubprocessFailed => e
-        retry_count ||= 0
-        retry_count += 1
-        retry if transitory_failure?(e) && retry_count < 2
-
-        handle_subprocess_error(e)
-      end
-
-      def handle_subprocess_error(error)
-        if RESOLVABILITY_ERROR_REGEXES.any? { |rgx| error.message =~ rgx }
-          ResolvabilityErrors.handle(error.message, credentials: credentials)
-        end
-
-        raise
-      end
-
-      def transitory_failure?(error)
-        return true if error.message.include?("EOF")
-
-        error.message.include?("Internal Server Error")
-      end
-
       def latest_version_resolvable_with_full_unlock?
         # Full unlock checks aren't implemented for Go (yet)
         false
@@ -137,10 +90,6 @@ def default_source
         { type: "default", source: dependency.name }
       end
 
-      def go_mod
-        @go_mod ||= dependency_files.find { |f| f.name == "go.mod" }
-      end
-
       def git_commit_checker
         @git_commit_checker ||=
           GitCommitChecker.new(

go_modules/lib/dependabot/go_modules/update_checker/latest_version_finder.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "excon"
+
+require "dependabot/shared_helpers"
+require "dependabot/errors"
+require "dependabot/go_modules/requirement"
+require "dependabot/go_modules/resolvability_errors"
+
+module Dependabot
+  module GoModules
+    class UpdateChecker
+      class LatestVersionFinder
+        RESOLVABILITY_ERROR_REGEXES = [
+          # Package url/proxy doesn't include any redirect meta tags
+          /no go-import meta tags/,
+          # Package url 404s
+          /404 Not Found/,
+          /Repository not found/,
+          /unrecognized import path/
+        ].freeze
+        PSEUDO_VERSION_REGEX = /\b\d{14}-[0-9a-f]{12}$/.freeze
+
+        def initialize(dependency:, dependency_files:, credentials:,
+                       ignored_versions:, raise_on_ignored: false)
+          @dependency          = dependency
+          @dependency_files    = dependency_files
+          @credentials         = credentials
+          @ignored_versions    = ignored_versions
+          @raise_on_ignored    = raise_on_ignored
+        end
+
+        def latest_version
+          @latest_version ||= fetch_latest_version
+        end
+
+        private
+
+        attr_reader :dependency, :dependency_files, :credentials, :ignored_versions
+
+        def fetch_latest_version
+          return dependency.version if dependency.version =~ PSEUDO_VERSION_REGEX
+
+          candidate_versions = available_versions
+          candidate_versions = filter_prerelease_versions(candidate_versions)
+          candidate_versions = filter_lower_versions(candidate_versions)
+          candidate_versions = filter_ignored_versions(candidate_versions)
+
+          candidate_versions.max
+        end
+
+        def available_versions
+          SharedHelpers.in_a_temporary_directory do
+            SharedHelpers.with_git_configured(credentials: credentials) do
+              File.write("go.mod", go_mod.content)
+
+              # Turn off the module proxy for now, as it's causing issues with
+              # private git dependencies
+              env = { "GOPRIVATE" => "*" }
+
+              version_strings = SharedHelpers.run_helper_subprocess(
+                command: NativeHelpers.helper_path,
+                env: env,
+                function: "getVersions",
+                args: {
+                  dependency: {
+                    name: dependency.name,
+                    version: "v" + dependency.version,
+                  }
+                }
+              )
+
+              version_strings.select { |v| version_class.correct?(v) }
+                             .map { |v| version_class.new(v) }
+            end
+          end
+        rescue SharedHelpers::HelperSubprocessFailed => e
+          retry_count ||= 0
+          retry_count += 1
+          retry if transitory_failure?(e) && retry_count < 2
+
+          handle_subprocess_error(e)
+        end
+
+        def handle_subprocess_error(error)
+          if RESOLVABILITY_ERROR_REGEXES.any? { |rgx| error.message =~ rgx }
+            ResolvabilityErrors.handle(error.message, credentials: credentials)
+          end
+
+          raise
+        end
+
+        def transitory_failure?(error)
+          return true if error.message.include?("EOF")
+
+          error.message.include?("Internal Server Error")
+        end
+
+        def go_mod
+          @go_mod ||= dependency_files.find { |f| f.name == "go.mod" }
+        end
+
+        def filter_prerelease_versions(versions_array)
+          return versions_array if wants_prerelease?
+
+          versions_array.reject(&:prerelease?)
+        end
+
+        def filter_lower_versions(versions_array)
+          versions_array.
+            select { |version| version > version_class.new(dependency.version) }
+        end
+
+        def filter_ignored_versions(versions_array)
+          filtered = versions_array.
+                     reject { |v| ignore_requirements.any? { |r| r.satisfied_by?(v) } }
+          raise AllVersionsIgnored if @raise_on_ignored && filtered.empty? && versions_array.any?
+
+          filtered
+        end
+
+        def wants_prerelease?
+          @wants_prerelease ||=
+            begin
+              current_version = dependency.version
+              current_version && version_class.correct?(current_version) &&
+                version_class.new(current_version).prerelease?
+            end
+        end
+
+        def ignore_requirements
+          ignored_versions.flat_map { |req| requirement_class.requirements_array(req) }
+        end
+
+        def requirement_class
+          Utils.requirement_class_for_package_manager(
+            dependency.package_manager
+          )
+        end
+
+        def version_class
+          Utils.version_class_for_package_manager(dependency.package_manager)
+        end
+      end
+    end
+  end
+end

go_modules/spec/dependabot/go_modules/update_checker_spec.rb

@@ -80,8 +80,13 @@ module foobar
       end
     end
 
-    it "doesn't update to (Dependabot) ignored versions" do
-      # TODO: let(:ignored_versions) { ["..."] }
+    context "with Dependabot ignored versions" do
+      let(:ignored_versions) { ["> 1.0.1"] }
+
+      it "doesn't update to (Dependabot) ignored versions" do
+        expect(latest_resolvable_version).
+          to eq(Dependabot::GoModules::Version.new("1.0.1"))
+      end
     end
 
     context "when on a pre-release" do
@@ -179,6 +184,16 @@ module foobar
       end
     end
 
+    context "with a retracted update version" do
+      # latest release v1.0.1 is retracted
+      let(:dependency_name) { "github.com/dependabot-fixtures/go-modules-retracted" }
+
+      pending "doesn't update to the retracted version" do
+        expect(latest_resolvable_version).
+          to eq(Dependabot::GoModules::Version.new("1.0.0"))
+      end
+    end
+
     context "when the package url is internal/invalid" do
       let(:dependency_files) { [go_mod] }
       let(:project_name) { "unrecognized_import" }