Get hashes from private Python registries, not just PyPI

.editorconfig

@@ -15,3 +15,7 @@ indent_style = tab
 
 [*.php]
 indent_size = 4
+
+[*.py]
+indent_size = 4
+max_line_length = 80

python/helpers/lib/hasher.py

@@ -5,19 +5,22 @@
 from poetry.factory import Factory
 
 
-def get_dependency_hash(dependency_name, dependency_version, algorithm, index_url=hashin.DEFAULT_INDEX_URL):
+def get_dependency_hash(dependency_name, dependency_version, algorithm,
+                        index_url=hashin.DEFAULT_INDEX_URL):
     try:
-      hashes = hashin.get_package_hashes(
-          dependency_name,
-          version=dependency_version,
-          algorithm=algorithm,
-          index_url=index_url
-      )
-      return json.dumps({"result": hashes["hashes"]})
+        hashes = hashin.get_package_hashes(
+            dependency_name,
+            version=dependency_version,
+            algorithm=algorithm,
+            index_url=index_url
+        )
+        return json.dumps({"result": hashes["hashes"]})
     except hashin.PackageNotFoundError as e:
-      return json.dumps({"error": repr(e), "error_class:" :
-                         e.__class__.__name__, "trace:":
-                         ''.join(traceback.format_stack())})
+        return json.dumps({
+            "error": repr(e),
+            "error_class:": e.__class__.__name__,
+            "trace:": ''.join(traceback.format_stack())
+        })
 
 
 def get_pipfile_hash(directory):

python/lib/dependabot/python/file_updater/pip_compile_file_updater.rb

@@ -34,7 +34,7 @@ class PipCompileFileUpdater
         attr_reader :dependency_files
         attr_reader :credentials
 
-        def initialize(dependencies:, dependency_files:, credentials:, index_urls:)
+        def initialize(dependencies:, dependency_files:, credentials:, index_urls: nil)
           @dependencies = dependencies
           @dependency_files = dependency_files
           @credentials = credentials
@@ -267,7 +267,7 @@ def freeze_dependency_requirement(file)
             dependency_name: dependency.name,
             old_requirement: old_req[:requirement],
             new_requirement: "==#{dependency.version}",
-            index_urls: index_urls
+            index_urls: @index_urls
           ).updated_content
         end
 
@@ -286,7 +286,7 @@ def update_dependency_requirement(file)
             dependency_name: dependency.name,
             old_requirement: old_req[:requirement],
             new_requirement: new_req[:requirement],
-            index_urls: index_urls
+            index_urls: @index_urls
           ).updated_content
         end
 
@@ -392,16 +392,29 @@ def deps_to_augment_hashes_for(updated_content, original_content)
         end
 
         def package_hashes_for(name:, version:, algorithm:)
-          args = [name, version, algorithm]
-
-          index_urls.map do |index_url|
-            args << index_url unless index_url.nil?
-            SharedHelpers.run_helper_subprocess(
-              command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
-              function: "get_dependency_hash",
-              args: args
-            ).map { |h| "--hash=#{algorithm}:#{h['hash']}" }
+          index_urls = @index_urls || [nil]
+          hashes = []
+
+          index_urls.each do |index_url|
+            args = [name, version, algorithm]
+            args << index_url if index_url
+
+            begin
+              native_helper_hashes = SharedHelpers.run_helper_subprocess(
+                command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
+                function: "get_dependency_hash",
+                args: args
+              ).map { |h| "--hash=#{algorithm}:#{h['hash']}" }
+
+              hashes.concat(native_helper_hashes)
+            rescue SharedHelpers::HelperSubprocessFailed => e
+              raise unless e.message.include?("PackageNotFoundError")
+
+              next
+            end
           end
+
+          hashes
         end
 
         def hash_separator(requirement_string)

python/lib/dependabot/python/file_updater/requirement_file_updater.rb

@@ -60,7 +60,7 @@ def updated_requirement_or_setup_file_content(new_req, old_req)
             old_requirement: old_req.fetch(:requirement),
             new_requirement: new_req.fetch(:requirement),
             new_hash_version: dependency.version,
-            index_urls: index_urls
+            index_urls: @index_urls
           ).updated_content
         end
 

python/lib/dependabot/python/file_updater/requirement_replacer.rb

@@ -141,17 +141,27 @@ def hash_separator(requirement)
         end
 
         def package_hashes_for(name:, version:, algorithm:)
+          index_urls = @index_urls || [nil]
+
           index_urls.map do |index_url|
+            args = [name, version, algorithm]
             args << index_url unless index_url.nil?
-            result = SharedHelpers.run_helper_subprocess(
-              command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
-              function: "get_dependency_hash",
-              args: [name, version, algorithm]
-            )
 
-            result.map { |h| "--hash=#{algorithm}:#{h['hash']}" } if result.id_a?(Array)
+            begin
+              result = SharedHelpers.run_helper_subprocess(
+                command: "pyenv exec python3 #{NativeHelpers.python_helper_path}",
+                function: "get_dependency_hash",
+                args: args
+              )
+            rescue SharedHelpers::HelperSubprocessFailed => e
+              raise unless e.message.include?("PackageNotFoundError")
+
+              next
+            end
+
+            next if result.first.key?("error_class") && result.first["error_class"] == PACKAGE_NOT_FOUND_ERROR
 
-            next if result["error_class"] == PACKAGE_NOT_FOUND_ERROR
+            return result.map { |h| "--hash=#{algorithm}:#{h['hash']}" } if result.is_a?(Array)
           end
 
           raise Dependabot::DependencyFileNotResolvable, "Unable to find hashes for package #{name}"

python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb

@@ -363,7 +363,7 @@
 
       subject(:dependencies) { parser.dependency_set.dependencies }
 
-      its(:length) { is_expected.to be > 0 }
+      its(:length) { is_expected.to be.positive? }
     end
   end
 end

python/spec/dependabot/python/file_updater/pip_compile_file_updater_spec.rb

@@ -551,4 +551,86 @@
       end
     end
   end
+
+  describe "#package_hashes_for" do
+    let(:name) { "package_name" }
+    let(:version) { "1.0.0" }
+    let(:algorithm) { "sha256" }
+
+    context "when index_urls is not set" do
+      let(:updater) do
+        described_class.new(
+          dependencies: [],
+          dependency_files: [],
+          credentials: []
+        )
+      end
+
+      before do
+        allow(Dependabot::SharedHelpers).to receive(:run_helper_subprocess).and_return([{ "hash" => "123abc" }])
+      end
+
+      it "returns hash" do
+        result = updater.send(:package_hashes_for, name: name, version: version, algorithm: algorithm)
+        expect(result).to eq(["--hash=sha256:123abc"])
+      end
+    end
+
+    context "when multiple index_urls are set" do
+      let(:updater) do
+        described_class.new(
+          dependencies: [],
+          dependency_files: [],
+          credentials: [],
+          index_urls: [nil, "http://example.com"]
+        )
+      end
+
+      before do
+        allow(Dependabot::SharedHelpers).to receive(:run_helper_subprocess)
+          .and_return([{ "hash" => "123abc" }], [{ "hash" => "312cba" }])
+      end
+
+      it "returns returns two hashes" do
+        result = updater.send(:package_hashes_for, name: name, version: version, algorithm: algorithm)
+        expect(result).to eq(%w(--hash=sha256:123abc --hash=sha256:312cba))
+      end
+    end
+
+    context "when multiple index_urls are set but package does not exist in PyPI" do
+      let(:updater) do
+        described_class.new(
+          dependencies: [],
+          dependency_files: [],
+          credentials: [],
+          index_urls: [nil, "http://example.com"]
+        )
+      end
+
+      before do
+        allow(Dependabot::SharedHelpers).to receive(:run_helper_subprocess).with({
+          args: %w(package_name 1.0.0
+                   sha256),
+          command: "pyenv exec python3 /opt/python/run.py",
+          function: "get_dependency_hash"
+        }).and_raise(
+          Dependabot::SharedHelpers::HelperSubprocessFailed.new(
+            message: "Error message", error_context: {}
+          )
+        )
+
+        allow(Dependabot::SharedHelpers).to receive(:run_helper_subprocess)
+          .with({
+            args: %w(package_name 1.0.0 sha256 http://example.com),
+            command: "pyenv exec python3 /opt/python/run.py",
+            function: "get_dependency_hash"
+          }).and_return([{ "hash" => "123abc" }])
+      end
+
+      it "returns returns two hashes" do
+        result = updater.send(:package_hashes_for, name: name, version: version, algorithm: algorithm)
+        expect(result).to eq(["--hash=sha256:123abc"])
+      end
+    end
+  end
 end

python/spec/dependabot/python/file_updater/requirement_file_updater_spec.rb

@@ -90,11 +90,11 @@
         its(:content) { is_expected.to include "psycopg2==2.8.1  # Comment!\n" }
       end
 
-      context "with hashes" do
+      context "with an unknown package" do
         let(:dependency) do
           Dependabot::Dependency.new(
-            name: "asml_patch_helper",
-            version: "18.2.0",
+            name: "some_unknown_package",
+            version: "24.3.3",
             requirements: [{
               file: "requirements.txt",
               requirement: updated_requirement_string,
@@ -111,12 +111,26 @@
           )
         end
 
-        let(:requirements_fixture_name) { "asml.txt" }
+        let(:requirements_fixture_name) { "hashes_unknown_package.txt" }
         let(:previous_requirement_string) { "==24.3.3" }
         let(:updated_requirement_string) { "==24.4.0" }
-        its(:content) do
-          is_expected.to include "asml_patch_helper==24.4.0 \\\n"
-          is_expected.to include "    --hash=sha256:94723f660aa638ac9154c6ffefca0de718d9bfa741eb96a99894300f764960ed\n"
+
+        context "when package is not in default index" do
+          it "raises an error" do
+            expect { updated_files }.to raise_error(Dependabot::DependencyFileNotResolvable)
+          end
+        end
+
+        context "when package is in default index" do
+          before do
+            allow(Dependabot::SharedHelpers).to receive(:run_helper_subprocess)
+              .and_return([{ "hash" => "1234567890abcdef" }])
+          end
+
+          its(:content) do
+            is_expected.to include "some_unknown_package==24.4.0"
+            is_expected.to include "--hash=sha256:1234567890abcdef"
+          end
         end
       end
 

python/spec/dependabot/python/file_updater/requirement_replacer_spec.rb

@@ -82,20 +82,6 @@
         it { is_expected.to include("Flask-SQLAlchemy\n") }
         it { is_expected.to include("zope.SQLAlchemy\n") }
       end
-
-      context "with a hash algorithm" do
-        let(:old_requirement) { "attrs==18.2.0" }
-        let(:new_requirement) { "attrs==18.3.0" }
-        let(:requirement_content) do
-          <<~REQUIREMENT
-            attrs==18.2.0 \
-                --hash=sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69 \
-                --hash=sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb
-          REQUIREMENT
-        end
-        let(:dependency_name) { "attrs" }
-        it { is_expected.to eq("attrs==1.11.5") }
-      end
     end
   end
 end

python/spec/dependabot/python/file_updater_spec.rb

@@ -391,5 +391,37 @@
         updated_files.each { |f| expect(f).to be_a(Dependabot::DependencyFile) }
       end
     end
+
+    describe "#pip_compile_index_urls" do
+      let(:instance) do
+        described_class.new(
+          dependencies: [],
+          dependency_files: [],
+          credentials: credentials
+        )
+      end
+
+      let(:credentials) { [double(replaces_base?: replaces_base)] }
+      let(:replaces_base) { false }
+
+      before do
+        allow_any_instance_of(Dependabot::Python::FileUpdater).to receive(:check_required_files).and_return(true)
+        allow(Dependabot::Python::AuthedUrlBuilder).to receive(:authed_url).and_return("authed_url")
+      end
+
+      context "when credentials replace base" do
+        let(:replaces_base) { true }
+
+        it "returns authed urls for these credentials" do
+          expect(instance.send(:pip_compile_index_urls)).to eq(["authed_url"])
+        end
+      end
+
+      context "when credentials do not replace base" do
+        it "returns nil and authed urls for all credentials" do
+          expect(instance.send(:pip_compile_index_urls)).to eq([nil, "authed_url"])
+        end
+      end
+    end
   end
 end

python/spec/fixtures/pip_compile_files/hashes.in

@@ -0,0 +1,3 @@
+attrs==18.2.0
+  --hash=sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69
+  --hash=sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb