From ce6e4790dfc49b627874c741d56c75089a88d4e1 Mon Sep 17 00:00:00 2001
From: Albert Tugushev <albert@tugushev.ru>
Date: Sun, 19 Apr 2020 14:37:33 +0700
Subject: [PATCH 1/6] Add support for pip's 2020 dependency resolver

---
 piptools/repositories/base.py    |   6 +
 piptools/repositories/local.py   |   6 +
 piptools/repositories/pypi.py    |  16 +-
 piptools/resolver.py             | 411 ++++++++++++++++++++++++++++---
 piptools/scripts/compile.py      |  29 ++-
 piptools/utils.py                |  41 +++
 piptools/writer.py               |  38 ++-
 tests/conftest.py                |  55 ++++-
 tests/test_cli_compile.py        | 130 +++++++++-
 tests/test_resolver.py           |  26 ++
 tests/test_top_level_editable.py |   0
 11 files changed, 686 insertions(+), 72 deletions(-)
 create mode 100644 tests/test_top_level_editable.py

diff --git a/piptools/repositories/base.py b/piptools/repositories/base.py
index 3ffec38d6..7e6c354a9 100644
--- a/piptools/repositories/base.py
+++ b/piptools/repositories/base.py
@@ -3,6 +3,7 @@
 from contextlib import contextmanager
 from typing import Iterator, Optional, Set
 
+from pip._internal.commands.install import InstallCommand
 from pip._internal.index.package_finder import PackageFinder
 from pip._internal.models.index import PyPI
 from pip._internal.network.session import PipSession
@@ -61,3 +62,8 @@ def session(self) -> PipSession:
     @abstractmethod
     def finder(self) -> PackageFinder:
         """Returns a package finder to interact with simple repository API (PEP 503)"""
+
+    @property
+    @abstractmethod
+    def command(self) -> InstallCommand:
+        """Return an install command."""
diff --git a/piptools/repositories/local.py b/piptools/repositories/local.py
index 754b6076a..99d990296 100644
--- a/piptools/repositories/local.py
+++ b/piptools/repositories/local.py
@@ -2,6 +2,7 @@
 from contextlib import contextmanager
 from typing import Iterator, Mapping, Optional, Set, cast
 
+from pip._internal.commands.install import InstallCommand
 from pip._internal.index.package_finder import PackageFinder
 from pip._internal.models.candidate import InstallationCandidate
 from pip._internal.req import InstallRequirement
@@ -61,6 +62,11 @@ def finder(self) -> PackageFinder:
     def session(self) -> Session:
         return self.repository.session
 
+    @property
+    def command(self) -> InstallCommand:
+        """Return an install command instance."""
+        return self.repository.command
+
     def clear_caches(self) -> None:
         self.repository.clear_caches()
 
diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py
index cc99c0c6f..533a16deb 100644
--- a/piptools/repositories/pypi.py
+++ b/piptools/repositories/pypi.py
@@ -73,10 +73,9 @@ def __init__(self, pip_args: List[str], cache_dir: str):
         # Use pip's parser for pip.conf management and defaults.
         # General options (find_links, index_url, extra_index_url, trusted_host,
         # and pre) are deferred to pip.
-        self.command: InstallCommand = create_command("install")
-        extra_pip_args = ["--use-deprecated", "legacy-resolver"]
+        self._command: InstallCommand = create_command("install")
 
-        options, _ = self.command.parse_args(pip_args + extra_pip_args)
+        options, _ = self.command.parse_args(pip_args)
         if options.cache_dir:
             options.cache_dir = normalize_path(options.cache_dir)
         options.require_hashes = False
@@ -103,8 +102,7 @@ def __init__(self, pip_args: List[str], cache_dir: str):
         self._cache_dir = normalize_path(str(cache_dir))
         self._download_dir = os.path.join(self._cache_dir, "pkgs")
 
-        if PIP_VERSION[0] < 22:
-            self._setup_logging()
+        self._setup_logging()
 
     def clear_caches(self) -> None:
         rmtree(self._download_dir, ignore_errors=True)
@@ -121,6 +119,11 @@ def session(self) -> PipSession:
     def finder(self) -> PackageFinder:
         return self._finder
 
+    @property
+    def command(self) -> InstallCommand:
+        """Return an install command instance."""
+        return self._command
+
     def find_all_candidates(self, req_name: str) -> List[InstallationCandidate]:
         if req_name not in self._available_candidates_cache:
             candidates = self.finder.find_all_candidates(req_name)
@@ -464,6 +467,9 @@ def _setup_logging(self) -> None:
             user_log_file=self.options.log,
         )
 
+        if PIP_VERSION[0] >= 22:
+            return
+
         # Sync pip's console handler stream with LogContext.stream
         logger = logging.getLogger()
         for handler in logger.handlers:
diff --git a/piptools/resolver.py b/piptools/resolver.py
index 012e9f030..61a287cf2 100644
--- a/piptools/resolver.py
+++ b/piptools/resolver.py
@@ -1,24 +1,53 @@
+import collections
 import copy
+from abc import ABCMeta, abstractmethod
 from functools import partial
 from itertools import chain, count, groupby
-from typing import Dict, Iterable, Iterator, List, Optional, Set, Tuple
+from typing import (
+    Any,
+    DefaultDict,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+)
 
 import click
+from pip._internal.cache import WheelCache
+from pip._internal.exceptions import DistributionNotFound
 from pip._internal.req import InstallRequirement
 from pip._internal.req.constructors import install_req_from_line
+from pip._internal.resolution.resolvelib.base import Candidate
+from pip._internal.resolution.resolvelib.candidates import ExtrasCandidate
+from pip._internal.resolution.resolvelib.resolver import Resolver
+from pip._internal.utils.logging import indent_log
+from pip._internal.utils.temp_dir import TempDirectory, global_tempdir_manager
+from pip._vendor.packaging.specifiers import SpecifierSet
+from pip._vendor.packaging.utils import canonicalize_name
+from pip._vendor.resolvelib.resolvers import ResolutionImpossible, Result
 
 from piptools.cache import DependencyCache
 from piptools.repositories.base import BaseRepository
 
-from ._compat.pip_compat import update_env_context_manager
+from ._compat import PIP_VERSION
+from ._compat.pip_compat import get_build_tracker, update_env_context_manager
+from .exceptions import PipToolsError
 from .logging import log
 from .utils import (
     UNSAFE_PACKAGES,
+    as_tuple,
+    copy_install_requirement,
     format_requirement,
     format_specifier,
     is_pinned_requirement,
     is_url_requirement,
     key_from_ireq,
+    key_from_req,
+    remove_value,
+    strip_extras,
 )
 
 green = partial(click.style, fg="green")
@@ -133,10 +162,55 @@ def combine_install_requirements(
     return combined_ireq
 
 
-class Resolver:
+class BaseResolver(metaclass=ABCMeta):
+    repository: BaseRepository
+    unsafe_constraints: Set[InstallRequirement]
+
+    @abstractmethod
+    def resolve(self, max_rounds: int) -> Set[InstallRequirement]:
+        """
+        Find concrete package versions for all the given InstallRequirements
+        and their recursive dependencies and return a set of pinned
+        ``InstallRequirement``'s.
+        """
+
+    def resolve_hashes(
+        self, ireqs: Set[InstallRequirement]
+    ) -> Dict[InstallRequirement, Set[str]]:
+        """Find acceptable hashes for all of the given ``InstallRequirement``s."""
+        log.debug("")
+        log.debug("Generating hashes:")
+        with self.repository.allow_all_wheels(), log.indentation():
+            return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs}
+
+    def _filter_out_unsafe_constraints(
+        self,
+        ireqs: Set[InstallRequirement],
+        reverse_dependencies: Dict[str, Set[str]],
+    ) -> None:
+        """
+        Remove from a given set of ``InstallRequirement``'s unsafe constraints.
+
+        Reverse_dependencies is used to filter out packages that are only
+        required by unsafe packages. This logic is incomplete, as it would
+        fail to filter sub-sub-dependencies of unsafe packages. None of the
+        UNSAFE_PACKAGES currently have any dependencies at all (which makes
+        sense for installation tools) so this seems sufficient.
+        """
+        for req in ireqs.copy():
+            required_by = reverse_dependencies.get(req.name.lower(), set())
+            if req.name in UNSAFE_PACKAGES or (
+                required_by and all(name in UNSAFE_PACKAGES for name in required_by)
+            ):
+                self.unsafe_constraints.add(req)
+                ireqs.remove(req)
+
+
+class LegacyResolver(BaseResolver):
     def __init__(
         self,
         constraints: Iterable[InstallRequirement],
+        existing_constraints: Dict[str, InstallRequirement],
         repository: BaseRepository,
         cache: DependencyCache,
         prereleases: Optional[bool] = False,
@@ -157,28 +231,26 @@ def __init__(
         self.allow_unsafe = allow_unsafe
         self.unsafe_constraints: Set[InstallRequirement] = set()
 
+        options = self.repository.options
+        if "legacy-resolver" not in options.deprecated_features_enabled:
+            raise PipToolsError("Legacy resolver deprecated feature must be enabled.")
+
+        # Make sure there is no enabled 2020-resolver
+        options.features_enabled = remove_value(
+            options.features_enabled, "2020-resolver"
+        )
+
     @property
     def constraints(self) -> Set[InstallRequirement]:
         return set(
             self._group_constraints(chain(self.our_constraints, self.their_constraints))
         )
 
-    def resolve_hashes(
-        self, ireqs: Set[InstallRequirement]
-    ) -> Dict[InstallRequirement, Set[str]]:
-        """
-        Finds acceptable hashes for all of the given InstallRequirements.
-        """
-        log.debug("")
-        log.debug("Generating hashes:")
-        with self.repository.allow_all_wheels(), log.indentation():
-            return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs}
-
     def resolve(self, max_rounds: int = 10) -> Set[InstallRequirement]:
         """
-        Finds concrete package versions for all the given InstallRequirements
-        and their recursive dependencies.  The end result is a flat list of
-        (name, version) tuples.  (Or an editable package.)
+        Find concrete package versions for all the given InstallRequirements
+        and their recursive dependencies and return a set of pinned
+        ``InstallRequirement``'s.
 
         Resolves constraints one round at a time, until they don't change
         anymore.  Protects against infinite loops by breaking out after a max
@@ -216,21 +288,11 @@ def resolve(self, max_rounds: int = 10) -> Set[InstallRequirement]:
         results = {req for req in best_matches if not req.constraint}
 
         # Filter out unsafe requirements.
-        self.unsafe_constraints = set()
         if not self.allow_unsafe:
-            # reverse_dependencies is used to filter out packages that are only
-            # required by unsafe packages. This logic is incomplete, as it would
-            # fail to filter sub-sub-dependencies of unsafe packages. None of the
-            # UNSAFE_PACKAGES currently have any dependencies at all (which makes
-            # sense for installation tools) so this seems sufficient.
-            reverse_dependencies = self.reverse_dependencies(results)
-            for req in results.copy():
-                required_by = reverse_dependencies.get(req.name.lower(), set())
-                if req.name in UNSAFE_PACKAGES or (
-                    required_by and all(name in UNSAFE_PACKAGES for name in required_by)
-                ):
-                    self.unsafe_constraints.add(req)
-                    results.remove(req)
+            self._filter_out_unsafe_constraints(
+                ireqs=results,
+                reverse_dependencies=self.reverse_dependencies(results),
+            )
 
         return results
 
@@ -445,3 +507,290 @@ def reverse_dependencies(
             ireq for ireq in ireqs if not (ireq.editable or is_url_requirement(ireq))
         ]
         return self.dependency_cache.reverse_dependencies(non_editable)
+
+
+class BacktrackingResolver(BaseResolver):
+    """A wrapper for backtracking resolver."""
+
+    def __init__(
+        self,
+        constraints: Iterable[InstallRequirement],
+        existing_constraints: Dict[str, InstallRequirement],
+        repository: BaseRepository,
+        allow_unsafe: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        self.constraints = list(constraints)
+        self.repository = repository
+        self.allow_unsafe = allow_unsafe
+
+        options = self.options = self.repository.options
+        self.session = self.repository.session
+        self.finder = self.repository.finder
+        self.command = self.repository.command
+        self.unsafe_constraints: Set[InstallRequirement] = set()
+
+        self.existing_constraints = existing_constraints
+        self._constraints_map = {key_from_ireq(ireq): ireq for ireq in constraints}
+
+        # Make sure there is no enabled legacy resolver
+        options.deprecated_features_enabled = remove_value(
+            options.deprecated_features_enabled, "legacy-resolver"
+        )
+
+    def resolve(self, max_rounds: int = 10) -> Set[InstallRequirement]:
+        """
+        Find concrete package versions for all the given InstallRequirements
+        and their recursive dependencies and return a set of pinned
+        ``InstallRequirement``'s.
+        """
+        with update_env_context_manager(
+            PIP_EXISTS_ACTION="i"
+        ), get_build_tracker() as build_tracker, global_tempdir_manager(), indent_log():
+            # Mark direct/primary/user_supplied packages
+            for ireq in self.constraints:
+                ireq.user_supplied = True
+
+            # Pass compiled requirements from `requirements.txt`
+            # as constraints to resolver
+            compatible_existing_constraints: Dict[str, InstallRequirement] = {}
+            for ireq in self.existing_constraints.values():
+                # Skip if the compiled install requirement conflicts with
+                # the primary install requirement.
+                primary_ireq = self._constraints_map.get(key_from_ireq(ireq))
+                if primary_ireq is not None:
+                    _, version, _ = as_tuple(ireq)
+                    prereleases = ireq.specifier.prereleases
+                    if not primary_ireq.specifier.contains(version, prereleases):
+                        continue
+
+                ireq.extras = set()  # pip does not support extras in constraints
+                ireq.constraint = True
+                ireq.user_supplied = False
+                compatible_existing_constraints[key_from_ireq(ireq)] = ireq
+
+            wheel_cache = WheelCache(
+                self.options.cache_dir, self.options.format_control
+            )
+
+            temp_dir = TempDirectory(
+                delete=not self.options.no_clean,
+                kind="resolve",
+                globally_managed=True,
+            )
+
+            preparer_kwargs = {
+                "temp_build_dir": temp_dir,
+                "options": self.options,
+                "session": self.session,
+                "finder": self.finder,
+                "use_user_site": False,
+            }
+
+            if PIP_VERSION[:2] <= (22, 0):
+                preparer_kwargs["req_tracker"] = build_tracker
+            else:
+                preparer_kwargs["build_tracker"] = build_tracker
+
+            preparer = self.command.make_requirement_preparer(**preparer_kwargs)
+
+            resolver = self.command.make_resolver(
+                preparer=preparer,
+                finder=self.finder,
+                options=self.options,
+                wheel_cache=wheel_cache,
+                use_user_site=False,
+                ignore_installed=True,
+                ignore_requires_python=False,
+                force_reinstall=False,
+                use_pep517=self.options.use_pep517,
+                upgrade_strategy="to-satisfy-only",
+            )
+
+            self.command.trace_basic_info(self.finder)
+
+            for current_round in count(start=1):  # pragma: no branch
+                if current_round > max_rounds:
+                    raise RuntimeError(
+                        "No stable configuration of concrete packages "
+                        "could be found for the given constraints after "
+                        f"{max_rounds} rounds of resolving.\n"
+                        "This is likely a bug."
+                    )
+
+                log.debug("")
+                log.debug(magenta(f"{f'ROUND {current_round}':^60}"))
+
+                is_resolved = self._do_resolve(
+                    resolver=resolver,
+                    compatible_existing_constraints=compatible_existing_constraints,
+                )
+                if is_resolved:
+                    break
+
+        resolver_result = resolver._result
+        assert isinstance(resolver_result, Result)
+
+        # Get reverse requirements from the resolver result graph.
+        reverse_dependencies = self._get_reverse_dependencies(resolver_result)
+
+        # Prepare set of install requirements from resolver result.
+        result_ireqs = self._get_install_requirements(
+            resolver_result=resolver_result,
+            reverse_dependencies=reverse_dependencies,
+        )
+
+        # Filter out unsafe requirements.
+        if not self.allow_unsafe:
+            self._filter_out_unsafe_constraints(
+                ireqs=result_ireqs,
+                reverse_dependencies=reverse_dependencies,
+            )
+
+        return result_ireqs
+
+    def _do_resolve(
+        self,
+        resolver: Resolver,
+        compatible_existing_constraints: Dict[str, InstallRequirement],
+    ) -> bool:
+        """
+        Return true on successful resolution, otherwise remove problematic
+        requirements from existing constraints and return false.
+        """
+        try:
+            resolver.resolve(
+                root_reqs=self.constraints
+                + list(compatible_existing_constraints.values()),
+                check_supported_wheels=not self.options.target_dir,
+            )
+        except DistributionNotFound as e:
+            cause_exc = e.__cause__
+            if cause_exc is None:
+                raise
+
+            if not isinstance(cause_exc, ResolutionImpossible):
+                raise
+
+            # Collect all incompatible install requirement names
+            cause_ireq_names = {
+                key_from_req(cause.requirement) for cause in cause_exc.causes
+            }
+
+            # Looks like resolution is impossible, try to fix
+            for cause_ireq_name in cause_ireq_names:
+                # Find the cause requirement in existing requirements,
+                # otherwise raise error
+                cause_existing_ireq = compatible_existing_constraints.get(
+                    cause_ireq_name
+                )
+                if cause_existing_ireq is None:
+                    raise
+
+                # Remove existing incompatible constraint that causes error
+                log.warning(
+                    f"Discarding {cause_existing_ireq} to proceed the resolution"
+                )
+                del compatible_existing_constraints[cause_ireq_name]
+
+            return False
+
+        return True
+
+    def _get_install_requirements(
+        self,
+        resolver_result: Result,
+        reverse_dependencies: Dict[str, Set[str]],
+    ) -> Set[InstallRequirement]:
+        """Return a set of install requirements from resolver results."""
+        result_ireqs: Dict[str, InstallRequirement] = {}
+
+        # Transform candidates to install requirements
+        resolved_candidates = tuple(resolver_result.mapping.values())
+        for candidate in resolved_candidates:
+            ireq = self._get_install_requirement_from_candidate(
+                candidate=candidate,
+                reverse_dependencies=reverse_dependencies,
+            )
+            if ireq is None:
+                continue
+
+            project_name = canonicalize_name(candidate.project_name)
+            result_ireqs[project_name] = ireq
+
+        # Merge extras to install requirements
+        extras_candidates = (
+            candidate
+            for candidate in resolved_candidates
+            if isinstance(candidate, ExtrasCandidate)
+        )
+        for extras_candidate in extras_candidates:
+            project_name = canonicalize_name(extras_candidate.project_name)
+            ireq = result_ireqs[project_name]
+            ireq.extras |= extras_candidate.extras
+            ireq.req.extras |= extras_candidate.extras
+
+        return set(result_ireqs.values())
+
+    @staticmethod
+    def _get_reverse_dependencies(
+        resolver_result: Result,
+    ) -> Dict[str, Set[str]]:
+        reverse_dependencies: DefaultDict[str, Set[str]] = collections.defaultdict(set)
+
+        for candidate in resolver_result.mapping.values():
+            stripped_name = strip_extras(canonicalize_name(candidate.name))
+
+            for parent_name in resolver_result.graph.iter_parents(candidate.name):
+                # Skip root dependency which is always None
+                if parent_name is None:
+                    continue
+
+                # Skip a dependency that equals to the candidate. This could be
+                # the dependency with extras.
+                stripped_parent_name = strip_extras(canonicalize_name(parent_name))
+                if stripped_name == stripped_parent_name:
+                    continue
+
+                reverse_dependencies[stripped_name].add(stripped_parent_name)
+
+        return dict(reverse_dependencies)
+
+    def _get_install_requirement_from_candidate(
+        self, candidate: Candidate, reverse_dependencies: Dict[str, Set[str]]
+    ) -> Optional[InstallRequirement]:
+        ireq = candidate.get_install_requirement()
+        if ireq is None:
+            return None
+
+        # Determine a pin operator
+        version_pin_operator = "=="
+        version_as_str = str(candidate.version)
+        for specifier in ireq.specifier:
+            if specifier.operator == "===" and specifier.version == version_as_str:
+                version_pin_operator = "==="
+                break
+
+        # Prepare pinned install requirement. Copy it from candidate's install
+        # requirement so that it could be mutated later.
+        pinned_ireq = copy_install_requirement(ireq)
+
+        # Canonicalize name
+        assert ireq.name is not None
+        pinned_ireq.req.name = canonicalize_name(ireq.name)
+
+        # Pin requirement to a resolved version
+        pinned_ireq.req.specifier = SpecifierSet(
+            f"{version_pin_operator}{candidate.version}"
+        )
+
+        # Save reverse dependencies for annotation
+        ireq_key = key_from_ireq(ireq)
+        pinned_ireq._required_by = reverse_dependencies.get(ireq_key, set())
+
+        # Save source for annotation
+        source_ireq = self._constraints_map.get(ireq_key)
+        if source_ireq is not None and ireq_key not in self.existing_constraints:
+            pinned_ireq._source_ireqs = [source_ireq]
+
+        return pinned_ireq
diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py
index 32b7d5668..0c98c6b2d 100755
--- a/piptools/scripts/compile.py
+++ b/piptools/scripts/compile.py
@@ -21,7 +21,7 @@
 from ..logging import log
 from ..repositories import LocalRequirementsRepository, PyPIRepository
 from ..repositories.base import BaseRepository
-from ..resolver import Resolver
+from ..resolver import BacktrackingResolver, LegacyResolver
 from ..utils import (
     UNSAFE_PACKAGES,
     dedup,
@@ -225,6 +225,14 @@ def _get_default_option(option_name: str) -> Any:
 @click.option(
     "--pip-args", "pip_args_str", help="Arguments to pass directly to the pip command."
 )
+@click.option(
+    "--resolver",
+    "resolver_name",
+    type=click.Choice(("legacy", "backtracking")),
+    default="legacy",
+    envvar="PIP_TOOLS_RESOLVER",
+    help="Choose the dependency resolver.",
+)
 @click.option(
     "--emit-index-url/--no-emit-index-url",
     is_flag=True,
@@ -268,6 +276,7 @@ def cli(
     emit_find_links: bool,
     cache_dir: str,
     pip_args_str: Optional[str],
+    resolver_name: str,
     emit_index_url: bool,
     emit_options: bool,
 ) -> None:
@@ -334,9 +343,10 @@ def cli(
         pip_args.extend(["--pre"])
     for host in trusted_host:
         pip_args.extend(["--trusted-host", host])
-
     if not build_isolation:
         pip_args.append("--no-build-isolation")
+    if resolver_name == "legacy":
+        pip_args.extend(["--use-deprecated", "legacy-resolver"])
     pip_args.extend(right_args)
 
     repository: BaseRepository
@@ -350,6 +360,10 @@ def cli(
 
     existing_pins_to_upgrade = set()
 
+    # Exclude packages from --upgrade-package/-P from the existing
+    # constraints, and separately gather pins to be upgraded
+    existing_pins = {}
+
     # Proxy with a LocalRequirementsRepository if --upgrade is not specified
     # (= default invocation)
     if not upgrade and os.path.exists(output_file.name):
@@ -363,9 +377,6 @@ def cli(
             options=tmp_repository.options,
         )
 
-        # Exclude packages from --upgrade-package/-P from the existing
-        # constraints, and separately gather pins to be upgraded
-        existing_pins = {}
         for ireq in filter(is_pinned_requirement, ireqs):
             key = key_from_ireq(ireq)
             if key in upgrade_install_reqs:
@@ -462,10 +473,12 @@ def cli(
             for find_link in dedup(repository.finder.find_links):
                 log.debug(redact_auth_from_url(find_link))
 
+    resolver_cls = LegacyResolver if resolver_name == "legacy" else BacktrackingResolver
     try:
-        resolver = Resolver(
-            constraints,
-            repository,
+        resolver = resolver_cls(
+            constraints=constraints,
+            existing_constraints=existing_pins,
+            repository=repository,
             prereleases=repository.finder.allow_all_prereleases or pre,
             cache=DependencyCache(cache_dir),
             clear_caches=rebuild,
diff --git a/piptools/utils.py b/piptools/utils.py
index ccd94721d..fe60be42d 100644
--- a/piptools/utils.py
+++ b/piptools/utils.py
@@ -3,6 +3,7 @@
 import itertools
 import json
 import os
+import re
 import shlex
 import typing
 from typing import (
@@ -441,3 +442,43 @@ def get_sys_path_for_python_executable(python_executable: str) -> List[str]:
     assert isinstance(paths, list)
     assert all(isinstance(i, str) for i in paths)
     return [os.path.abspath(path) for path in paths]
+
+
+def remove_value(lst: List[_T], value: _T) -> List[_T]:
+    """
+    Returns new list without a given value.
+    """
+    return [item for item in lst if item != value]
+
+
+_strip_extras_re = re.compile(r"\[.+?\]")
+
+
+def strip_extras(name: str) -> str:
+    """Strip extras from package name, e.g. pytest[testing] -> pytest."""
+    return re.sub(_strip_extras_re, "", name)
+
+
+def copy_install_requirement(template: InstallRequirement) -> InstallRequirement:
+    """Make a copy of an ``InstallRequirement``."""
+    ireq = InstallRequirement(
+        req=copy.deepcopy(template.req),
+        comes_from=template.comes_from,
+        editable=template.editable,
+        link=template.link,
+        markers=template.markers,
+        use_pep517=template.use_pep517,
+        isolated=template.isolated,
+        install_options=template.install_options,
+        global_options=template.global_options,
+        hash_options=template.hash_options,
+        constraint=template.constraint,
+        extras=template.extras,
+        user_supplied=template.user_supplied,
+    )
+
+    # If the original_link was None, keep it so. Passing `link` as an
+    # argument to `InstallRequirement` sets it as the original_link:
+    ireq.original_link = template.original_link
+
+    return ireq
diff --git a/piptools/writer.py b/piptools/writer.py
index a83817122..9489004bd 100644
--- a/piptools/writer.py
+++ b/piptools/writer.py
@@ -2,13 +2,25 @@
 import re
 import sys
 from itertools import chain
-from typing import BinaryIO, Dict, Iterable, Iterator, List, Optional, Set, Tuple
+from typing import (
+    BinaryIO,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+    cast,
+)
 
 from click import unstyle
 from click.core import Context
 from pip._internal.models.format_control import FormatControl
 from pip._internal.req.req_install import InstallRequirement
 from pip._vendor.packaging.markers import Marker
+from pip._vendor.packaging.utils import canonicalize_name
 
 from .logging import log
 from .utils import (
@@ -18,6 +30,7 @@
     format_requirement,
     get_compile_command,
     key_from_ireq,
+    strip_extras,
 )
 
 MESSAGE_UNHASHED_PACKAGE = comment(
@@ -45,10 +58,10 @@
 strip_comes_from_line_re = re.compile(r" \(line \d+\)$")
 
 
-def _comes_from_as_string(ireq: InstallRequirement) -> str:
-    if isinstance(ireq.comes_from, str):
-        return strip_comes_from_line_re.sub("", ireq.comes_from)
-    return key_from_ireq(ireq.comes_from)
+def _comes_from_as_string(comes_from: Union[str, InstallRequirement]) -> str:
+    if isinstance(comes_from, str):
+        return strip_comes_from_line_re.sub("", comes_from)
+    return cast(str, canonicalize_name(key_from_ireq(comes_from)))
 
 
 def annotation_style_split(required_by: Set[str]) -> str:
@@ -263,7 +276,7 @@ def _format_requirement(
 
         line = format_requirement(ireq, marker=marker, hashes=ireq_hashes)
         if self.strip_extras:
-            line = re.sub(r"\[.+?\]", "", line)
+            line = strip_extras(line)
 
         if not self.annotate:
             return line
@@ -272,12 +285,17 @@ def _format_requirement(
         required_by = set()
         if hasattr(ireq, "_source_ireqs"):
             required_by |= {
-                _comes_from_as_string(src_ireq)
+                _comes_from_as_string(src_ireq.comes_from)
                 for src_ireq in ireq._source_ireqs
                 if src_ireq.comes_from
             }
-        elif ireq.comes_from:
-            required_by.add(_comes_from_as_string(ireq))
+
+        if ireq.comes_from:
+            required_by.add(_comes_from_as_string(ireq.comes_from))
+
+        if hasattr(ireq, "_required_by"):
+            for name in ireq._required_by:
+                required_by.add(name)
 
         if required_by:
             if self.annotation_style == "split":
@@ -288,6 +306,8 @@ def _format_requirement(
                 sep = "\n    " if ireq_hashes else "  "
             else:  # pragma: no cover
                 raise ValueError("Invalid value for annotation style")
+            if self.strip_extras:
+                annotation = strip_extras(annotation)
             # 24 is one reasonable column size to use here, that we've used in the past
             lines = f"{line:24}{sep}{comment(annotation)}".splitlines()
             line = "\n".join(ln.rstrip() for ln in lines)
diff --git a/tests/conftest.py b/tests/conftest.py
index 8e868cccd..bee50792e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,14 +1,16 @@
 import json
-import optparse
 import os
 import subprocess
 import sys
 from contextlib import contextmanager
+from dataclasses import dataclass, field
 from functools import partial
 from textwrap import dedent
+from typing import List, Optional
 
 import pytest
 from click.testing import CliRunner
+from pip._internal.commands.install import InstallCommand
 from pip._internal.index.package_finder import PackageFinder
 from pip._internal.models.candidate import InstallationCandidate
 from pip._internal.network.session import PipSession
@@ -24,7 +26,7 @@
 from piptools.exceptions import NoCandidateFound
 from piptools.repositories import PyPIRepository
 from piptools.repositories.base import BaseRepository
-from piptools.resolver import Resolver
+from piptools.resolver import BacktrackingResolver, LegacyResolver
 from piptools.utils import (
     as_tuple,
     is_url_requirement,
@@ -37,8 +39,17 @@
 from .utils import looks_like_ci
 
 
+@dataclass
+class FakeOptions:
+    features_enabled: List[str] = field(default_factory=list)
+    deprecated_features_enabled: List[str] = field(default_factory=list)
+    target_dir: Optional[str] = None
+
+
 class FakeRepository(BaseRepository):
-    def __init__(self):
+    def __init__(self, options: FakeOptions):
+        self._options = options
+
         with open(os.path.join(TEST_DATA_PATH, "fake-index.json")) as f:
             self.index = json.load(f)
 
@@ -91,8 +102,8 @@ def allow_all_wheels(self):
         yield
 
     @property
-    def options(self) -> optparse.Values:
-        """Not used"""
+    def options(self):
+        return self._options
 
     @property
     def session(self) -> PipSession:
@@ -102,6 +113,10 @@ def session(self) -> PipSession:
     def finder(self) -> PackageFinder:
         """Not used"""
 
+    @property
+    def command(self) -> InstallCommand:
+        """Not used"""
+
 
 class FakeInstalledDistribution:
     def __init__(self, line, deps=None):
@@ -145,13 +160,20 @@ def fake_dist():
 
 @pytest.fixture
 def repository():
-    return FakeRepository()
+    return FakeRepository(
+        options=FakeOptions(deprecated_features_enabled=["legacy-resolver"])
+    )
 
 
 @pytest.fixture
 def pypi_repository(tmpdir):
     return PyPIRepository(
-        ["--index-url", PyPIRepository.DEFAULT_INDEX_URL],
+        [
+            "--index-url",
+            PyPIRepository.DEFAULT_INDEX_URL,
+            "--use-deprecated",
+            "legacy-resolver",
+        ],
         cache_dir=(tmpdir / "pypi-repo"),
     )
 
@@ -166,12 +188,27 @@ def resolver(depcache, repository):
     # TODO: It'd be nicer if Resolver instance could be set up and then
     #       use .resolve(...) on the specset, instead of passing it to
     #       the constructor like this (it's not reusable)
-    return partial(Resolver, repository=repository, cache=depcache)
+    return partial(
+        LegacyResolver, repository=repository, cache=depcache, existing_constraints={}
+    )
+
+
+@pytest.fixture
+def backtracking_resolver(depcache):
+    # TODO: It'd be nicer if Resolver instance could be set up and then
+    #       use .resolve(...) on the specset, instead of passing it to
+    #       the constructor like this (it's not reusable)
+    return partial(
+        BacktrackingResolver,
+        repository=FakeRepository(options=FakeOptions()),
+        cache=depcache,
+        existing_constraints={},
+    )
 
 
 @pytest.fixture
 def base_resolver(depcache):
-    return partial(Resolver, cache=depcache)
+    return partial(LegacyResolver, cache=depcache, existing_constraints={})
 
 
 @pytest.fixture
diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py
index 4e95f61a2..59e25b3ef 100644
--- a/tests/test_cli_compile.py
+++ b/tests/test_cli_compile.py
@@ -9,10 +9,30 @@
 from pip._internal.utils.urls import path_to_url
 
 from piptools.scripts.compile import cli
+from piptools.utils import COMPILE_EXCLUDE_OPTIONS
 
 from .constants import MINIMAL_WHEELS_PATH, PACKAGES_PATH
 
 
+@pytest.fixture(
+    autouse=True,
+    params=[
+        pytest.param("legacy", id="legacy resolver"),
+        pytest.param("backtracking", id="backtracking resolver"),
+    ],
+)
+def current_resolver(request, monkeypatch):
+    # Hide --resolver option from pip-compile header, so that we don't have to
+    # inject it every time to tests outputs.
+    exclude_options = COMPILE_EXCLUDE_OPTIONS | {"--resolver"}
+    monkeypatch.setattr("piptools.utils.COMPILE_EXCLUDE_OPTIONS", exclude_options)
+
+    # Setup given resolver name
+    resolver_name = request.param
+    monkeypatch.setenv("PIP_TOOLS_RESOLVER", resolver_name)
+    return resolver_name
+
+
 @pytest.fixture(autouse=True)
 def _temp_dep_cache(tmpdir, monkeypatch):
     monkeypatch.setenv("PIP_TOOLS_CACHE_DIR", str(tmpdir / "cache"))
@@ -426,6 +446,14 @@ def test_editable_package_constraint_without_non_editable_duplicate(pip_conf, ru
     piptools keeps editable constraint,
     without also adding a duplicate "non-editable" requirement variation
     """
+
+    if current_resolver != "legacy":
+        pytest.skip(
+            "This test is actual only for legacy resolver. "
+            "Constraints refactored in 2020 resolver. "
+            "See https://github.com/pypa/pip/issues/9020 for details."
+        )
+
     fake_package_dir = os.path.join(PACKAGES_PATH, "small_fake_a")
     fake_package_dir = path_to_url(fake_package_dir)
     with open("constraints.txt", "w") as constraints:
@@ -451,6 +479,13 @@ def test_editable_package_in_constraints(pip_conf, runner, req_editable):
     piptools can compile an editable that appears in both primary requirements
     and constraints
     """
+    if current_resolver != "legacy":
+        pytest.skip(
+            "This test is actual only for legacy resolver. "
+            "Constraints refactored in 2020 resolver. "
+            "See https://github.com/pypa/pip/issues/9020 for details."
+        )
+
     fake_package_dir = os.path.join(PACKAGES_PATH, "small_fake_with_deps")
     fake_package_dir = path_to_url(fake_package_dir)
 
@@ -489,6 +524,9 @@ def test_locally_available_editable_package_is_not_archived_in_cache_dir(
     """
     piptools will not create an archive for a locally available editable requirement
     """
+    if current_resolver != "legacy":
+        pytest.skip("Test relevant to legacy resolver.")
+
     cache_dir = tmpdir.mkdir("cache_dir")
 
     fake_package_dir = os.path.join(PACKAGES_PATH, "small_fake_with_deps")
@@ -649,7 +687,7 @@ def test_relative_file_uri_package(pip_conf, runner):
 def test_direct_reference_with_extras(runner):
     with open("requirements.in", "w") as req_in:
         req_in.write(
-            "piptools[testing,coverage] @ git+https://github.com/jazzband/pip-tools@6.2.0"
+            "pip-tools[testing,coverage] @ git+https://github.com/jazzband/pip-tools@6.2.0"
         )
     out = runner.invoke(cli, ["-n", "--rebuild"])
     assert out.exit_code == 0
@@ -1022,6 +1060,9 @@ def test_bad_setup_file(runner):
 
 
 def test_no_candidates(pip_conf, runner):
+    if current_resolver != "legacy":
+        pytest.skip("Only legacy resolver throws this errors.")
+
     with open("requirements", "w") as req_in:
         req_in.write("small-fake-a>0.3b1,<0.3b2")
 
@@ -1032,6 +1073,9 @@ def test_no_candidates(pip_conf, runner):
 
 
 def test_no_candidates_pre(pip_conf, runner):
+    if current_resolver != "legacy":
+        pytest.skip("Only legacy resolver throws this errors.")
+
     with open("requirements", "w") as req_in:
         req_in.write("small-fake-a>0.3b1,<0.3b1")
 
@@ -1207,30 +1251,73 @@ def test_annotate_option(pip_conf, runner, options, expected):
 
     out = runner.invoke(cli, [*options, "-n", "--no-emit-find-links"])
 
+    assert out.exit_code == 0, out
     assert out.stderr == dedent(expected)
-    assert out.exit_code == 0
 
 
 @pytest.mark.parametrize(
     ("option", "expected"),
     (
-        ("--allow-unsafe", "small-fake-a==0.1"),
-        ("--no-allow-unsafe", "# small-fake-a"),
-        (None, "# small-fake-a"),
+        pytest.param(
+            "--allow-unsafe",
+            dedent(
+                """\
+                small-fake-a==0.1
+                small-fake-b==0.3
+                small-fake-with-deps==0.1
+                """
+            ),
+            id="allow all packages",
+        ),
+        pytest.param(
+            "--no-allow-unsafe",
+            dedent(
+                """\
+                small-fake-b==0.3
+
+                # The following packages are considered to be unsafe in a requirements file:
+                # small-fake-a
+                # small-fake-with-deps
+                """
+            ),
+            id="comment out small-fake-with-deps and its dependencies",
+        ),
+        pytest.param(
+            None,
+            dedent(
+                """\
+                small-fake-b==0.3
+
+                # The following packages are considered to be unsafe in a requirements file:
+                # small-fake-a
+                # small-fake-with-deps
+                """
+            ),
+            id="allow unsafe is default option",
+        ),
     ),
 )
 def test_allow_unsafe_option(pip_conf, monkeypatch, runner, option, expected):
     """
     Unsafe packages are printed as expected with and without --allow-unsafe.
     """
-    monkeypatch.setattr("piptools.resolver.UNSAFE_PACKAGES", {"small-fake-a"})
+    monkeypatch.setattr("piptools.resolver.UNSAFE_PACKAGES", {"small-fake-with-deps"})
     with open("requirements.in", "w") as req_in:
-        req_in.write(path_to_url(os.path.join(PACKAGES_PATH, "small_fake_with_deps")))
+        req_in.write("small-fake-b\n")
+        req_in.write("small-fake-with-deps")
 
-    out = runner.invoke(cli, ["--no-annotate", option] if option else [])
+    out = runner.invoke(
+        cli,
+        [
+            "--no-header",
+            "--no-emit-options",
+            "--no-annotate",
+            *([option] if option else []),
+        ],
+    )
 
-    assert expected in out.stderr.splitlines()
-    assert out.exit_code == 0
+    assert out.exit_code == 0, out
+    assert out.stderr == expected
 
 
 @pytest.mark.parametrize(
@@ -1491,6 +1578,9 @@ def test_unreachable_index_urls(runner, cli_options, expected_message):
     """
     Test pip-compile raises an error if index URLs are not reachable.
     """
+    if current_resolver != "legacy":
+        pytest.skip("Only legacy resolver throws this errors.")
+
     with open("requirements.in", "w") as reqs_in:
         reqs_in.write("some-package")
 
@@ -2138,3 +2228,23 @@ def test_cli_compile_strip_extras(runner, make_package, make_sdist, tmpdir):
     assert out.exit_code == 0, out
     assert "test-package-2==0.1" in out.stderr
     assert "[more]" not in out.stderr
+
+
+def test_resolution_failure(runner):
+    """Test resolution impossible for unknown package."""
+    with open("requirements.in", "w") as reqs_out:
+        reqs_out.write("unknown-package")
+
+    out = runner.invoke(cli)
+
+    assert out.exit_code != 0, out
+
+
+def test_resolver_reaches_max_rounds(runner):
+    """Test resolver reched max rounds and raises error."""
+    with open("requirements.in", "w") as reqs_out:
+        reqs_out.write("six")
+
+    out = runner.invoke(cli, ["--max-rounds", 0])
+
+    assert out.exit_code != 0, out
diff --git a/tests/test_resolver.py b/tests/test_resolver.py
index a9beca183..0e9b2fef1 100644
--- a/tests/test_resolver.py
+++ b/tests/test_resolver.py
@@ -1,4 +1,5 @@
 import pytest
+from pip._internal.exceptions import DistributionNotFound
 from pip._internal.utils.urls import path_to_url
 
 from piptools.exceptions import NoCandidateFound
@@ -500,3 +501,28 @@ def test_requirement_summary_with_other_objects(from_line):
     requirement_summary = RequirementSummary(from_line("test_package==1.2.3"))
     other_object = object()
     assert requirement_summary != other_object
+
+
+@pytest.mark.parametrize(
+    ("exception", "cause"),
+    (
+        pytest.param(DistributionNotFound, None, id="without cause"),
+        pytest.param(DistributionNotFound, ZeroDivisionError, id="with cause"),
+    ),
+)
+def test_catch_distribution_not_found_error(backtracking_resolver, exception, cause):
+    """
+    Test internal edge-cases when backtracking resolver catches
+    and re-raises ``DistributionNotFound`` error with/without causes.
+    """
+    resolver = backtracking_resolver([])
+
+    class FakePipResolver:
+        def resolve(self, *args, **kwargs):
+            raise exception from cause
+
+    with pytest.raises(DistributionNotFound):
+        resolver._do_resolve(
+            resolver=FakePipResolver(),
+            compatible_existing_constraints={},
+        )
diff --git a/tests/test_top_level_editable.py b/tests/test_top_level_editable.py
new file mode 100644
index 000000000..e69de29bb

From 57726d8ff9ebd5a694d80f5656e97c093222a9c6 Mon Sep 17 00:00:00 2001
From: Albert Tugushev <albert@tugushev.ru>
Date: Sat, 19 Feb 2022 20:30:04 +0700
Subject: [PATCH 2/6] Use legacy_resolver_only decorator

Co-Authored-By: Sviatoslav Sydorenko <wk.cvs.github@sydorenko.org.ua>
---
 tests/test_cli_compile.py | 39 ++++++++++++---------------------------
 1 file changed, 12 insertions(+), 27 deletions(-)

diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py
index 59e25b3ef..4171a53a1 100644
--- a/tests/test_cli_compile.py
+++ b/tests/test_cli_compile.py
@@ -13,6 +13,12 @@
 
 from .constants import MINIMAL_WHEELS_PATH, PACKAGES_PATH
 
+legacy_resolver_only = pytest.mark.parametrize(
+    "current_resolver",
+    ("legacy",),
+    indirect=("current_resolver",),
+)
+
 
 @pytest.fixture(
     autouse=True,
@@ -441,19 +447,12 @@ def test_editable_package_without_non_editable_duplicate(pip_conf, runner):
     assert "small-fake-a==" not in out.stderr
 
 
+@legacy_resolver_only
 def test_editable_package_constraint_without_non_editable_duplicate(pip_conf, runner):
     """
     piptools keeps editable constraint,
     without also adding a duplicate "non-editable" requirement variation
     """
-
-    if current_resolver != "legacy":
-        pytest.skip(
-            "This test is actual only for legacy resolver. "
-            "Constraints refactored in 2020 resolver. "
-            "See https://github.com/pypa/pip/issues/9020 for details."
-        )
-
     fake_package_dir = os.path.join(PACKAGES_PATH, "small_fake_a")
     fake_package_dir = path_to_url(fake_package_dir)
     with open("constraints.txt", "w") as constraints:
@@ -473,19 +472,13 @@ def test_editable_package_constraint_without_non_editable_duplicate(pip_conf, ru
     assert "small-fake-a==" not in out.stderr
 
 
+@legacy_resolver_only
 @pytest.mark.parametrize("req_editable", ((True,), (False,)))
 def test_editable_package_in_constraints(pip_conf, runner, req_editable):
     """
     piptools can compile an editable that appears in both primary requirements
     and constraints
     """
-    if current_resolver != "legacy":
-        pytest.skip(
-            "This test is actual only for legacy resolver. "
-            "Constraints refactored in 2020 resolver. "
-            "See https://github.com/pypa/pip/issues/9020 for details."
-        )
-
     fake_package_dir = os.path.join(PACKAGES_PATH, "small_fake_with_deps")
     fake_package_dir = path_to_url(fake_package_dir)
 
@@ -518,15 +511,13 @@ def test_editable_package_vcs(runner):
     assert "click" in out.stderr  # dependency of pip-tools
 
 
+@legacy_resolver_only
 def test_locally_available_editable_package_is_not_archived_in_cache_dir(
     pip_conf, tmpdir, runner
 ):
     """
     piptools will not create an archive for a locally available editable requirement
     """
-    if current_resolver != "legacy":
-        pytest.skip("Test relevant to legacy resolver.")
-
     cache_dir = tmpdir.mkdir("cache_dir")
 
     fake_package_dir = os.path.join(PACKAGES_PATH, "small_fake_with_deps")
@@ -1059,10 +1050,8 @@ def test_bad_setup_file(runner):
     assert f"Failed to parse {os.path.abspath('setup.py')}" in out.stderr
 
 
+@legacy_resolver_only
 def test_no_candidates(pip_conf, runner):
-    if current_resolver != "legacy":
-        pytest.skip("Only legacy resolver throws this errors.")
-
     with open("requirements", "w") as req_in:
         req_in.write("small-fake-a>0.3b1,<0.3b2")
 
@@ -1072,10 +1061,8 @@ def test_no_candidates(pip_conf, runner):
     assert "Skipped pre-versions:" in out.stderr
 
 
+@legacy_resolver_only
 def test_no_candidates_pre(pip_conf, runner):
-    if current_resolver != "legacy":
-        pytest.skip("Only legacy resolver throws this errors.")
-
     with open("requirements", "w") as req_in:
         req_in.write("small-fake-a>0.3b1,<0.3b1")
 
@@ -1574,13 +1561,11 @@ def test_options_in_requirements_file(runner, options):
         ),
     ),
 )
+@legacy_resolver_only
 def test_unreachable_index_urls(runner, cli_options, expected_message):
     """
     Test pip-compile raises an error if index URLs are not reachable.
     """
-    if current_resolver != "legacy":
-        pytest.skip("Only legacy resolver throws this errors.")
-
     with open("requirements.in", "w") as reqs_in:
         reqs_in.write("some-package")
 

From cd9b40d01ffb7588b897a3eed759ce6a724939b8 Mon Sep 17 00:00:00 2001
From: Albert Tugushev <albert@tugushev.ru>
Date: Sat, 19 Feb 2022 20:57:01 +0700
Subject: [PATCH 3/6] Rename remove_value() -> omit_list_value()

Co-Authored-By: Sviatoslav Sydorenko <wk.cvs.github@sydorenko.org.ua>
---
 piptools/resolver.py | 6 +++---
 piptools/utils.py    | 6 ++----
 piptools/writer.py   | 4 +---
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/piptools/resolver.py b/piptools/resolver.py
index 61a287cf2..19fbd82b8 100644
--- a/piptools/resolver.py
+++ b/piptools/resolver.py
@@ -46,7 +46,7 @@
     is_url_requirement,
     key_from_ireq,
     key_from_req,
-    remove_value,
+    omit_list_value,
     strip_extras,
 )
 
@@ -236,7 +236,7 @@ def __init__(
             raise PipToolsError("Legacy resolver deprecated feature must be enabled.")
 
         # Make sure there is no enabled 2020-resolver
-        options.features_enabled = remove_value(
+        options.features_enabled = omit_list_value(
             options.features_enabled, "2020-resolver"
         )
 
@@ -534,7 +534,7 @@ def __init__(
         self._constraints_map = {key_from_ireq(ireq): ireq for ireq in constraints}
 
         # Make sure there is no enabled legacy resolver
-        options.deprecated_features_enabled = remove_value(
+        options.deprecated_features_enabled = omit_list_value(
             options.deprecated_features_enabled, "legacy-resolver"
         )
 
diff --git a/piptools/utils.py b/piptools/utils.py
index fe60be42d..7be3e5289 100644
--- a/piptools/utils.py
+++ b/piptools/utils.py
@@ -444,10 +444,8 @@ def get_sys_path_for_python_executable(python_executable: str) -> List[str]:
     return [os.path.abspath(path) for path in paths]
 
 
-def remove_value(lst: List[_T], value: _T) -> List[_T]:
-    """
-    Returns new list without a given value.
-    """
+def omit_list_value(lst: List[_T], value: _T) -> List[_T]:
+    """Produce a new list with a given value skipped."""
     return [item for item in lst if item != value]
 
 
diff --git a/piptools/writer.py b/piptools/writer.py
index 9489004bd..6ab95f8ce 100644
--- a/piptools/writer.py
+++ b/piptools/writer.py
@@ -293,9 +293,7 @@ def _format_requirement(
         if ireq.comes_from:
             required_by.add(_comes_from_as_string(ireq.comes_from))
 
-        if hasattr(ireq, "_required_by"):
-            for name in ireq._required_by:
-                required_by.add(name)
+        required_by |= set(getattr(ireq, "_required_by", set()))
 
         if required_by:
             if self.annotation_style == "split":

From 059a271525621c7960dcc9b3f0c19353f27ca441 Mon Sep 17 00:00:00 2001
From: Andy Kluger <AndyKluger@gmail.com>
Date: Tue, 26 Apr 2022 18:48:57 -0400
Subject: [PATCH 4/6] Add test: "ignore incompatible existing pins"

It shouldn't raise an error when a preexisting output file's
pins conflict with the input files' constraints.

https://github.com/jazzband/pip-tools/pull/1539#issuecomment-1110091988
---
 tests/test_cli_compile.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py
index 4171a53a1..c34f15b80 100644
--- a/tests/test_cli_compile.py
+++ b/tests/test_cli_compile.py
@@ -702,6 +702,20 @@ def test_input_file_without_extension(pip_conf, runner):
     assert os.path.exists("requirements.txt")
 
 
+def test_ignore_incompatible_existing_pins(pip_conf, runner):
+    """
+    Successfully compile when existing output pins conflict with input.
+    """
+    with open("requirements.in", "w") as req_in:
+        req_in.write("small-fake-b>0.1")
+    with open("requirements.txt", "w") as req_in:
+        req_in.write("small-fake-b==0.1")
+
+    out = runner.invoke(cli, [])
+
+    assert out.exit_code == 0
+
+
 def test_upgrade_packages_option(pip_conf, runner):
     """
     piptools respects --upgrade-package/-P inline list.

From 7d5caa975d0653fe7622b991f20d127e95dfadb0 Mon Sep 17 00:00:00 2001
From: Andy Kluger <AndyKluger@gmail.com>
Date: Thu, 28 Apr 2022 11:10:43 -0400
Subject: [PATCH 5/6] Amend test_ignore_incompatible_existing_pins to cover
 subdeps

https://github.com/jazzband/pip-tools/pull/1539#issuecomment-1112037387
---
 tests/test_cli_compile.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py
index c34f15b80..359ce3f1a 100644
--- a/tests/test_cli_compile.py
+++ b/tests/test_cli_compile.py
@@ -706,10 +706,10 @@ def test_ignore_incompatible_existing_pins(pip_conf, runner):
     """
     Successfully compile when existing output pins conflict with input.
     """
+    with open("requirements.txt", "w") as req_txt:
+        req_txt.write("small-fake-a==0.2\nsmall-fake-b==0.2")
     with open("requirements.in", "w") as req_in:
-        req_in.write("small-fake-b>0.1")
-    with open("requirements.txt", "w") as req_in:
-        req_in.write("small-fake-b==0.1")
+        req_in.write("small-fake-with-deps\nsmall-fake-b<0.2")
 
     out = runner.invoke(cli, [])
 

From c5e6ce26f6265de321ad00ec3547b04b8a48b016 Mon Sep 17 00:00:00 2001
From: Albert Tugushev <albert@tugushev.ru>
Date: Tue, 28 Jun 2022 22:57:27 +0200
Subject: [PATCH 6/6] Delegate to template-copying logic when combining install
 reqs

Co-authored-by: Richard Frank <rich@quantopian.com>
---
 piptools/resolver.py | 16 +++----------
 piptools/utils.py    | 54 ++++++++++++++++++++++++++++----------------
 2 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/piptools/resolver.py b/piptools/resolver.py
index 19fbd82b8..f9d3d1250 100644
--- a/piptools/resolver.py
+++ b/piptools/resolver.py
@@ -139,24 +139,14 @@ def combine_install_requirements(
             key=lambda x: (len(str(x)), str(x)),
         )
 
-    combined_ireq = InstallRequirement(
+    combined_ireq = copy_install_requirement(
+        template=source_ireqs[0],
         req=req,
         comes_from=comes_from,
-        editable=source_ireqs[0].editable,
-        link=link_attrs["link"],
-        markers=source_ireqs[0].markers,
-        use_pep517=source_ireqs[0].use_pep517,
-        isolated=source_ireqs[0].isolated,
-        install_options=source_ireqs[0].install_options,
-        global_options=source_ireqs[0].global_options,
-        hash_options=source_ireqs[0].hash_options,
         constraint=constraint,
         extras=extras,
-        user_supplied=source_ireqs[0].user_supplied,
+        **link_attrs,
     )
-    # e.g. If the original_link was None, keep it so. Passing `link` as an
-    # argument to `InstallRequirement` sets it as the original_link:
-    combined_ireq.original_link = link_attrs["original_link"]
     combined_ireq._source_ireqs = source_ireqs
 
     return combined_ireq
diff --git a/piptools/utils.py b/piptools/utils.py
index 7be3e5289..47f363751 100644
--- a/piptools/utils.py
+++ b/piptools/utils.py
@@ -7,6 +7,7 @@
 import shlex
 import typing
 from typing import (
+    Any,
     Callable,
     Dict,
     Iterable,
@@ -457,26 +458,41 @@ def strip_extras(name: str) -> str:
     return re.sub(_strip_extras_re, "", name)
 
 
-def copy_install_requirement(template: InstallRequirement) -> InstallRequirement:
-    """Make a copy of an ``InstallRequirement``."""
-    ireq = InstallRequirement(
-        req=copy.deepcopy(template.req),
-        comes_from=template.comes_from,
-        editable=template.editable,
-        link=template.link,
-        markers=template.markers,
-        use_pep517=template.use_pep517,
-        isolated=template.isolated,
-        install_options=template.install_options,
-        global_options=template.global_options,
-        hash_options=template.hash_options,
-        constraint=template.constraint,
-        extras=template.extras,
-        user_supplied=template.user_supplied,
-    )
+def copy_install_requirement(
+    template: InstallRequirement, **extra_kwargs: Any
+) -> InstallRequirement:
+    """Make a copy of a template ``InstallRequirement`` with extra kwargs."""
+    # Prepare install requirement kwargs.
+    kwargs = {
+        "comes_from": template.comes_from,
+        "editable": template.editable,
+        "link": template.link,
+        "markers": template.markers,
+        "use_pep517": template.use_pep517,
+        "isolated": template.isolated,
+        "install_options": template.install_options,
+        "global_options": template.global_options,
+        "hash_options": template.hash_options,
+        "constraint": template.constraint,
+        "extras": template.extras,
+        "user_supplied": template.user_supplied,
+    }
+    kwargs.update(extra_kwargs)
+
+    # Original link does not belong to install requirements constructor,
+    # pop it now to update later.
+    original_link = kwargs.pop("original_link", None)
+
+    # Copy template.req if not specified in extra kwargs.
+    if "req" not in kwargs:
+        kwargs["req"] = copy.deepcopy(template.req)
+
+    ireq = InstallRequirement(**kwargs)
 
     # If the original_link was None, keep it so. Passing `link` as an
-    # argument to `InstallRequirement` sets it as the original_link:
-    ireq.original_link = template.original_link
+    # argument to `InstallRequirement` sets it as the original_link.
+    ireq.original_link = (
+        template.original_link if original_link is None else original_link
+    )
 
     return ireq