python3Packages.bootstrapped-build: to replace pip

pkgs/development/interpreters/python/hooks/build-build-hook.sh

@@ -0,0 +1,20 @@
+# Setup hook to build Python projects using bootstrapped-build
+echo "Sourcing build-build-hook"
+
+pipBuildPhase() {
+    echo "Executing buildBuildPhase"
+    runHook preBuild
+
+    mkdir -p dist
+    echo "Building a wheel..."
+    @pythonInterpreter@ -m build --wheel --no-isolation --wheel-dir dist .
+    echo "Finished building a wheel..."
+
+    runHook postBuild
+    echo "Finished executing buildBuildPhase"
+}
+
+if [ -z "${dontUseBuildBuild-}" ] && [ -z "${buildPhase-}" ]; then
+    echo "Using buildBuildPhase"
+    buildPhase=buildBuildPhase
+fi

pkgs/development/interpreters/python/hooks/default.nix

@@ -53,6 +53,24 @@ in {
       };
     } ./flit-build-hook.sh) {};
 
+  buildBuildHook = callPackage ({ makePythonHook, bootstrapped-build }:
+    makePythonHook {
+      name = "build-build-hook.sh";
+      deps = [ bootstrapped-build ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./build-build-hook.sh) {};
+
+  installerInstallHook = callPackage ({ makePythonHook, bootstrapped-build }:
+    makePythonHook {
+      name = "build-build-hook.sh";
+      deps = [ bootstrapped-build ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./build-build-hook.sh) {};
+
   pipBuildHook = callPackage ({ makePythonHook, pip, wheel }:
     makePythonHook {
       name = "pip-build-hook.sh";

pkgs/development/interpreters/python/hooks/installer-install-hook.sh

@@ -0,0 +1,24 @@
+# Setup hook for installing wheels using installer
+echo "Sourcing installer-install-hook"
+
+declare -a installerInstallFlags
+
+installerInstallPhase() {
+    echo "Executing installerInstallPhase"
+    runHook preInstall
+
+    mkdir -p "$out/@pythonSitePackages@"
+    export PYTHONPATH="$out/@pythonSitePackages@:$PYTHONPATH"
+
+    pushd dist || return 1
+    @pythonInterpreter@ -m install_python_wheel ./*.whl --prefix "$out" --prefix-to-remove "${python.pythonForBuild}" --compile-bytecode "0"
+    popd || return 1
+
+    runHook postInstall
+    echo "Finished executing installerInstallPhase"
+}
+
+if [ -z "${dontUseInstallerInstall-}" ] && [ -z "${installPhase-}" ]; then
+    echo "Using installerInstallPhase"
+    installPhase=installerInstallPhase
+fi

pkgs/development/python-modules/bootstrapped-build/default.nix

@@ -0,0 +1,110 @@
+{ stdenv
+, lib
+, python
+, requiredPythonModules
+, unzip
+, wrapPython
+
+# Bootstrapping deps
+, flit-core
+, pep517
+, tomli
+, pyparsing
+, six
+, installer
+, wheel
+, setuptools
+, packaging
+, build
+}:
+
+let
+  runtimeDeps = [
+    build
+    setuptools
+    wheel
+    flit-core
+    packaging
+    pep517
+    tomli
+    pyparsing
+    six
+    installer
+  ];
+  # runtimeDeps = lib.remove python (requiredPythonModules [ build installer setuptools wheel ]);
+  data = map (drv: lib.getAttrs ["pname" "src" ] drv) runtimeDeps;
+  # json = builtins.toFile "instructions.json" (builtins.toJSON data);
+  srcs = with lib; catAttrs "src" runtimeDeps;
+in stdenv.mkDerivation rec {
+  name = "${python.libPrefix}-bootstrapped-build-installer";
+
+  inherit srcs;
+  sourceRoot = ".";
+
+  postUnpack = ''
+    mv pep517* pep517
+    mv flit*/flit_core flit_core
+    rm -rf flit*source
+    mv tomli* tomli
+    mv pyparsing* pyparsing
+    mv six* six
+    mv installer* installer
+    mv wheel* wheel
+    mv setuptools* setuptools
+    mv packaging* packaging
+    mv build* build
+    for folder in ./*; do
+        echo $folder
+        export PYTHONPATH="$(pwd)/$folder:$PYTHONPATH"
+    done;
+    export PYTHONPATH="$(pwd)/tomli/src:$(pwd)/installer/src:$(pwd)/wheel/src:$(pwd)/build/src:$(pwd)/setuptools/pkg_resources:$PYTHONPATH"
+
+  '';
+
+  strictDeps = true;
+
+  nativeBuildInputs = [
+    unzip
+    wrapPython
+    python
+  ];
+
+  passthru.bootstrapPackages = srcs;
+
+  buildPhase = ''
+    runHook preBuild
+
+    # To get bdist_wheel we need to create an sdist first for some reason
+    echo "Building wheel sdist"
+    ${python.pythonForBuild.interpreter} -m build --sdist "wheel" --no-isolation --skip-dependency-check
+
+    # Build wheels for all packages
+    for pkg in ./*; do
+        if [[ -d $pkg ]]; then
+          echo "Building $pkg wheel"
+          ${python.pythonForBuild.interpreter} -m build --wheel "$pkg" --no-isolation --skip-dependency-check
+        fi
+    done;
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+    # Install our custom install script
+    mkdir -p "$out/${python.sitePackages}/install_python_wheel"
+    cp ${./install_python_wheel.py} "$out/${python.sitePackages}/install_python_wheel/__main__.py"
+
+    export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
+
+    for whl in $(find -name "*.whl"); do
+        ${python.pythonForBuild.interpreter} -m install_python_wheel "$whl" --prefix "$out" --prefix-to-remove "${python.pythonForBuild}" --compile-bytecode "0"
+    done;
+
+    runHook postInstall
+  '';
+
+  postFixup = ''
+    wrapPythonPrograms
+  '';
+}

pkgs/development/python-modules/bootstrapped-build/install_python_wheel.py

@@ -0,0 +1,109 @@
+
+"""Script for installing Python packages using the installer API when a prefix is needed.
+
+Based on https://github.com/pradyunsg/installer/commit/777a499d5a351c07a631147c96e41db8cdfdbcfc.
+"""
+
+import argparse
+import os.path
+import sys
+import sysconfig
+from typing import Dict, Optional, Sequence
+
+import installer
+import installer.destinations
+import installer.sources
+import installer.utils
+
+
+def main_parser() -> argparse.ArgumentParser:
+    """Construct the main parser."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("wheel", type=str, help="wheel file to install")
+    parser.add_argument(
+        "--destdir",
+        "-d",
+        metavar="path",
+        type=str,
+        help="destination directory",
+    )
+    parser.add_argument(
+        "--prefix",
+        metavar="path",
+        type=str,
+        help="prefix directory",
+    )
+    parser.add_argument(
+        "--prefix-to-remove",
+        metavar="path",
+        type=str,
+        help="prefix directory to remove",
+    )
+    parser.add_argument(
+        "--compile-bytecode",
+        action="append",
+        metavar="level",
+        type=int,
+        choices=[0, 1, 2],
+        help="generate bytecode for the specified optimization level(s) (default=0, 1)",
+    )
+    parser.add_argument(
+        "--no-compile-bytecode",
+        action="store_true",
+        help="don't generate bytecode for installed modules",
+    )
+    return parser
+
+
+def get_scheme_dict(distribution_name: str) -> Dict[str, str]:
+    """Calculate the scheme dictionary for the current Python environment."""
+    scheme_dict = sysconfig.get_paths()
+
+    # calculate 'headers' path, not currently in sysconfig - see
+    # https://bugs.python.org/issue44445. This is based on what distutils does.
+    # TODO: figure out original vs normalised distribution names
+    scheme_dict["headers"] = os.path.join(
+        sysconfig.get_path(
+            "include", vars={"installed_base": sysconfig.get_config_var("base")}
+        ),
+        distribution_name,
+    )
+
+    return scheme_dict
+
+
+def prefix_scheme_dict(scheme: Dict[str, str], prefix: str, prefix_to_remove: str):
+    """Replace the existing prefix with the new prefix"""
+    # Cannot use os.path.commonpath because there can be items in the scheme that
+    # consist of only the prefix and nothing more.
+    replace_prefix = lambda path: path.replace(prefix_to_remove, prefix)
+    new_scheme = {key: replace_prefix(path) for key, path in scheme.items()}
+    return new_scheme
+
+
+def main(cli_args: Sequence[str], program: Optional[str] = None) -> None:
+    """Process arguments and perform the install."""
+    parser = main_parser()
+    if program:
+        parser.prog = program
+    args = parser.parse_args(cli_args)
+
+    bytecode_levels = args.compile_bytecode
+    if args.no_compile_bytecode:
+        bytecode_levels = []
+    elif not bytecode_levels:
+        bytecode_levels = [0, 1]
+
+    with installer.sources.WheelFile.open(args.wheel) as source:
+        destination = installer.destinations.SchemeDictionaryDestination(
+            prefix_scheme_dict(get_scheme_dict(source.distribution), args.prefix, args.prefix_to_remove),
+            sys.executable,
+            installer.utils.get_launcher_kind(),
+            bytecode_optimization_levels=bytecode_levels,
+            destdir=args.destdir,
+        )
+        installer.install(source, destination, {})
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main(sys.argv[1:], "python -m installer")

pkgs/development/python-modules/build/default.nix

@@ -28,6 +28,7 @@ buildPythonPackage rec {
     repo = pname;
     rev = version;
     hash = "sha256-P0DFBYsL2Ce/JwfYss64+CY/IvzYZEiz9wuEslij+oU=";
+    name = "${pname}-${version}-source";
   };
 
   nativeBuildInputs = [

pkgs/development/python-modules/flit-core/default.nix

@@ -4,6 +4,9 @@
 , flit
 }:
 
+# This is a package for bootstrapping the Python packages set.
+# What is passed to `src` needs to have already been unpacked.
+
 buildPythonPackage rec {
   pname = "flit-core";
   inherit (flit) version;

pkgs/development/python-modules/flit/default.nix

@@ -26,6 +26,7 @@ buildPythonPackage rec {
     repo = "flit";
     rev = version;
     sha256 = "sha256-zKgaeK3fskz2TuHvIWlxBrdZIWfIJHhaqopZ3+V36wY=";
+    name = "${pname}-${version}-source";
   };
 
   nativeBuildInputs = [

pkgs/development/python-modules/installer/default.nix

@@ -17,6 +17,7 @@ buildPythonPackage rec {
     repo = pname;
     rev = version;
     sha256 = "sha256-vhZYUhUcD5fnjkyEqFMvggVGH9Ri8iNgqRgSBQTOCtM=";
+    name = "${pname}-${version}-source";
   };
 
   nativeBuildInputs = [ flit-core ];

pkgs/development/python-modules/pyparsing/default.nix

@@ -18,6 +18,7 @@ let
       repo = pname;
       rev = "pyparsing_${version}";
       sha256 = "sha256-aCRyJQyLf8qQ6NO41q+HC856TjIHzIt0vyVBLV+3teE=";
+      name = "${pname}-${version}-source";
     };
 
     nativeBuildInputs = [

pkgs/development/python-modules/setuptools/default.nix

@@ -41,7 +41,8 @@ let
 
     installPhase = ''
       echo "Moving sdist..."
-      mv dist/${name} $out
+      mkdir -p $out
+      tar -xzf dist/${name} ${pname}-${version}.post0/* -C $out/ --strip-components=1
     '';
   };
 in buildPythonPackage rec {

pkgs/development/python-modules/tomli/default.nix

@@ -22,6 +22,7 @@ buildPythonPackage rec {
     repo = pname;
     rev = version;
     sha256 = "sha256-v0ZMrHIIaGeORwD4JiBeLthmnKZODK5odZVL0SY4etA=";
+    name = "${pname}-${version}-source";
   };
 
   nativeBuildInputs = [ flit-core ];

pkgs/top-level/python-packages.nix

@@ -8,6 +8,8 @@
 
 self: super: with self; {
 
+  bootstrapped-build = toPythonModule (callPackage ../development/python-modules/bootstrapped-build { });
+
   bootstrapped-pip = toPythonModule (callPackage ../development/python-modules/bootstrapped-pip { });
 
   setuptools = callPackage ../development/python-modules/setuptools { };