From eb469c756de4282e37da52cc346e70ba9d116e06 Mon Sep 17 00:00:00 2001 From: Jan Holthuis Date: Fri, 30 Sep 2022 11:44:18 +0200 Subject: [PATCH] Rust as 1st class language --- CONTRIBUTING.md | 4 +- azure-pipelines.yml | 2 + pre_commit/languages/rust.py | 118 +++++++++++++++++++++++++++++------ tests/languages/rust_test.py | 70 +++++++++++++++++++++ tests/repository_test.py | 4 +- 5 files changed, 174 insertions(+), 24 deletions(-) create mode 100644 tests/languages/rust_test.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 310c17ee8..0817681a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,9 +65,9 @@ to implement. The current implemented languages are at varying levels: - 0th class - pre-commit does not require any dependencies for these languages as they're not actually languages (current examples: fail, pygrep) - 1st class - pre-commit will bootstrap a full interpreter requiring nothing to - be installed globally (current examples: node, ruby) + be installed globally (current examples: node, ruby, rust) - 2nd class - pre-commit requires the user to install the language globally but - will install tools in an isolated fashion (current examples: python, go, rust, + will install tools in an isolated fashion (current examples: python, go, swift, docker). - 3rd class - pre-commit requires the user to install both the tool and the language globally (current examples: script, system) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 454f6f137..34c94f54a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,6 +17,8 @@ jobs: parameters: toxenvs: [py37] os: windows + additional_variables: + TEMP: C:\Temp pre_test: - task: UseRubyVersion@0 - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" diff --git a/pre_commit/languages/rust.py b/pre_commit/languages/rust.py index 01c373061..5e4ecafa6 100644 --- a/pre_commit/languages/rust.py +++ b/pre_commit/languages/rust.py @@ -1,13 +1,20 @@ from __future__ import annotations import contextlib +import functools import os.path +import platform +import shutil +import sys +import tempfile +import urllib.request from typing import Generator from typing import Sequence import toml import pre_commit.constants as C +from pre_commit import parse_shebang from pre_commit.envcontext import envcontext from pre_commit.envcontext import PatchesT from pre_commit.envcontext import Var @@ -16,24 +23,61 @@ from pre_commit.prefix import Prefix from pre_commit.util import clean_path_on_failure from pre_commit.util import cmd_output_b +from pre_commit.util import make_executable +from pre_commit.util import win_exe ENVIRONMENT_DIR = 'rustenv' -get_default_version = helpers.basic_get_default_version health_check = helpers.basic_health_check -def get_env_patch(target_dir: str) -> PatchesT: +@functools.lru_cache(maxsize=1) +def get_default_version() -> str: + # If rust is already installed, we can save a bunch of setup time by + # using the installed version. + # + # Just detecting the executable does not suffice, because if rustup is + # installed but no toolchain is available, then `cargo` exists but + # cannot be used without installing a toolchain first. + if cmd_output_b('cargo', '--version', retcode=None)[0] == 0: + return 'system' + else: + return C.DEFAULT + + +def _rust_toolchain(language_version: str) -> str: + """Transform the language version into a rust toolchain version.""" + if language_version == C.DEFAULT: + return 'stable' + else: + return language_version + + +def _envdir(prefix: Prefix, version: str) -> str: + directory = helpers.environment_dir(ENVIRONMENT_DIR, version) + return prefix.path(directory) + + +def get_env_patch(target_dir: str, version: str) -> PatchesT: return ( + ('CARGO_HOME', target_dir), ('PATH', (os.path.join(target_dir, 'bin'), os.pathsep, Var('PATH'))), + # Only set RUSTUP_TOOLCHAIN if we don't want use the system's default + # toolchain + *( + (('RUSTUP_TOOLCHAIN', _rust_toolchain(version)),) + if version != 'system' else () + ), ) @contextlib.contextmanager -def in_env(prefix: Prefix) -> Generator[None, None, None]: - target_dir = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), - ) - with envcontext(get_env_patch(target_dir)): +def in_env( + prefix: Prefix, + language_version: str, +) -> Generator[None, None, None]: + with envcontext( + get_env_patch(_envdir(prefix, language_version), language_version), + ): yield @@ -52,15 +96,45 @@ def _add_dependencies( f.truncate() +def install_rust_with_toolchain(toolchain: str) -> None: + with tempfile.TemporaryDirectory() as rustup_dir: + with envcontext((('RUSTUP_HOME', rustup_dir),)): + # acquire `rustup` if not present + if parse_shebang.find_executable('rustup') is None: + # We did not detect rustup and need to download it first. + if sys.platform == 'win32': # pragma: win32 cover + if platform.machine() == 'x86_64': + url = 'https://win.rustup.rs/x86_64' + else: + url = 'https://win.rustup.rs/i686' + else: # pragma: win32 no cover + url = 'https://sh.rustup.rs' + + resp = urllib.request.urlopen(url) + + rustup_init = os.path.join(rustup_dir, win_exe('rustup-init')) + with open(rustup_init, 'wb') as f: + shutil.copyfileobj(resp, f) + make_executable(rustup_init) + + # install rustup into `$CARGO_HOME/bin` + cmd_output_b( + rustup_init, '-y', '--quiet', '--no-modify-path', + '--default-toolchain', 'none', + ) + + cmd_output_b( + 'rustup', 'toolchain', 'install', '--no-self-update', + toolchain, + ) + + def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: - helpers.assert_version_default('rust', version) - directory = prefix.path( - helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), - ) + directory = _envdir(prefix, version) # There are two cases where we might want to specify more dependencies: # as dependencies for the library being built, and as binary packages @@ -84,17 +158,21 @@ def install_environment( packages_to_install: set[tuple[str, ...]] = {('--path', '.')} for cli_dep in cli_deps: cli_dep = cli_dep[len('cli:'):] - package, _, version = cli_dep.partition(':') - if version != '': - packages_to_install.add((package, '--version', version)) + package, _, crate_version = cli_dep.partition(':') + if crate_version != '': + packages_to_install.add((package, '--version', crate_version)) else: packages_to_install.add((package,)) - for args in packages_to_install: - cmd_output_b( - 'cargo', 'install', '--bins', '--root', directory, *args, - cwd=prefix.prefix_dir, - ) + with in_env(prefix, version): + if version != 'system': + install_rust_with_toolchain(_rust_toolchain(version)) + + for args in packages_to_install: + cmd_output_b( + 'cargo', 'install', '--bins', '--root', directory, *args, + cwd=prefix.prefix_dir, + ) def run_hook( @@ -102,5 +180,5 @@ def run_hook( file_args: Sequence[str], color: bool, ) -> tuple[int, bytes]: - with in_env(hook.prefix): + with in_env(hook.prefix, hook.language_version): return helpers.run_xargs(hook, hook.cmd, file_args, color=color) diff --git a/tests/languages/rust_test.py b/tests/languages/rust_test.py new file mode 100644 index 000000000..9bf97830a --- /dev/null +++ b/tests/languages/rust_test.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from unittest import mock + +import pytest + +import pre_commit.constants as C +from pre_commit import parse_shebang +from pre_commit.languages import rust +from pre_commit.prefix import Prefix +from pre_commit.util import cmd_output + +ACTUAL_GET_DEFAULT_VERSION = rust.get_default_version.__wrapped__ + + +@pytest.fixture +def cmd_output_b_mck(): + with mock.patch.object(rust, 'cmd_output_b') as mck: + yield mck + + +def test_sets_system_when_rust_is_available(cmd_output_b_mck): + cmd_output_b_mck.return_value = (0, b'', b'') + assert ACTUAL_GET_DEFAULT_VERSION() == 'system' + + +def test_uses_default_when_rust_is_not_available(cmd_output_b_mck): + cmd_output_b_mck.return_value = (127, b'', b'error: not found') + assert ACTUAL_GET_DEFAULT_VERSION() == C.DEFAULT + + +@pytest.mark.parametrize('language_version', (C.DEFAULT, '1.56.0')) +def test_installs_with_bootstrapped_rustup(tmpdir, language_version): + tmpdir.join('src', 'main.rs').ensure().write( + 'fn main() {\n' + ' println!("Hello, world!");\n' + '}\n', + ) + tmpdir.join('Cargo.toml').ensure().write( + '[package]\n' + 'name = "hello_world"\n' + 'version = "0.1.0"\n' + 'edition = "2021"\n', + ) + prefix = Prefix(str(tmpdir)) + + find_executable_exes = [] + + original_find_executable = parse_shebang.find_executable + + def mocked_find_executable(exe: str) -> str | None: + """ + Return `None` the first time `find_executable` is called to ensure + that the bootstrapping code is executed, then just let the function + work as normal. + + Also log the arguments to ensure that everything works as expected. + """ + find_executable_exes.append(exe) + if len(find_executable_exes) == 1: + return None + return original_find_executable(exe) + + with mock.patch.object(parse_shebang, 'find_executable') as find_exe_mck: + find_exe_mck.side_effect = mocked_find_executable + rust.install_environment(prefix, language_version, ()) + assert find_executable_exes == ['rustup', 'rustup', 'cargo'] + + with rust.in_env(prefix, language_version): + assert cmd_output('hello_world')[1] == 'Hello, world!\n' diff --git a/tests/repository_test.py b/tests/repository_test.py index 11d452ca4..0d4cb651b 100644 --- a/tests/repository_test.py +++ b/tests/repository_test.py @@ -471,7 +471,7 @@ def test_additional_rust_cli_dependencies_installed( hook = _get_hook(config, store, 'rust-hook') binaries = os.listdir( hook.prefix.path( - helpers.environment_dir(rust.ENVIRONMENT_DIR, C.DEFAULT), 'bin', + helpers.environment_dir(rust.ENVIRONMENT_DIR, 'system'), 'bin', ), ) # normalize for windows @@ -490,7 +490,7 @@ def test_additional_rust_lib_dependencies_installed( hook = _get_hook(config, store, 'rust-hook') binaries = os.listdir( hook.prefix.path( - helpers.environment_dir(rust.ENVIRONMENT_DIR, C.DEFAULT), 'bin', + helpers.environment_dir(rust.ENVIRONMENT_DIR, 'system'), 'bin', ), ) # normalize for windows