diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a1366aff830..f1774b459bb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,7 +74,7 @@ jobs: os: windows-latest rust: stable-msvc other: i686-pc-windows-msvc - - name: Windows x86_64 gnu nightly + - name: Windows x86_64 gnu nightly # runs out of space while trying to link the test suite os: windows-latest rust: nightly-gnu other: i686-pc-windows-gnu @@ -102,7 +102,16 @@ jobs: if: "!contains(matrix.rust, 'stable')" - run: cargo test - # The testsuite generates a huge amount of data, and fetch-smoke-test was + - name: Clear intermediate test output + run: | + df -h + rm -rf target/tmp + df -h + - name: gitoxide tests (all git-related tests) + run: cargo test git + env: + __CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2: 1 + # The testsuite generates a huge amount of data, and fetch-smoke-test was # running out of disk space. - name: Clear test output run: | @@ -156,6 +165,18 @@ jobs: - run: rustup update stable && rustup default stable - run: cargo test --manifest-path crates/resolver-tests/Cargo.toml + test_gitoxide: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: rustup update --no-self-update stable && rustup default stable + - run: rustup target add i686-unknown-linux-gnu + - run: sudo apt update -y && sudo apt install gcc-multilib libsecret-1-0 libsecret-1-dev -y + - run: rustup component add rustfmt || echo "rustfmt not available" + - run: cargo test + env: + __CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2: 1 + build_std: runs-on: ubuntu-latest env: @@ -196,7 +217,7 @@ jobs: permissions: contents: none name: bors build finished - needs: [docs, rustfmt, test, resolver, build_std] + needs: [docs, rustfmt, test, resolver, build_std, test_gitoxide] runs-on: ubuntu-latest if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto-cargo'" steps: diff --git a/Cargo.toml b/Cargo.toml index c6d470a2f52..7b2512cfa65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,8 @@ filetime = "0.2.9" flate2 = { version = "1.0.3", default-features = false, features = ["zlib"] } git2 = "0.16.0" git2-curl = "0.17.0" +gix = { version = "0.38.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree"] } +gix-features-for-configuration-only = { version = "0.27.0", package = "gix-features", features = [ "parallel" ] } glob = "0.3.0" hex = "0.4" hmac = "0.12.1" diff --git a/crates/cargo-test-support/src/git.rs b/crates/cargo-test-support/src/git.rs index 18c4646b37d..6fde9646736 100644 --- a/crates/cargo-test-support/src/git.rs +++ b/crates/cargo-test-support/src/git.rs @@ -247,3 +247,10 @@ pub fn tag(repo: &git2::Repository, name: &str) { false )); } + +/// Returns true if gitoxide is globally activated. +/// +/// That way, tests that normally use `git2` can transparently use `gitoxide`. +pub fn cargo_uses_gitoxide() -> bool { + std::env::var_os("__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2").map_or(false, |value| value == "1") +} diff --git a/src/cargo/core/features.rs b/src/cargo/core/features.rs index 83b3165f6ff..87652540ffa 100644 --- a/src/cargo/core/features.rs +++ b/src/cargo/core/features.rs @@ -716,6 +716,8 @@ unstable_cli_options!( doctest_xcompile: bool = ("Compile and run doctests for non-host target using runner config"), dual_proc_macros: bool = ("Build proc-macros for both the host and the target"), features: Option> = (HIDDEN), + gitoxide: Option = ("Use gitoxide for the given git interactions, or all of them if no argument is given"), + jobserver_per_rustc: bool = (HIDDEN), minimal_versions: bool = ("Resolve minimal dependency versions instead of maximum"), direct_minimal_versions: bool = ("Resolve minimal dependency versions instead of maximum (direct dependencies only)"), mtime_on_use: bool = ("Configure Cargo to update the mtime of used files"), @@ -827,6 +829,74 @@ where parse_check_cfg(crates.into_iter()).map_err(D::Error::custom) } +#[derive(Debug, Copy, Clone, Default, Deserialize)] +pub struct GitoxideFeatures { + /// All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index. + pub fetch: bool, + /// When cloning the index, perform a shallow clone. Maintain shallowness upon subsequent fetches. + pub shallow_index: bool, + /// When cloning git dependencies, perform a shallow clone and maintain shallowness on subsequent fetches. + pub shallow_deps: bool, + /// Checkout git dependencies using `gitoxide` (submodules are still handled by git2 ATM, and filters + /// like linefeed conversions are unsupported). + pub checkout: bool, + /// A feature flag which doesn't have any meaning except for preventing + /// `__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2=1` builds to enable all safe `gitoxide` features. + /// That way, `gitoxide` isn't actually used even though it's enabled. + pub internal_use_git2: bool, +} + +impl GitoxideFeatures { + fn all() -> Self { + GitoxideFeatures { + fetch: true, + shallow_index: true, + checkout: true, + shallow_deps: true, + internal_use_git2: false, + } + } + + /// Features we deem safe for everyday use - typically true when all tests pass with them + /// AND they are backwards compatible. + fn safe() -> Self { + GitoxideFeatures { + fetch: true, + shallow_index: false, + checkout: true, + shallow_deps: false, + internal_use_git2: false, + } + } +} + +fn parse_gitoxide( + it: impl Iterator>, +) -> CargoResult> { + let mut out = GitoxideFeatures::default(); + let GitoxideFeatures { + fetch, + shallow_index, + checkout, + shallow_deps, + internal_use_git2, + } = &mut out; + + for e in it { + match e.as_ref() { + "fetch" => *fetch = true, + "shallow-index" => *shallow_index = true, + "shallow-deps" => *shallow_deps = true, + "checkout" => *checkout = true, + "internal-use-git2" => *internal_use_git2 = true, + _ => { + bail!("unstable 'gitoxide' only takes `fetch`, 'shallow-index', 'shallow-deps' and 'checkout' as valid inputs") + } + } + } + Ok(Some(out)) +} + fn parse_check_cfg( it: impl Iterator>, ) -> CargoResult> { @@ -879,6 +949,13 @@ impl CliUnstable { for flag in flags { self.add(flag, &mut warnings)?; } + + if self.gitoxide.is_none() + && std::env::var_os("__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2") + .map_or(false, |value| value == "1") + { + self.gitoxide = GitoxideFeatures::safe().into(); + } Ok(warnings) } @@ -967,6 +1044,13 @@ impl CliUnstable { "doctest-xcompile" => self.doctest_xcompile = parse_empty(k, v)?, "doctest-in-workspace" => self.doctest_in_workspace = parse_empty(k, v)?, "panic-abort-tests" => self.panic_abort_tests = parse_empty(k, v)?, + "jobserver-per-rustc" => self.jobserver_per_rustc = parse_empty(k, v)?, + "gitoxide" => { + self.gitoxide = v.map_or_else( + || Ok(Some(GitoxideFeatures::all())), + |v| parse_gitoxide(v.split(',')), + )? + } "host-config" => self.host_config = parse_empty(k, v)?, "target-applies-to-host" => self.target_applies_to_host = parse_empty(k, v)?, "publish-timeout" => self.publish_timeout = parse_empty(k, v)?, diff --git a/src/cargo/ops/mod.rs b/src/cargo/ops/mod.rs index d04cb84716c..d54ae244655 100644 --- a/src/cargo/ops/mod.rs +++ b/src/cargo/ops/mod.rs @@ -52,7 +52,7 @@ mod cargo_uninstall; mod common_for_install_and_uninstall; mod fix; mod lockfile; -mod registry; +pub(crate) mod registry; mod resolve; pub mod tree; mod vendor; diff --git a/src/cargo/ops/registry.rs b/src/cargo/ops/registry.rs index edbd2d4cd52..c7ef023f1a2 100644 --- a/src/cargo/ops/registry.rs +++ b/src/cargo/ops/registry.rs @@ -619,9 +619,6 @@ pub fn configure_http_handle(config: &Config, handle: &mut Easy) -> CargoResult< handle.useragent(&format!("cargo {}", version()))?; } - // Empty string accept encoding expands to the encodings supported by the current libcurl. - handle.accept_encoding("")?; - fn to_ssl_version(s: &str) -> CargoResult { let version = match s { "default" => SslVersion::Default, @@ -631,13 +628,15 @@ pub fn configure_http_handle(config: &Config, handle: &mut Easy) -> CargoResult< "tlsv1.2" => SslVersion::Tlsv12, "tlsv1.3" => SslVersion::Tlsv13, _ => bail!( - "Invalid ssl version `{}`,\ - choose from 'default', 'tlsv1', 'tlsv1.0', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'.", - s + "Invalid ssl version `{s}`,\ + choose from 'default', 'tlsv1', 'tlsv1.0', 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'." ), }; Ok(version) } + + // Empty string accept encoding expands to the encodings supported by the current libcurl. + handle.accept_encoding("")?; if let Some(ssl_version) = &http.ssl_version { match ssl_version { SslVersionConfig::Single(s) => { diff --git a/src/cargo/sources/git/mod.rs b/src/cargo/sources/git/mod.rs index 225bad1671c..6c230be937b 100644 --- a/src/cargo/sources/git/mod.rs +++ b/src/cargo/sources/git/mod.rs @@ -1,5 +1,10 @@ pub use self::source::GitSource; pub use self::utils::{fetch, GitCheckout, GitDatabase, GitRemote}; mod known_hosts; +mod oxide; mod source; mod utils; + +pub mod fetch { + pub type Error = gix::env::collate::fetch::Error; +} diff --git a/src/cargo/sources/git/oxide.rs b/src/cargo/sources/git/oxide.rs new file mode 100644 index 00000000000..cccd127c43d --- /dev/null +++ b/src/cargo/sources/git/oxide.rs @@ -0,0 +1,338 @@ +//! This module contains all code sporting `gitoxide` for operations on `git` repositories and it mirrors +//! `utils` closely for now. One day it can be renamed into `utils` once `git2` isn't required anymore. + +use crate::ops::HttpTimeout; +use crate::util::{human_readable_bytes, network, MetricsCounter, Progress}; +use crate::{CargoResult, Config}; +use cargo_util::paths; +use gix::bstr::{BString, ByteSlice}; +use log::debug; +use std::cell::RefCell; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Weak}; +use std::time::{Duration, Instant}; + +/// For the time being, `repo_path` makes it easy to instantiate a gitoxide repo just for fetching. +/// In future this may change to be the gitoxide repository itself. +pub fn with_retry_and_progress( + repo_path: &std::path::Path, + config: &Config, + cb: &(dyn Fn( + &std::path::Path, + &AtomicBool, + &mut gix::progress::tree::Item, + &mut dyn FnMut(&gix::bstr::BStr), + ) -> Result<(), crate::sources::git::fetch::Error> + + Send + + Sync), +) -> CargoResult<()> { + std::thread::scope(|s| { + let mut progress_bar = Progress::new("Fetch", config); + network::with_retry(config, || { + let progress_root: Arc = + gix::progress::tree::root::Options { + initial_capacity: 10, + message_buffer_capacity: 10, + } + .into(); + let root = Arc::downgrade(&progress_root); + let thread = s.spawn(move || { + let mut progress = progress_root.add_child("operation"); + let mut urls = RefCell::new(Default::default()); + let res = cb( + &repo_path, + &AtomicBool::default(), + &mut progress, + &mut |url| { + *urls.borrow_mut() = Some(url.to_owned()); + }, + ); + amend_authentication_hints(res, urls.get_mut().take()) + }); + translate_progress_to_bar(&mut progress_bar, root)?; + thread.join().expect("no panic in scoped thread") + }) + }) +} + +fn translate_progress_to_bar( + progress_bar: &mut Progress<'_>, + root: Weak, +) -> CargoResult<()> { + let read_pack_bytes: gix::progress::Id = + gix::odb::pack::bundle::write::ProgressId::ReadPackBytes.into(); + let delta_index_objects: gix::progress::Id = + gix::odb::pack::index::write::ProgressId::IndexObjects.into(); + let resolve_objects: gix::progress::Id = + gix::odb::pack::index::write::ProgressId::ResolveObjects.into(); + + // We choose `N=10` here to make a `300ms * 10slots ~= 3000ms` + // sliding window for tracking the data transfer rate (in bytes/s). + let mut last_update = Instant::now(); + let mut counter = MetricsCounter::<10>::new(0, last_update); + + let mut tasks = Vec::with_capacity(10); + let update_interval = std::time::Duration::from_millis(300); + let short_check_interval = Duration::from_millis(50); + + while let Some(root) = root.upgrade() { + let not_yet = last_update.elapsed() < update_interval; + if not_yet { + std::thread::sleep(short_check_interval); + } + root.sorted_snapshot(&mut tasks); + + fn progress_by_id( + id: gix::progress::Id, + task: &gix::progress::Task, + ) -> Option<&gix::progress::Value> { + (task.id == id).then(|| task.progress.as_ref()).flatten() + } + fn find_in( + tasks: &[(K, gix::progress::Task)], + cb: impl Fn(&gix::progress::Task) -> Option<&gix::progress::Value>, + ) -> Option<&gix::progress::Value> { + tasks.iter().find_map(|(_, t)| cb(t)) + } + + if let Some(objs) = find_in(&tasks, |t| progress_by_id(resolve_objects, t)) { + // Resolving deltas. + let objects = objs.step.load(Ordering::Relaxed); + let total_objects = objs.done_at.expect("known amount of objects"); + let msg = format!(", ({objects}/{total_objects}) resolving deltas"); + + progress_bar.tick(objects, total_objects, &msg)?; + } else if let Some((objs, read_pack)) = + find_in(&tasks, |t| progress_by_id(read_pack_bytes, t)).and_then(|read| { + find_in(&tasks, |t| progress_by_id(delta_index_objects, t)) + .map(|delta| (delta, read)) + }) + { + // Receiving objects. + let objects = objs.step.load(Ordering::Relaxed); + let total_objects = objs.done_at.expect("known amount of objects"); + let received_bytes = read_pack.step.load(Ordering::Relaxed); + + let now = Instant::now(); + if !not_yet { + counter.add(received_bytes, now); + last_update = now; + } + let (rate, unit) = human_readable_bytes(counter.rate() as u64); + let msg = format!(", {rate:.2}{unit}/s"); + + progress_bar.tick(objects, total_objects, &msg)?; + } + } + Ok(()) +} + +fn amend_authentication_hints( + res: Result<(), crate::sources::git::fetch::Error>, + last_url_for_authentication: Option, +) -> CargoResult<()> { + let Err(err) = res else { return Ok(()) }; + let e = match &err { + crate::sources::git::fetch::Error::PrepareFetch( + gix::remote::fetch::prepare::Error::RefMap(gix::remote::ref_map::Error::Handshake(err)), + ) => Some(err), + _ => None, + }; + if let Some(e) = e { + use anyhow::Context; + let auth_message = match e { + gix::protocol::handshake::Error::Credentials(_) => { + "\n* attempted to find username/password via \ + git's `credential.helper` support, but failed" + .into() + } + gix::protocol::handshake::Error::InvalidCredentials { .. } => { + "\n* attempted to find username/password via \ + `credential.helper`, but maybe the found \ + credentials were incorrect" + .into() + } + gix::protocol::handshake::Error::Transport(_) => { + let msg = concat!( + "network failure seems to have happened\n", + "if a proxy or similar is necessary `net.git-fetch-with-cli` may help here\n", + "https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli" + ); + return Err(anyhow::Error::from(err)).context(msg); + } + _ => None, + }; + if let Some(auth_message) = auth_message { + let mut msg = "failed to authenticate when downloading \ + repository" + .to_string(); + if let Some(url) = last_url_for_authentication { + msg.push_str(": "); + msg.push_str(url.to_str_lossy().as_ref()); + } + msg.push('\n'); + msg.push_str(auth_message); + msg.push_str("\n\n"); + msg.push_str("if the git CLI succeeds then `net.git-fetch-with-cli` may help here\n"); + msg.push_str( + "https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli", + ); + return Err(anyhow::Error::from(err)).context(msg); + } + } + Err(err.into()) +} + +/// The reason we are opening a git repository. +/// +/// This can affect the way we open it and the cost associated with it. +pub enum OpenMode { + /// We need `git_binary` configuration as well for being able to see credential helpers + /// that are configured with the `git` installation itself. + /// However, this is slow on windows (~150ms) and most people won't need it as they use the + /// standard index which won't ever need authentication, so we only enable this when needed. + ForFetch, +} + +impl OpenMode { + /// Sometimes we don't need to pay for figuring out the system's git installation, and this tells + /// us if that is the case. + pub fn needs_git_binary_config(&self) -> bool { + match self { + OpenMode::ForFetch => true, + } + } +} + +/// Produce a repository with everything pre-configured according to `config`. Most notably this includes +/// transport configuration. Knowing its `purpose` helps to optimize the way we open the repository. +/// Use `config_overrides` to configure the new repository. +pub fn open_repo( + repo_path: &std::path::Path, + config_overrides: Vec, + purpose: OpenMode, +) -> Result { + gix::open_opts(repo_path, { + let mut opts = gix::open::Options::default(); + opts.permissions.config = gix::permissions::Config::all(); + opts.permissions.config.git_binary = purpose.needs_git_binary_config(); + opts.with(gix::sec::Trust::Full) + .config_overrides(config_overrides) + }) +} + +/// Convert `git` related cargo configuration into the respective `git` configuration which can be +/// used when opening new repositories. +pub fn cargo_config_to_gitoxide_overrides(config: &Config) -> CargoResult> { + use gix::config::tree::{gitoxide, Core, Http, Key}; + let timeout = HttpTimeout::new(config)?; + let http = config.http_config()?; + + let mut values = vec![ + gitoxide::Http::CONNECT_TIMEOUT.validated_assignment_fmt(&timeout.dur.as_millis())?, + Http::LOW_SPEED_LIMIT.validated_assignment_fmt(&timeout.low_speed_limit)?, + Http::LOW_SPEED_TIME.validated_assignment_fmt(&timeout.dur.as_secs())?, + // Assure we are not depending on committer information when updating refs after cloning. + Core::LOG_ALL_REF_UPDATES.validated_assignment_fmt(&false)?, + ]; + if let Some(proxy) = &http.proxy { + values.push(Http::PROXY.validated_assignment_fmt(proxy)?); + } + if let Some(check_revoke) = http.check_revoke { + values.push(Http::SCHANNEL_CHECK_REVOKE.validated_assignment_fmt(&check_revoke)?); + } + if let Some(cainfo) = &http.cainfo { + values.push( + Http::SSL_CA_INFO.validated_assignment_fmt(&cainfo.resolve_path(config).display())?, + ); + } + + values.push(if let Some(user_agent) = &http.user_agent { + Http::USER_AGENT.validated_assignment_fmt(user_agent) + } else { + Http::USER_AGENT.validated_assignment_fmt(&format!("cargo {}", crate::version())) + }?); + if let Some(ssl_version) = &http.ssl_version { + use crate::util::config::SslVersionConfig; + match ssl_version { + SslVersionConfig::Single(version) => { + values.push(Http::SSL_VERSION.validated_assignment_fmt(&version)?); + } + SslVersionConfig::Range(range) => { + values.push( + gitoxide::Http::SSL_VERSION_MIN + .validated_assignment_fmt(&range.min.as_deref().unwrap_or("default"))?, + ); + values.push( + gitoxide::Http::SSL_VERSION_MAX + .validated_assignment_fmt(&range.max.as_deref().unwrap_or("default"))?, + ); + } + } + } else if cfg!(windows) { + // This text is copied from https://github.com/rust-lang/cargo/blob/39c13e67a5962466cc7253d41bc1099bbcb224c3/src/cargo/ops/registry.rs#L658-L674 . + // This is a temporary workaround for some bugs with libcurl and + // schannel and TLS 1.3. + // + // Our libcurl on Windows is usually built with schannel. + // On Windows 11 (or Windows Server 2022), libcurl recently (late + // 2022) gained support for TLS 1.3 with schannel, and it now defaults + // to 1.3. Unfortunately there have been some bugs with this. + // https://github.com/curl/curl/issues/9431 is the most recent. Once + // that has been fixed, and some time has passed where we can be more + // confident that the 1.3 support won't cause issues, this can be + // removed. + // + // Windows 10 is unaffected. libcurl does not support TLS 1.3 on + // Windows 10. (Windows 10 sorta had support, but it required enabling + // an advanced option in the registry which was buggy, and libcurl + // does runtime checks to prevent it.) + values.push(gitoxide::Http::SSL_VERSION_MIN.validated_assignment_fmt(&"default")?); + values.push(gitoxide::Http::SSL_VERSION_MAX.validated_assignment_fmt(&"tlsv1.2")?); + } + if let Some(debug) = http.debug { + values.push(gitoxide::Http::VERBOSE.validated_assignment_fmt(&debug)?); + } + if let Some(multiplexing) = http.multiplexing { + let http_version = multiplexing.then(|| "HTTP/2").unwrap_or("HTTP/1.1"); + // Note that failing to set the HTTP version in `gix-transport` isn't fatal, + // which is why we don't have to try to figure out if HTTP V2 is supported in the + // currently linked version (see `try_old_curl!()`) + values.push(Http::VERSION.validated_assignment_fmt(&http_version)?); + } + + Ok(values) +} + +pub fn reinitialize(git_dir: &Path) -> CargoResult<()> { + fn init(path: &Path, bare: bool) -> CargoResult<()> { + let mut opts = git2::RepositoryInitOptions::new(); + // Skip anything related to templates, they just call all sorts of issues as + // we really don't want to use them yet they insist on being used. See #6240 + // for an example issue that comes up. + opts.external_template(false); + opts.bare(bare); + git2::Repository::init_opts(&path, &opts)?; + Ok(()) + } + // Here we want to drop the current repository object pointed to by `repo`, + // so we initialize temporary repository in a sub-folder, blow away the + // existing git folder, and then recreate the git repo. Finally we blow away + // the `tmp` folder we allocated. + debug!("reinitializing git repo at {:?}", git_dir); + let tmp = git_dir.join("tmp"); + let bare = !git_dir.ends_with(".git"); + init(&tmp, false)?; + for entry in git_dir.read_dir()? { + let entry = entry?; + if entry.file_name().to_str() == Some("tmp") { + continue; + } + let path = entry.path(); + drop(paths::remove_file(&path).or_else(|_| paths::remove_dir_all(&path))); + } + init(git_dir, bare)?; + paths::remove_dir_all(&tmp)?; + Ok(()) +} diff --git a/src/cargo/sources/git/utils.rs b/src/cargo/sources/git/utils.rs index 1979cf8a3a0..17da5e59598 100644 --- a/src/cargo/sources/git/utils.rs +++ b/src/cargo/sources/git/utils.rs @@ -2,6 +2,8 @@ //! authentication/cloning. use crate::core::{GitReference, Verbosity}; +use crate::sources::git::oxide; +use crate::sources::git::oxide::cargo_config_to_gitoxide_overrides; use crate::util::errors::CargoResult; use crate::util::{human_readable_bytes, network, Config, IntoUrl, MetricsCounter, Progress}; use anyhow::{anyhow, Context as _}; @@ -16,6 +18,7 @@ use std::fmt; use std::path::{Path, PathBuf}; use std::process::Command; use std::str; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use url::Url; @@ -802,7 +805,7 @@ pub fn with_fetch_options( pub fn fetch( repo: &mut git2::Repository, - url: &str, + orig_url: &str, reference: &GitReference, config: &Config, ) -> CargoResult<()> { @@ -818,7 +821,7 @@ pub fn fetch( // If we're fetching from GitHub, attempt GitHub's special fast path for // testing if we've already got an up-to-date copy of the repository - let oid_to_fetch = match github_fast_path(repo, url, reference, config) { + let oid_to_fetch = match github_fast_path(repo, orig_url, reference, config) { Ok(FastPathRev::UpToDate) => return Ok(()), Ok(FastPathRev::NeedsFetch(rev)) => Some(rev), Ok(FastPathRev::Indeterminate) => None, @@ -880,53 +883,157 @@ pub fn fetch( // flavors of authentication possible while also still giving us all the // speed and portability of using `libgit2`. if let Some(true) = config.net_config()?.git_fetch_with_cli { - return fetch_with_cli(repo, url, &refspecs, tags, config); + return fetch_with_cli(repo, orig_url, &refspecs, tags, config); } + if config + .cli_unstable() + .gitoxide + .map_or(false, |git| git.fetch) + { + let git2_repo = repo; + let config_overrides = cargo_config_to_gitoxide_overrides(config)?; + let repo_reinitialized = AtomicBool::default(); + let res = oxide::with_retry_and_progress( + &git2_repo.path().to_owned(), + config, + &|repo_path, + should_interrupt, + mut progress, + url_for_authentication: &mut dyn FnMut(&gix::bstr::BStr)| { + // The `fetch` operation here may fail spuriously due to a corrupt + // repository. It could also fail, however, for a whole slew of other + // reasons (aka network related reasons). We want Cargo to automatically + // recover from corrupt repositories, but we don't want Cargo to stomp + // over other legitimate errors. + // + // Consequently we save off the error of the `fetch` operation and if it + // looks like a "corrupt repo" error then we blow away the repo and try + // again. If it looks like any other kind of error, or if we've already + // blown away the repository, then we want to return the error as-is. + loop { + let res = oxide::open_repo( + repo_path, + config_overrides.clone(), + oxide::OpenMode::ForFetch, + ) + .map_err(crate::sources::git::fetch::Error::from) + .and_then(|repo| { + debug!("initiating fetch of {:?} from {}", refspecs, orig_url); + let url_for_authentication = &mut *url_for_authentication; + let remote = repo + .remote_at(orig_url)? + .with_fetch_tags(if tags { + gix::remote::fetch::Tags::All + } else { + gix::remote::fetch::Tags::Included + }) + .with_refspecs( + refspecs.iter().map(|s| s.as_str()), + gix::remote::Direction::Fetch, + ) + .map_err(crate::sources::git::fetch::Error::Other)?; + let url = remote + .url(gix::remote::Direction::Fetch) + .expect("set at init") + .to_owned(); + let connection = + remote.connect(gix::remote::Direction::Fetch, &mut progress)?; + let mut authenticate = connection.configured_credentials(url)?; + let connection = connection.with_credentials( + move |action: gix::protocol::credentials::helper::Action| { + if let Some(url) = action + .context() + .and_then(|ctx| ctx.url.as_ref().filter(|url| *url != orig_url)) + { + url_for_authentication(url.as_ref()); + } + authenticate(action) + }, + ); + let outcome = connection + .prepare_fetch(gix::remote::ref_map::Options::default())? + .receive(should_interrupt)?; + Ok(outcome) + }); + let err = match res { + Ok(_) => break, + Err(e) => e, + }; + debug!("fetch failed: {}", err); + + if !repo_reinitialized.load(Ordering::Relaxed) + // We check for errors that could occour if the configuration, refs or odb files are corrupted. + // We don't check for errors related to writing as `gitoxide` is expected to create missing leading + // folder before writing files into it, or else not even open a directory as git repository (which is + // also handled here). + && err.is_corrupted() + { + repo_reinitialized.store(true, Ordering::Relaxed); + debug!( + "looks like this is a corrupt repository, reinitializing \ + and trying again" + ); + if oxide::reinitialize(repo_path).is_ok() { + continue; + } + } - debug!("doing a fetch for {}", url); - let git_config = git2::Config::open_default()?; - with_fetch_options(&git_config, url, config, &mut |mut opts| { - if tags { - opts.download_tags(git2::AutotagOption::All); + return Err(err.into()); + } + Ok(()) + }, + ); + if repo_reinitialized.load(Ordering::Relaxed) { + *git2_repo = git2::Repository::open(git2_repo.path())?; } - // The `fetch` operation here may fail spuriously due to a corrupt - // repository. It could also fail, however, for a whole slew of other - // reasons (aka network related reasons). We want Cargo to automatically - // recover from corrupt repositories, but we don't want Cargo to stomp - // over other legitimate errors. - // - // Consequently we save off the error of the `fetch` operation and if it - // looks like a "corrupt repo" error then we blow away the repo and try - // again. If it looks like any other kind of error, or if we've already - // blown away the repository, then we want to return the error as-is. - let mut repo_reinitialized = false; - loop { - debug!("initiating fetch of {:?} from {}", refspecs, url); - let res = repo - .remote_anonymous(url)? - .fetch(&refspecs, Some(&mut opts), None); - let err = match res { - Ok(()) => break, - Err(e) => e, - }; - debug!("fetch failed: {}", err); - - if !repo_reinitialized && matches!(err.class(), ErrorClass::Reference | ErrorClass::Odb) - { - repo_reinitialized = true; - debug!( - "looks like this is a corrupt repository, reinitializing \ + res + } else { + debug!("doing a fetch for {}", orig_url); + let git_config = git2::Config::open_default()?; + with_fetch_options(&git_config, orig_url, config, &mut |mut opts| { + if tags { + opts.download_tags(git2::AutotagOption::All); + } + // The `fetch` operation here may fail spuriously due to a corrupt + // repository. It could also fail, however, for a whole slew of other + // reasons (aka network related reasons). We want Cargo to automatically + // recover from corrupt repositories, but we don't want Cargo to stomp + // over other legitimate errors. + // + // Consequently we save off the error of the `fetch` operation and if it + // looks like a "corrupt repo" error then we blow away the repo and try + // again. If it looks like any other kind of error, or if we've already + // blown away the repository, then we want to return the error as-is. + let mut repo_reinitialized = false; + loop { + debug!("initiating fetch of {:?} from {}", refspecs, orig_url); + let res = repo + .remote_anonymous(orig_url)? + .fetch(&refspecs, Some(&mut opts), None); + let err = match res { + Ok(()) => break, + Err(e) => e, + }; + debug!("fetch failed: {}", err); + + if !repo_reinitialized + && matches!(err.class(), ErrorClass::Reference | ErrorClass::Odb) + { + repo_reinitialized = true; + debug!( + "looks like this is a corrupt repository, reinitializing \ and trying again" - ); - if reinitialize(repo).is_ok() { - continue; + ); + if reinitialize(repo).is_ok() { + continue; + } } - } - return Err(err.into()); - } - Ok(()) - }) + return Err(err.into()); + } + Ok(()) + }) + } } fn fetch_with_cli( diff --git a/src/cargo/util/network.rs b/src/cargo/util/network.rs index 4b27160b009..70c38b6d42b 100644 --- a/src/cargo/util/network.rs +++ b/src/cargo/util/network.rs @@ -79,6 +79,15 @@ fn maybe_spurious(err: &Error) -> bool { return true; } } + + use gix::protocol::transport::IsSpuriousError; + + if let Some(err) = err.downcast_ref::() { + if err.is_spurious() { + return true; + } + } + false } diff --git a/src/doc/contrib/src/tests/running.md b/src/doc/contrib/src/tests/running.md index dc306fbb4b2..e91702f96bd 100644 --- a/src/doc/contrib/src/tests/running.md +++ b/src/doc/contrib/src/tests/running.md @@ -40,6 +40,16 @@ the `CARGO_RUN_BUILD_STD_TESTS=1` environment variable and running `cargo test `rust-src` component installed with `rustup component add rust-src --toolchain=nightly`. +## Running with `gitoxide` as default git backend in tests + +By default, the `git2` backend is used for most git operations. As tests need to explicitly +opt-in to use nightly features and feature flags, adjusting all tests to run with nightly +and `-Zgitoxide` is unfeasible. + +This is why the private environment variable named `__CARGO_USE_GITOXIDE_INSTEAD_OF_GIT2` can be +set while running tests to automatically enable the `-Zgitoxide` flag implicitly, allowing to +test `gitoxide` for the entire cargo test suite. + ## Running public network tests Some (very rare) tests involve connecting to the public internet. diff --git a/src/doc/src/reference/unstable.md b/src/doc/src/reference/unstable.md index d31b99a1f08..2a8984ec1b5 100644 --- a/src/doc/src/reference/unstable.md +++ b/src/doc/src/reference/unstable.md @@ -100,6 +100,8 @@ Each new feature described below should explain how to use it. * [`cargo logout`](#cargo-logout) --- Adds the `logout` command to remove the currently saved registry token. * [publish-timeout](#publish-timeout) --- Controls the timeout between uploading the crate and being available in the index * [registry-auth](#registry-auth) --- Adds support for authenticated registries, and generate registry authentication tokens using asymmetric cryptography. +* Other + * [gitoxide](#gitoxide) --- Use `gitoxide` instead of `git2` for a set of operations. ### allow-features @@ -1277,6 +1279,21 @@ codegen-backend = true codegen-backend = "cranelift" ``` +### gitoxide + +With the 'gitoxide' unstable feature, all or the the specified git operations will be performed by +the `gitoxide` crate instead of `git2`. + +While `-Zgitoxide` enables all currently implemented features, one can individually select git operations +to run with `gitoxide` with the `-Zgitoxide=operation[,operationN]` syntax. + +Valid operations are the following: + +* `fetch` - All fetches are done with `gitoxide`, which includes git dependencies as well as the crates index. +* `shallow-index` *(planned)* - perform a shallow clone of the index. +* `shallow-deps` *(planned)* - perform a shallow clone of git dependencies. +* `checkout` *(planned)* - checkout the worktree, with support for filters and submodules. + ## Stabilized and removed features ### Compile progress diff --git a/tests/testsuite/bad_config.rs b/tests/testsuite/bad_config.rs index dabdc149cfe..ca51b101e39 100644 --- a/tests/testsuite/bad_config.rs +++ b/tests/testsuite/bad_config.rs @@ -1,5 +1,6 @@ //! Tests for some invalid .cargo/config files. +use cargo_test_support::git::cargo_uses_gitoxide; use cargo_test_support::registry::{self, Package}; use cargo_test_support::{basic_manifest, project, rustc_host}; @@ -334,23 +335,45 @@ fn bad_git_dependency() { let p = project() .file( "Cargo.toml", - r#" + &format!( + r#" [package] name = "foo" version = "0.0.0" authors = [] [dependencies] - foo = { git = "file:.." } + foo = {{ git = "{url}" }} "#, + url = if cargo_uses_gitoxide() { + "git://host.xz" + } else { + "file:.." + } + ), ) .file("src/lib.rs", "") .build(); - p.cargo("check -v") - .with_status(101) - .with_stderr( - "\ + let expected_stderr = if cargo_uses_gitoxide() { + "\ +[UPDATING] git repository `git://host.xz` +[ERROR] failed to get `foo` as a dependency of package `foo v0.0.0 [..]` + +Caused by: + failed to load source for dependency `foo` + +Caused by: + Unable to update git://host.xz + +Caused by: + failed to clone into: [..] + +Caused by: + URLs need to specify the path to the repository +" + } else { + "\ [UPDATING] git repository `file:///` [ERROR] failed to get `foo` as a dependency of package `foo v0.0.0 [..]` @@ -365,8 +388,11 @@ Caused by: Caused by: [..]'file:///' is not a valid local file URI[..] -", - ) +" + }; + p.cargo("check -v") + .with_status(101) + .with_stderr(expected_stderr) .run(); } diff --git a/tests/testsuite/git.rs b/tests/testsuite/git.rs index 48e3eaaba20..b170c204f75 100644 --- a/tests/testsuite/git.rs +++ b/tests/testsuite/git.rs @@ -9,6 +9,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::thread; +use cargo_test_support::git::cargo_uses_gitoxide; use cargo_test_support::paths::{self, CargoPathExt}; use cargo_test_support::registry::Package; use cargo_test_support::{basic_lib_manifest, basic_manifest, git, main_file, path2url, project}; @@ -1827,6 +1828,51 @@ fn fetch_downloads() { p.cargo("fetch").with_stdout("").run(); } +#[cargo_test] +fn fetch_downloads_with_git2_first_then_with_gitoxide_and_vice_versa() { + let bar = git::new("bar", |project| { + project + .file("Cargo.toml", &basic_manifest("bar", "0.5.0")) + .file("src/lib.rs", "pub fn bar() -> i32 { 1 }") + }); + let feature_configuration = if cargo_uses_gitoxide() { + // When we are always using `gitoxide` by default, create the registry with git2 as well as the download… + "-Zgitoxide=internal-use-git2" + } else { + // …otherwise create the registry and the git download with `gitoxide`. + "-Zgitoxide=fetch" + }; + + let p = project() + .file( + "Cargo.toml", + &format!( + r#" + [package] + name = "foo" + version = "0.5.0" + authors = [] + [dependencies.bar] + git = '{url}' + "#, + url = bar.url() + ), + ) + .file("src/main.rs", "fn main() {}") + .build(); + p.cargo("fetch") + .arg(feature_configuration) + .masquerade_as_nightly_cargo(&["unstable features must be available for -Z gitoxide"]) + .with_stderr(&format!( + "[UPDATING] git repository `{url}`", + url = bar.url() + )) + .run(); + + Package::new("bar", "1.0.0").publish(); // trigger a crates-index change. + p.cargo("fetch").with_stdout("").run(); +} + #[cargo_test] fn warnings_in_git_dep() { let bar = git::new("bar", |project| { diff --git a/tests/testsuite/git_auth.rs b/tests/testsuite/git_auth.rs index 9f2bece8b8d..3a6e0b60af4 100644 --- a/tests/testsuite/git_auth.rs +++ b/tests/testsuite/git_auth.rs @@ -8,6 +8,7 @@ use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::sync::Arc; use std::thread::{self, JoinHandle}; +use cargo_test_support::git::cargo_uses_gitoxide; use cargo_test_support::paths; use cargo_test_support::{basic_manifest, project}; @@ -157,8 +158,7 @@ Caused by: https://[..] Caused by: -", - addr = addr +" )) .run(); @@ -206,15 +206,16 @@ fn https_something_happens() { p.cargo("check -v") .with_status(101) .with_stderr_contains(&format!( - "[UPDATING] git repository `https://{addr}/foo/bar`", - addr = addr + "[UPDATING] git repository `https://{addr}/foo/bar`" )) .with_stderr_contains(&format!( "\ Caused by: {errmsg} ", - errmsg = if cfg!(windows) { + errmsg = if cargo_uses_gitoxide() { + "[..]SSL connect error [..]" + } else if cfg!(windows) { "[..]failed to send request: [..]" } else if cfg!(target_os = "macos") { // macOS is difficult to tests as some builds may use Security.framework, @@ -258,18 +259,40 @@ fn ssh_something_happens() { .file("src/main.rs", "") .build(); - p.cargo("check -v") - .with_status(101) - .with_stderr_contains(&format!( - "[UPDATING] git repository `ssh://{addr}/foo/bar`", - addr = addr - )) - .with_stderr_contains( + let (expected_ssh_message, expected_update) = if cargo_uses_gitoxide() { + // Due to the usage of `ssh` and `ssh.exe` respectively, the messages change. + // This will be adjusted to use `ssh2` to get rid of this dependency and have uniform messaging. + let message = if cfg!(windows) { + // The order of multiple possible messages isn't deterministic within `ssh`, and `gitoxide` detects both + // but gets to report only the first. Thus this test can flip-flop from one version of the error to the other + // and we can't test for that. + // We'd want to test for: + // "[..]ssh: connect to host 127.0.0.1 [..]" + // ssh: connect to host example.org port 22: No route to host + // "[..]banner exchange: Connection to 127.0.0.1 [..]" + // banner exchange: Connection to 127.0.0.1 port 62250: Software caused connection abort + // But since there is no common meaningful sequence or word, we can only match a small telling sequence of characters. + "[..]onnect[..]" + } else { + "[..]Connection [..] by [..]" + }; + ( + message, + format!("[..]Unable to update ssh://{addr}/foo/bar"), + ) + } else { + ( "\ Caused by: [..]failed to start SSH session: Failed getting banner[..] ", + format!("[UPDATING] git repository `ssh://{addr}/foo/bar`"), ) + }; + p.cargo("check -v") + .with_status(101) + .with_stderr_contains(&expected_update) + .with_stderr_contains(expected_ssh_message) .run(); t.join().ok().unwrap(); } @@ -294,7 +317,7 @@ fn net_err_suggests_fetch_with_cli() { p.cargo("check -v") .with_status(101) - .with_stderr( + .with_stderr(format!( "\ [UPDATING] git repository `ssh://needs-proxy.invalid/git` warning: spurious network error[..] @@ -316,9 +339,14 @@ Caused by: https://[..] Caused by: - failed to resolve address for needs-proxy.invalid[..] + {trailer} ", - ) + trailer = if cargo_uses_gitoxide() { + "An IO error occurred when talking to the server\n\nCaused by:\n ssh: Could not resolve hostname needs-proxy.invalid[..]" + } else { + "failed to resolve address for needs-proxy.invalid[..]" + } + )) .run(); p.change_file( @@ -389,8 +417,8 @@ Caused by: Caused by: [..] -", - addr = addr +{trailer}", + trailer = if cargo_uses_gitoxide() { "\nCaused by:\n [..]" } else { "" } )) .run(); diff --git a/tests/testsuite/git_gc.rs b/tests/testsuite/git_gc.rs index 4a8228f87c9..fd4fe30a979 100644 --- a/tests/testsuite/git_gc.rs +++ b/tests/testsuite/git_gc.rs @@ -5,6 +5,7 @@ use std::ffi::OsStr; use std::path::PathBuf; use cargo_test_support::git; +use cargo_test_support::git::cargo_uses_gitoxide; use cargo_test_support::paths; use cargo_test_support::project; use cargo_test_support::registry::Package; @@ -96,6 +97,11 @@ fn use_git_gc() { #[cargo_test] fn avoid_using_git() { + if cargo_uses_gitoxide() { + // file protocol without git binary is currently not possible - needs built-in upload-pack. + // See https://github.com/Byron/gitoxide/issues/734 (support for the file protocol) progress updates. + return; + } let path = env::var_os("PATH").unwrap_or_default(); let mut paths = env::split_paths(&path).collect::>(); let idx = paths diff --git a/tests/testsuite/ssh.rs b/tests/testsuite/ssh.rs index d812be275d9..31c51c3dd3d 100644 --- a/tests/testsuite/ssh.rs +++ b/tests/testsuite/ssh.rs @@ -6,6 +6,7 @@ //! NOTE: The container tests almost certainly won't work on Windows. use cargo_test_support::containers::{Container, ContainerHandle, MkFile}; +use cargo_test_support::git::cargo_uses_gitoxide; use cargo_test_support::{paths, process, project, Project}; use std::fs; use std::io::Write; @@ -415,7 +416,11 @@ fn invalid_github_key() { .build(); p.cargo("fetch") .with_status(101) - .with_stderr_contains(" error: SSH host key has changed for `github.com`") + .with_stderr_contains(if cargo_uses_gitoxide() { + " git@github.com: Permission denied (publickey)." + } else { + " error: SSH host key has changed for `github.com`" + }) .run(); } @@ -447,16 +452,7 @@ fn bundled_github_works() { ) .file("src/lib.rs", "") .build(); - let err = if cfg!(windows) { - "error authenticating: unable to connect to agent pipe; class=Ssh (23)" - } else { - "error authenticating: failed connecting with agent; class=Ssh (23)" - }; - p.cargo("fetch") - .env("SSH_AUTH_SOCK", &bogus_auth_sock) - .with_status(101) - .with_stderr(&format!( - "\ + let shared_stderr = "\ [UPDATING] git repository `ssh://git@github.com/rust-lang/bitflags.git` error: failed to get `bitflags` as a dependency of package `foo v0.1.0 ([ROOT]/foo)` @@ -472,34 +468,45 @@ Caused by: Caused by: failed to authenticate when downloading repository - * attempted ssh-agent authentication, but no usernames succeeded: `git` + *"; + let err = if cfg!(windows) { + "error authenticating: unable to connect to agent pipe; class=Ssh (23)" + } else { + "error authenticating: failed connecting with agent; class=Ssh (23)" + }; + let expected = if cargo_uses_gitoxide() { + format!( + "{shared_stderr} attempted to find username/password via `credential.helper`, but maybe the found credentials were incorrect if the git CLI succeeds then `net.git-fetch-with-cli` may help here https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli Caused by: - {err} + Credentials provided for \"ssh://git@github.com/rust-lang/bitflags.git\" were not accepted by the remote + +Caused by: + git@github.com: Permission denied (publickey). " - )) - .run(); + ) + } else { + format!( + "{shared_stderr} attempted ssh-agent authentication, but no usernames succeeded: `git` - // Explicit :22 should also work with bundled. - p.change_file( - "Cargo.toml", - r#" - [package] - name = "foo" - version = "0.1.0" + if the git CLI succeeds then `net.git-fetch-with-cli` may help here + https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli - [dependencies] - bitflags = { git = "ssh://git@github.com:22/rust-lang/bitflags.git", tag = "1.3.2" } - "#, - ); +Caused by: + {err} +" + ) + }; p.cargo("fetch") .env("SSH_AUTH_SOCK", &bogus_auth_sock) .with_status(101) - .with_stderr(&format!( - "\ + .with_stderr(&expected) + .run(); + + let shared_stderr = "\ [UPDATING] git repository `ssh://git@github.com:22/rust-lang/bitflags.git` error: failed to get `bitflags` as a dependency of package `foo v0.1.0 ([ROOT]/foo)` @@ -515,7 +522,25 @@ Caused by: Caused by: failed to authenticate when downloading repository - * attempted ssh-agent authentication, but no usernames succeeded: `git` + *"; + + let expected = if cargo_uses_gitoxide() { + format!( + "{shared_stderr} attempted to find username/password via `credential.helper`, but maybe the found credentials were incorrect + + if the git CLI succeeds then `net.git-fetch-with-cli` may help here + https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli + +Caused by: + Credentials provided for \"ssh://git@github.com:22/rust-lang/bitflags.git\" were not accepted by the remote + +Caused by: + git@github.com: Permission denied (publickey). +" + ) + } else { + format!( + "{shared_stderr} attempted ssh-agent authentication, but no usernames succeeded: `git` if the git CLI succeeds then `net.git-fetch-with-cli` may help here https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli @@ -523,7 +548,25 @@ Caused by: Caused by: {err} " - )) + ) + }; + + // Explicit :22 should also work with bundled. + p.change_file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + + [dependencies] + bitflags = { git = "ssh://git@github.com:22/rust-lang/bitflags.git", tag = "1.3.2" } + "#, + ); + p.cargo("fetch") + .env("SSH_AUTH_SOCK", &bogus_auth_sock) + .with_status(101) + .with_stderr(&expected) .run(); }