Auto merge of #11583 - ehuss:container-tests, r=epage

Add network container tests

This adds some tests which use Docker containers to provide HTTPS and SSH servers. This should help with validating that Cargo's networking and security are working correctly.  It can also potentially be used in the future for other tests that require more complex setups.

These tests are only run on Linux in CI. macOS does not have Docker there, and the Windows Docker does not support Linux containers. The tests should work on macOS if you run them locally with Docker Desktop installed. The SSH tests do not work on Windows due to issues with ssh-agent, but the HTTPS tests should work with Docker Desktop.

These tests require an opt-in environment variable to run:

* `CARGO_PUBLIC_NETWORK_TESTS=1` — This is for tests that contact the public internet.
* `CARGO_CONTAINER_TESTS=1` — This is for tests that use Docker.

.github/workflows/main.yml

@@ -34,6 +34,7 @@ jobs:
       CARGO_PROFILE_DEV_DEBUG: 1
       CARGO_PROFILE_TEST_DEBUG: 1
       CARGO_INCREMENTAL: 0
+      CARGO_PUBLIC_NETWORK_TESTS: 1
     strategy:
       matrix:
         include:
@@ -77,6 +78,9 @@ jobs:
     - run: sudo apt update -y && sudo apt install gcc-multilib libsecret-1-0 libsecret-1-dev -y
       if: matrix.os == 'ubuntu-latest'
     - run: rustup component add rustfmt || echo "rustfmt not available"
+    - name: Configure extra test environment
+      run: echo CARGO_CONTAINER_TESTS=1 >> $GITHUB_ENV
+      if: matrix.os == 'ubuntu-latest'
 
     # Deny warnings on CI to avoid warnings getting into the codebase.
     - run: cargo test --features 'deny-warnings'

crates/cargo-test-macro/src/lib.rs

@@ -55,6 +55,23 @@ pub fn cargo_test(attr: TokenStream, item: TokenStream) -> TokenStream {
                     "does not work on windows-gnu"
                 );
             }
+            "container_test" => {
+                // These tests must be opt-in because they require docker.
+                set_ignore!(
+                    option_env!("CARGO_CONTAINER_TESTS").is_none(),
+                    "CARGO_CONTAINER_TESTS must be set"
+                );
+            }
+            "public_network_test" => {
+                // These tests must be opt-in because they touch the public
+                // network. The use of these should be **EXTREMELY RARE**, and
+                // should only touch things which would nearly certainly work
+                // in CI (like github.com).
+                set_ignore!(
+                    option_env!("CARGO_PUBLIC_NETWORK_TESTS").is_none(),
+                    "CARGO_PUBLIC_NETWORK_TESTS must be set"
+                );
+            }
             "nightly" => {
                 requires_reason = true;
                 set_ignore!(is_not_nightly, "requires nightly");

crates/cargo-test-support/build.rs

@@ -3,4 +3,5 @@ fn main() {
         "cargo:rustc-env=NATIVE_ARCH={}",
         std::env::var("TARGET").unwrap()
     );
+    println!("cargo:rerun-if-changed=build.rs");
 }

crates/cargo-test-support/containers/apache/Dockerfile

@@ -0,0 +1,26 @@
+FROM httpd:2.4-alpine
+
+RUN apk add --no-cache git git-daemon openssl
+
+COPY bar /repos/bar
+WORKDIR /repos/bar
+RUN git config --global user.email "testuser@example.com" &&\
+    git config --global user.name "Test User" &&\
+    git init -b master . &&\
+    git add Cargo.toml src &&\
+    git commit -m "Initial commit" &&\
+    mv .git ../bar.git &&\
+    cd ../bar.git &&\
+    git config --bool core.bare true &&\
+    rm -rf ../bar
+WORKDIR /
+
+EXPOSE 443
+
+WORKDIR /usr/local/apache2/conf
+COPY httpd-cargo.conf .
+RUN cat httpd-cargo.conf >> httpd.conf
+RUN openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
+    -keyout server.key -out server.crt \
+    -subj "/emailAddress=webmaster@example.com/C=US/ST=California/L=San Francisco/O=Rust/OU=Cargo/CN=127.0.0.1"
+WORKDIR /

crates/cargo-test-support/containers/apache/bar/Cargo.toml

@@ -0,0 +1,4 @@
+[package]
+name = "bar"
+version = "1.0.0"
+edition = "2021"

crates/cargo-test-support/containers/apache/bar/src/lib.rs

@@ -0,0 +1 @@
+// Intentionally blank.

crates/cargo-test-support/containers/apache/httpd-cargo.conf

@@ -0,0 +1,12 @@
+SetEnv GIT_PROJECT_ROOT /repos
+SetEnv GIT_HTTP_EXPORT_ALL
+ScriptAlias /repos /usr/libexec/git-core/git-http-backend/
+LoadModule cgid_module modules/mod_cgid.so
+
+<Files "git-http-backend">
+    Require all granted
+</Files>
+
+Include conf/extra/httpd-ssl.conf
+LoadModule ssl_module modules/mod_ssl.so
+LoadModule socache_shmcb_module modules/mod_socache_shmcb.so

crates/cargo-test-support/containers/sshd/Dockerfile

@@ -0,0 +1,29 @@
+FROM alpine:3.17
+
+RUN apk add --no-cache openssh git
+RUN ssh-keygen -A
+
+RUN addgroup -S testuser && adduser -S testuser -G testuser -s /bin/ash
+# NOTE: Ideally the password should be set to *, but I am uncertain how to do
+# that in alpine. It shouldn't matter since PermitEmptyPasswords is "no".
+RUN passwd -u testuser
+
+RUN mkdir /repos && chown testuser /repos
+COPY --chown=testuser:testuser bar /repos/bar
+USER testuser
+WORKDIR /repos/bar
+RUN git config --global user.email "testuser@example.com" &&\
+    git config --global user.name "Test User" &&\
+    git init -b master . &&\
+    git add Cargo.toml src &&\
+    git commit -m "Initial commit" &&\
+    mv .git ../bar.git &&\
+    cd ../bar.git &&\
+    git config --bool core.bare true &&\
+    rm -rf ../bar
+WORKDIR /
+USER root
+
+EXPOSE 22
+
+ENTRYPOINT ["/usr/sbin/sshd", "-D", "-E", "/var/log/auth.log"]

crates/cargo-test-support/containers/sshd/bar/Cargo.toml

@@ -0,0 +1,4 @@
+[package]
+name = "bar"
+version = "1.0.0"
+edition = "2021"

crates/cargo-test-support/containers/sshd/bar/src/lib.rs

@@ -0,0 +1 @@
+// Intentionally blank.