Use http with fake-gcs (#2632)

.github/workflows/object_store.yml

@@ -60,74 +60,53 @@ jobs:
         run: cargo clippy -p object_store --all-features --all-targets -- -D warnings
 
   # test the crate
+  # This runs outside a container to workaround lack of support for passing arguments
+  # to service containers - https://github.com/orgs/community/discussions/26688
   linux-test:
     name: Emulator Tests
     runs-on: ubuntu-latest
-    services:
-      fake-gcs:
-        image: fsouza/fake-gcs-server
-        ports:
-          - 4443:4443
-      localstack:
-        image: localstack/localstack:0.14.4
-        ports:
-          - 4566:4566
-      ec2-metadata:
-        image: amazon/amazon-ec2-metadata-mock:v1.9.2
-        ports:
-          - 1338:1338
-        env:
-          # Only allow IMDSv2
-          AEMM_IMDSV2: "1"
-      azurite:
-        image: mcr.microsoft.com/azure-storage/azurite
-        ports:
-          - 10000:10002
-    container:
-      image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
-        # https://github.com/rust-lang/cargo/issues/10280
-        CARGO_NET_GIT_FETCH_WITH_CLI: "true"
-        RUST_BACKTRACE: "1"
-        # Run integration tests
-        TEST_INTEGRATION: 1
-        EC2_METADATA_ENDPOINT: http://ec2-metadata:1338
-        AZURE_USE_EMULATOR: "1"
-        AZURITE_BLOB_STORAGE_URL: "http://azurite:10000"
-        AZURITE_QUEUE_STORAGE_URL: "http://azurite:10001"
-        GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
-        OBJECT_STORE_BUCKET: test-bucket
+    env:
+      # Disable full debug symbol generation to speed up CI build and keep memory down
+      # "1" means line tables only, which is useful for panic tracebacks.
+      RUSTFLAGS: "-C debuginfo=1"
+      # https://github.com/rust-lang/cargo/issues/10280
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+      RUST_BACKTRACE: "1"
+      # Run integration tests
+      TEST_INTEGRATION: 1
+      EC2_METADATA_ENDPOINT: http://localhost:1338
+      AZURE_USE_EMULATOR: "1"
+      AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
+      AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001"
+      GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
+      OBJECT_STORE_BUCKET: test-bucket
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Configure Fake GCS Server (GCP emulation)
         run: |
-          curl --insecure -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "https://fake-gcs:4443/storage/v1/b"
-          echo '{"gcs_base_url": "https://fake-gcs:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT"
+          docker run -d -p 4443:4443 fsouza/fake-gcs-server -scheme http
+          curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
+          echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT"
 
       - name: Setup LocalStack (AWS emulation)
         env:
           AWS_DEFAULT_REGION: "us-east-1"
           AWS_ACCESS_KEY_ID: test
           AWS_SECRET_ACCESS_KEY: test
-          AWS_ENDPOINT: http://localstack:4566
+          AWS_ENDPOINT: http://localhost:4566
         run: |
-          cd /tmp
-          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-          unzip awscliv2.zip
-          ./aws/install
-          aws --endpoint-url=http://localstack:4566 s3 mb s3://test-bucket
+          docker run -d -p 4566:4566 localstack/localstack:0.14.4
+          docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2
+          aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
 
       - name: Configure Azurite (Azure emulation)
         # the magical connection string is from
         # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
         run: |
-          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
-          az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;'
+          docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite
+          az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;'
 
       - name: Setup Rust toolchain
         run: |
@@ -139,7 +118,7 @@ jobs:
           OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1"
           OBJECT_STORE_AWS_ACCESS_KEY_ID: test
           OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test
-          OBJECT_STORE_AWS_ENDPOINT: http://localstack:4566
+          OBJECT_STORE_AWS_ENDPOINT: http://localhost:4566
         run: |
           # run tests
           cargo test -p object_store --features=aws,azure,gcp

object_store/src/gcp/mod.rs

@@ -782,20 +782,6 @@ impl GoogleCloudStorageBuilder {
         self
     }
 
-    /// Use the specified http [`Client`] (defaults to [`Client::new`])
-    ///
-    /// This allows you to set custom client options such as allowing
-    /// non secure connections or custom headers.
-    ///
-    /// NOTE: Currently only available in `test`s to facilitate
-    /// testing, to avoid leaking details and preserve our ability to
-    /// make changes to the implementation.
-    #[cfg(test)]
-    pub fn with_client(mut self, client: Client) -> Self {
-        self.client = Some(client);
-        self
-    }
-
     /// Configure a connection to Google Cloud Storage, returning a
     /// new [`GoogleCloudStorage`] and consuming `self`
     pub fn build(self) -> Result<GoogleCloudStorage> {
@@ -923,13 +909,6 @@ mod test {
                         env::var("GOOGLE_SERVICE_ACCOUNT")
                             .expect("already checked GOOGLE_SERVICE_ACCOUNT")
                     )
-                    .with_client(
-                        // ignore HTTPS errors in tests so we can use fake-gcs server
-                        Client::builder()
-                            .danger_accept_invalid_certs(true)
-                            .build()
-                            .expect("Error creating http client for testing")
-                    )
             }
         }};
     }