From 635c10e3f6f334f2eeb6e814a3b0e4f3b45da01d Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 2 Sep 2022 06:24:59 +0100 Subject: [PATCH] Use http with fake-gcs --- .github/workflows/object_store.yml | 75 +++++++++++------------------- object_store/src/gcp/mod.rs | 21 --------- 2 files changed, 27 insertions(+), 69 deletions(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index 118dbd932a4..6996aa70663 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -60,74 +60,53 @@ jobs: run: cargo clippy -p object_store --all-features --all-targets -- -D warnings # test the crate + # This runs outside a container to workaround lack of support for passing arguments + # to service containers - https://github.com/orgs/community/discussions/26688 linux-test: name: Emulator Tests runs-on: ubuntu-latest - services: - fake-gcs: - image: fsouza/fake-gcs-server - ports: - - 4443:4443 - localstack: - image: localstack/localstack:0.14.4 - ports: - - 4566:4566 - ec2-metadata: - image: amazon/amazon-ec2-metadata-mock:v1.9.2 - ports: - - 1338:1338 - env: - # Only allow IMDSv2 - AEMM_IMDSV2: "1" - azurite: - image: mcr.microsoft.com/azure-storage/azurite - ports: - - 10000:10002 - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - # https://github.com/rust-lang/cargo/issues/10280 - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - RUST_BACKTRACE: "1" - # Run integration tests - TEST_INTEGRATION: 1 - EC2_METADATA_ENDPOINT: http://ec2-metadata:1338 - AZURE_USE_EMULATOR: "1" - AZURITE_BLOB_STORAGE_URL: "http://azurite:10000" - AZURITE_QUEUE_STORAGE_URL: "http://azurite:10001" - GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json" - OBJECT_STORE_BUCKET: test-bucket + env: + # Disable full debug symbol generation to speed up CI build and keep memory down + # "1" means line tables only, which is useful for panic tracebacks. + RUSTFLAGS: "-C debuginfo=1" + # https://github.com/rust-lang/cargo/issues/10280 + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + RUST_BACKTRACE: "1" + # Run integration tests + TEST_INTEGRATION: 1 + EC2_METADATA_ENDPOINT: http://localhost:1338 + AZURE_USE_EMULATOR: "1" + AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" + AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001" + GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json" + OBJECT_STORE_BUCKET: test-bucket steps: - uses: actions/checkout@v3 - name: Configure Fake GCS Server (GCP emulation) run: | - curl --insecure -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "https://fake-gcs:4443/storage/v1/b" - echo '{"gcs_base_url": "https://fake-gcs:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT" + docker run -d -p 4443:4443 fsouza/fake-gcs-server -scheme http + curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b" + echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT" - name: Setup LocalStack (AWS emulation) env: AWS_DEFAULT_REGION: "us-east-1" AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test - AWS_ENDPOINT: http://localstack:4566 + AWS_ENDPOINT: http://localhost:4566 run: | - cd /tmp - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - ./aws/install - aws --endpoint-url=http://localstack:4566 s3 mb s3://test-bucket + docker run -d -p 4566:4566 localstack/localstack:0.14.4 + docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2 + aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket - name: Configure Azurite (Azure emulation) # the magical connection string is from # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | bash - az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;' + docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite + az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;' - name: Setup Rust toolchain run: | @@ -139,7 +118,7 @@ jobs: OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1" OBJECT_STORE_AWS_ACCESS_KEY_ID: test OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test - OBJECT_STORE_AWS_ENDPOINT: http://localstack:4566 + OBJECT_STORE_AWS_ENDPOINT: http://localhost:4566 run: | # run tests cargo test -p object_store --features=aws,azure,gcp diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs index 65adf9128a4..0ef4d3564b6 100644 --- a/object_store/src/gcp/mod.rs +++ b/object_store/src/gcp/mod.rs @@ -782,20 +782,6 @@ impl GoogleCloudStorageBuilder { self } - /// Use the specified http [`Client`] (defaults to [`Client::new`]) - /// - /// This allows you to set custom client options such as allowing - /// non secure connections or custom headers. - /// - /// NOTE: Currently only available in `test`s to facilitate - /// testing, to avoid leaking details and preserve our ability to - /// make changes to the implementation. - #[cfg(test)] - pub fn with_client(mut self, client: Client) -> Self { - self.client = Some(client); - self - } - /// Configure a connection to Google Cloud Storage, returning a /// new [`GoogleCloudStorage`] and consuming `self` pub fn build(self) -> Result { @@ -923,13 +909,6 @@ mod test { env::var("GOOGLE_SERVICE_ACCOUNT") .expect("already checked GOOGLE_SERVICE_ACCOUNT") ) - .with_client( - // ignore HTTPS errors in tests so we can use fake-gcs server - Client::builder() - .danger_accept_invalid_certs(true) - .build() - .expect("Error creating http client for testing") - ) } }}; }