Add GCP scenario

closes pulp#713

.ci/bootstrap_catdog.sh

@@ -16,6 +16,7 @@ pip install -r test_requirements.txt
 mkdir -p ../pulp_catdog/.ci/assets/bindings
 echo 'pypi_username: the_pypi_user' >> ../pulp_catdog/template_config.yml
 sed -i "s/test_s3: false/test_s3: true/g" ../pulp_catdog/template_config.yml
+sed -i "s/test_gcp: false/test_gcp: true/g" ../pulp_catdog/template_config.yml
 sed -i "s/test_azure: false/test_azure: true/g" ../pulp_catdog/template_config.yml
 sed -i "s/disabled_redis_runners: \[\]/disabled_redis_runners: [s3]/g" ../pulp_catdog/template_config.yml
 ./plugin-template --all pulp_catdog

.github/workflows/bootstrap.yaml

@@ -93,6 +93,7 @@ jobs:
           - TEST: pulp
           - TEST: docs
           - TEST: azure
+          - TEST: gcp
           - TEST: s3
           - TEST: lowerbounds
     outputs:

CHANGES/713.feature

@@ -0,0 +1 @@
+Add GCP scenario

README.md

@@ -251,6 +251,9 @@ The following settings are stored in `template_config.yml`.
   test_azure            Include azure job for running tests using [azurite](https://github.com/Azure/Azurite)
                         to emulate Azure.
 
+  test_gcp              Include gcp job for running tests using [fake-gcs-server](https://github.com/fsouza/fake-gcs-server)
+                        to emulate GCP.
+
   test_s3               Include s3 job for running tests using [minio](https://github.com/minio/minio)
                         to emulate S3.
 

plugin-template

@@ -54,6 +54,7 @@ DEFAULT_SETTINGS = {
     "publish_docs_to_pulpprojectdotorg": False,
     "pulp_settings": None,
     "pulp_settings_azure": None,
+    "pulp_settings_gcp": None,
     "pulp_settings_s3": None,
     "pulp_settings_stream": None,
     "pulpcore_branch": "main",
@@ -79,6 +80,7 @@ DEFAULT_SETTINGS = {
     "test_reroute": True,
     "disabled_redis_runners": [],
     "test_azure": False,
+    "test_gcp": False,
     "test_s3": False,
     "test_stream": False,
     "update_github": True,

templates/github/.ci/ansible/Containerfile.j2.copy

@@ -20,6 +20,9 @@ RUN pip3 install \
 {%- if azure_test | default(false) -%}
 {{ " " }}django-storages[azure]>=1.12.2
 {%- endif -%}
+{%- if gcp_test | default(false) -%}
+{{ " " }}django-storages[google]>=1.13.2
+{%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}
 {{ " " }}python-dateutil rhsm

templates/github/.ci/ansible/settings.py.j2.copy

@@ -63,3 +63,10 @@ AZURE_OVERWRITE_FILES = True
 AZURE_URL_EXPIRATION_SECS = 120
 AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;'
 {% endif %}
+
+{% if gcp_test | default(false) %}
+DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
+MEDIA_ROOT = ""
+GS_BUCKET_NAME = "gcppulp"
+GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
+{% endif %}

templates/github/.github/workflows/ci.yml.j2

@@ -109,6 +109,9 @@ jobs:
           {%- if test_azure %}
           - TEST: azure
           {%- endif %}
+          {%- if test_gcp %}
+          - TEST: gcp
+          {%- endif %}
           {%- if test_s3 %}
           - TEST: s3
           {%- endif %}

templates/github/.github/workflows/nightly.yml.j2

@@ -36,6 +36,9 @@ jobs:
           {%- if test_azure %}
           - TEST: azure
           {%- endif %}
+          {%- if test_gcp %}
+          - TEST: gcp
+          {%- endif %}
           {%- if test_s3 %}
           - TEST: s3
           {%- endif %}

templates/github/.github/workflows/release.yml.j2

@@ -77,6 +77,9 @@ jobs:
           {%- if test_azure %}
           - TEST: azure
           {%- endif %}
+          {%- if test_gcp %}
+          - TEST: gcp
+          {%- endif %}
           {%- if test_s3 %}
           - TEST: s3
           {%- endif %}

templates/github/.github/workflows/scripts/before_script.sh.j2

@@ -28,7 +28,7 @@ tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
 cmd_prefix bash -c "echo '%wheel        ALL=(ALL)       NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
 cmd_prefix bash -c "usermod -a -G wheel pulp"
 
-SCENARIOS=("pulp" "performance" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds")
+SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds")
 if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
   # Many functional tests require these
   cmd_prefix dnf install -yq lsof which dnf-plugins-core

templates/github/.github/workflows/scripts/install.sh.j2

@@ -135,7 +135,7 @@ VARSYAML
 
 {%- if docker_fixtures %}
 
-SCENARIOS=("pulp" "performance" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds")
+SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds")
 if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
   sed -i -e '/^services:/a \
   - name: pulp-fixtures\
@@ -203,6 +203,27 @@ pulp_scenario_settings: {{ pulp_settings_azure | tojson }}\
 fi
 {%- endif %}
 
+{%- if test_gcp %}
+
+if [ "$TEST" = "gcp" ]; then
+  mkdir -p cigcp
+  cd cigcp
+  openssl req -newkey rsa:2048 -x509 -nodes -keyout gcpkey.pem -new -out gcpcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-gcp" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN"
+  sudo cp gcpcert.pem /usr/local/share/ca-certificates/gcpcert.crt
+  sudo dpkg-reconfigure ca-certificates
+  cd ..
+  sed -i -e '/^services:/a \
+  - name: ci-gcp\
+    image: fsouza/fake-gcs-server\
+    volumes:\
+      - storage_data:/etc/pulp\
+    command: "{% if pulp_scheme == "https" %} -cert-location /etc/pulp/gcpcert.pem {% else %} -scheme http{% endif %}"' vars/main.yaml
+  sed -i -e '$a gcp_test: true\
+pulp_scenario_settings: {{ pulp_settings_gcp | tojson }}\
+' vars/main.yaml
+fi
+{%- endif %}
+
 echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
 
 if [ "${PULP_API_ROOT:-}" ]; then