From 6ab2ecb7f65c2246a231531d95f660b4cb9cca6b Mon Sep 17 00:00:00 2001 From: Mariatta Wijaya Date: Tue, 9 Aug 2022 11:30:41 -0700 Subject: [PATCH] chore: Remove datastore schedule export Moving it to python-datastore samples. Corresponding datastore PR: https://github.com/googleapis/python-datastore/pull/344 --- datastore/schedule-export/README.md | 5 -- datastore/schedule-export/main.py | 73 ------------------ .../schedule-export/requirements-test.txt | 1 - datastore/schedule-export/requirements.txt | 1 - .../schedule-export/schedule_export_test.py | 74 ------------------- 5 files changed, 154 deletions(-) delete mode 100644 datastore/schedule-export/README.md delete mode 100644 datastore/schedule-export/main.py delete mode 100644 datastore/schedule-export/requirements-test.txt delete mode 100644 datastore/schedule-export/requirements.txt delete mode 100644 datastore/schedule-export/schedule_export_test.py diff --git a/datastore/schedule-export/README.md b/datastore/schedule-export/README.md deleted file mode 100644 index a8501cddc34b..000000000000 --- a/datastore/schedule-export/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Scheduling Datastore exports with Cloud Functions and Cloud Scheduler - -This sample application demonstrates how to schedule exports of your Datastore entities. To deploy this sample, see: - -[Scheduling exports](https://cloud.google.com/datastore/docs/schedule-export) diff --git a/datastore/schedule-export/main.py b/datastore/schedule-export/main.py deleted file mode 100644 index 19b26bddd4e6..000000000000 --- a/datastore/schedule-export/main.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2021 Google LLC All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 -import json -import os - -from googleapiclient.discovery import build -from googleapiclient.discovery_cache.base import Cache - - -class MemoryCache(Cache): - _CACHE = {} - - def get(self, url): - return MemoryCache._CACHE.get(url) - - def set(self, url, content): - MemoryCache._CACHE[url] = content - - -# The default cache (file_cache) is unavailable when using oauth2client >= 4.0.0 or google-auth, -# and it will log worrisome messages unless given another interface to use. -datastore = build("datastore", "v1", cache=MemoryCache()) -project_id = os.environ.get("GCP_PROJECT") - - -def datastore_export(event, context): - """Triggers a Datastore export from a Cloud Scheduler job. - - Args: - event (dict): event[data] must contain a json object encoded in - base-64. Cloud Scheduler encodes payloads in base-64 by default. - Object must include a 'bucket' value and can include 'kinds' - and 'namespaceIds' values. - context (google.cloud.functions.Context): The Cloud Functions event - metadata. - """ - - if "data" in event: - # Triggered via Cloud Scheduler, decode the inner data field of the json payload. - json_data = json.loads(base64.b64decode(event["data"]).decode("utf-8")) - else: - # Otherwise, for instance if triggered via the Cloud Console on a Cloud Function, the event is the data. - json_data = event - - bucket = json_data["bucket"] - entity_filter = {} - - if "kinds" in json_data: - entity_filter["kinds"] = json_data["kinds"] - - if "namespaceIds" in json_data: - entity_filter["namespaceIds"] = json_data["namespaceIds"] - - request_body = {"outputUrlPrefix": bucket, "entityFilter": entity_filter} - - export_request = datastore.projects().export( - projectId=project_id, body=request_body - ) - response = export_request.execute() - print(response) diff --git a/datastore/schedule-export/requirements-test.txt b/datastore/schedule-export/requirements-test.txt deleted file mode 100644 index 76593bb6e893..000000000000 --- a/datastore/schedule-export/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==7.0.1 \ No newline at end of file diff --git a/datastore/schedule-export/requirements.txt b/datastore/schedule-export/requirements.txt deleted file mode 100644 index 1c152028ad8b..000000000000 --- a/datastore/schedule-export/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -google-api-python-client==2.47.0 diff --git a/datastore/schedule-export/schedule_export_test.py b/datastore/schedule-export/schedule_export_test.py deleted file mode 100644 index daf72cf019d5..000000000000 --- a/datastore/schedule-export/schedule_export_test.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2019 Google LLC All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 - -from unittest.mock import Mock - -import main - -mock_context = Mock() -mock_context.event_id = '617187464135194' -mock_context.timestamp = '2020-04-15T22:09:03.761Z' - - -def test_datastore_export(capsys): - # Test an export without an entity filter - bucket = 'gs://my-bucket' - json_string = '{{ "bucket": "{bucket}" }}'.format(bucket=bucket) - - # Encode data like Cloud Scheduler - data = bytes(json_string, 'utf-8') - data_encoded = base64.b64encode(data) - event = {"data": data_encoded} - - # Mock the Datastore service - mockDatastore = Mock() - main.datastore = mockDatastore - - # Call tested function - main.datastore_export(event, mock_context) - out, err = capsys.readouterr() - export_args = mockDatastore.projects().export.call_args[1] - req_body = export_args['body'] - # Assert request includes test values - assert req_body['outputUrlPrefix'] == bucket - - -def test_datastore_export_entity_filter(capsys): - # Test an export with an entity filter - bucket = 'gs://my-bucket' - kinds = 'Users,Tasks' - namespaceIds = 'Customer831,Customer157' - json_string = '{{ "bucket": "{bucket}", "kinds": "{kinds}", "namespaceIds": "{namespaceIds}" }}'.format( - bucket=bucket, kinds=kinds, namespaceIds=namespaceIds) - - # Encode data like Cloud Scheduler - data = bytes(json_string, 'utf-8') - data_encoded = base64.b64encode(data) - event = {"data": data_encoded} - - # Mock the Datastore service - mockDatastore = Mock() - main.datastore = mockDatastore - - # Call tested function - main.datastore_export(event, mock_context) - out, err = capsys.readouterr() - export_args = mockDatastore.projects().export.call_args[1] - req_body = export_args['body'] - # Assert request includes test values - assert req_body['outputUrlPrefix'] == bucket - assert req_body['entityFilter']['kinds'] == kinds - assert req_body['entityFilter']['namespaceIds'] == namespaceIds