feat(datastore): add datastore

.circleci/config.yml

@@ -4,7 +4,8 @@ orbs:
   linter: thekevjames/linter@0.1
 
 jobs:
-  test:
+  # TODO: Remove old_nox when everything is split into modules
+  old_nox:
     docker:
       - image: thekevjames/nox:2019.5.30
     environment:
@@ -14,6 +15,19 @@ jobs:
       - checkout
       - run: nox
 
+  nox:
+    docker:
+      - image: thekevjames/nox:2019.5.30
+    environment:
+      GOOGLE_APPLICATION_CREDENTIALS: /key.json
+    parameters:
+      folder:
+        type: string
+    steps:
+      - run: echo ${GOOGLE_SERVICE_PUBLIC} | base64 -d > ${GOOGLE_APPLICATION_CREDENTIALS}
+      - checkout
+      - run: nox -f <<parameters.folder>>/noxfile.py
+
   pypi:
     docker:
       - image: python:3.7.3-slim
@@ -78,7 +92,14 @@ workflows:
             tags:
               only: /.*/
 
-      - test:
+      - old_nox:
+          filters:
+            tags:
+              only: /.*/
+
+      - nox:
+          name: test-datastore
+          folder: datastore
           filters:
             tags:
               only: /.*/
@@ -95,7 +116,8 @@ workflows:
             - lint-py35
             - lint-py36
             - lint-py37
-            - test
+            - old_nox
+            - test-datastore
       - github:
           context: org-global
           filters:
@@ -108,4 +130,5 @@ workflows:
             - lint-py35
             - lint-py36
             - lint-py37
-            - test
+            - old_nox
+            - test-datastore

.github/CONTRIBUTING.rst

@@ -8,11 +8,16 @@ overview of how we've set this project up.
 Testing
 -------
 
-Tests are run with `nox`_. See ``noxfile.py`` for the scaffolding and the
-``tests/unit`` and ``tests/integration`` folders for the actual test code.
+Tests are run with `nox`_. See each project's ``noxfile.py`` for the scaffolding
+and the ``tests/unit`` and ``tests/integration`` folders for the actual test
+code.
 
-You can get nox with ``pip install nox`` and run the project's tests with
-``nox``.
+You can get nox with ``pip install nox`` and run a specific project's tests with
+``nox -f <project-folder>/noxfile.py``.
+
+Some project are still part of the ``gcloud-rest`` package. We are working on
+separating them but for now, run those projects' test together by running
+``nox``. The test scaffolding is in the root ``noxfile.py``.
 
 Local Development
 ~~~~~~~~~~~~~~~~~
@@ -72,6 +77,31 @@ in CI against all changesets.
 You can also run ``pre-commit`` in an ad-hoc fashion by calling
 ``pre-commit run --all-files``.
 
+CircleCI will also make sure that the code is correct in Python 2.7.
+To check that locally, run ``pre-commit run -c .pre-commit-config.py27.yaml --all-files``
+in a Python 2.7 environment.
+
+It may be useful to have two Python virtual environments and run pre-commit in both.
+
+Python 2.7:
+
+.. code-block:: console
+
+    $ virtualenv venv-27
+    $ source venv-27/bin/activate
+    $ pip install pre-commit
+    $ pre-commit run -c .pre-commit-config.py27.yaml --all-files
+
+Python 3.7:
+
+.. code-block:: console
+
+    $ python3 -m venv venv-37
+    $ source venv-37/bin/activate
+    $ pip install pre-commit
+    $ pre-commit run --all-files
+
+
 Other than the above enforced standards, we like code that is easy-to-read for
 any new or returning contributors with relevant comments where appropriate.
 

.gitignore

@@ -1,10 +1,10 @@
-**/*.pyc
+**/*.egg-info/*
+**/.idea/*
+**/.nox/*
+**/.pytest_cache/*
 **/__pycache__/*
-*.egg-info/*
-.cache/*
-.coverage*
-.idea/*
-.nox/*
-build/*
-dist/*
-venv/*
+**/dist/*
+**/venv*/*
+*.pyc
+*/.coverage*
+*/build/*

.pre-commit-config.py27.yaml

@@ -39,7 +39,7 @@ repos:
         - -d locally-disabled
         - -d missing-docstring
         - -d too-few-public-methods
-        - -d too-many-arguments
+        - -d too-many-arguments # TODO stop ignoring this and fix the code
 -   repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.1.6
     hooks:

.pre-commit-config.yaml

@@ -41,6 +41,7 @@ repos:
         - -d too-few-public-methods
         - -d too-many-arguments
         - -d useless-object-inheritance  # necessary for Python 2 compatibility
+        - -d too-many-arguments # TODO stop ignoring this and fix the code
 -   repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.1.6
     hooks:

datastore/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 TalkIQ
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datastore/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.rst LICENSE requirements.txt
+recursive-include gcloud *.py
+recursive-include tests *.py
+global-exclude *.pyc __pycache__

datastore/README.rst

@@ -0,0 +1,184 @@
+RESTful Python Client for Google Cloud Datastore
+================================================
+
+|pypi| |pythons|
+
+Installation
+------------
+
+.. code-block:: console
+
+    $ pip install --upgrade gcloud-rest-datastore
+
+Usage
+-----
+
+We're still working on documentation; for now, this should help get you
+started:
+
+.. code-block:: python
+
+    from gcloud.rest.datastore import Datastore
+    from gcloud.rest.datastore import Direction
+    from gcloud.rest.datastore import Filter
+    from gcloud.rest.datastore import GQLQuery
+    from gcloud.rest.datastore import Key
+    from gcloud.rest.datastore import PathElement
+    from gcloud.rest.datastore import PropertyFilter
+    from gcloud.rest.datastore import PropertyFilterOperator
+    from gcloud.rest.datastore import PropertyOrder
+    from gcloud.rest.datastore import Query
+    from gcloud.rest.datastore import Value
+
+    ds = Datastore('my-gcloud-project', '/path/to/creds.json')
+    key1 = Key('my-gcloud-project', [PathElement('Kind', 'entityname')])
+    key2 = Key('my-gcloud-project', [PathElement('Kind', 'entityname2')])
+
+    # batched lookups
+    entities = ds.lookup([key1, key2])
+
+    # convenience functions for any datastore mutations
+    ds.insert(key1, {'a_boolean': True, 'meaning_of_life': 41})
+    ds.update(key1, {'a_boolean': True, 'meaning_of_life': 42})
+    ds.upsert(key1, {'animal': 'aardvark'})
+    ds.delete(key1)
+
+    # or build your own mutation sequences with full transaction support
+    transaction = ds.beginTransaction()
+    try:
+        mutations = [
+            ds.make_mutation(Operation.INSERT, key1, properties={'animal': 'sloth'}),
+            ds.make_mutation(Operation.UPSERT, key1, properties={'animal': 'aardvark'}),
+            ds.make_mutation(Operation.INSERT, key2, properties={'animal': 'aardvark'}),
+        ]
+        ds.commit(transaction, mutations=[mutation])
+    except Exception:
+        ds.rollback(transaction)
+
+    # support for partial keys
+    partial_key = Key('my-gcloud-project', [PathElement('Kind')])
+    # and ID allocation or reservation
+    allocated_keys = ds.allocateIds([partial_key])
+    ds.reserveIds(allocated_keys)
+
+    # query support
+    property_filter = PropertyFilter(prop='answer',
+                                     operator=PropertyFilterOperator.EQUAL,
+                                     value=Value(42))
+    property_order = PropertyOrder(prop='length',
+                                   direction=Direction.DESCENDING)
+    query = Query(kind='the_meaning_of_life',
+                  query_filter=Filter(property_filter),
+                  order=property_order)
+    results = ds.runQuery(query, session=s)
+
+    # alternatively, query support using GQL
+    gql_query = GQLQuery('SELECT * FROM the_meaning_of_life WHERE answer = @answer',
+                         named_bindings={'answer': 42})
+    results = ds.runQuery(gql_query, session=s)
+
+Custom Subclasses
+~~~~~~~~~~~~~~~~~
+
+``gcloud-rest-datastore`` provides class interfaces mirroring all official
+Google API types, ie. ``Key`` and ``PathElement``, ``Entity`` and
+``EntityResult``, ``QueryResultBatch``, and ``Value``. These types will be
+returned from arbitrary Datastore operations, for example
+``Datastore.allocateIds(...)`` will return a list of ``Key`` entities.
+
+For advanced usage, all of these datatypes may be overloaded. A common use-case
+may be to deserialize entities into more specific classes. For example, given a
+custom entity class such as:
+
+.. code-block:: python
+
+    class MyEntityKind(gcloud.rest.datastore.Entity):
+        def __init__(self, key, properties = None) -> None:
+            self.key = key
+            self.is_an_aardvark = (properties or {}).get('aardvark', False)
+
+        def __repr__(self):
+            return "I'm an aardvark!" if self.is_an_aardvark else "Sorry, nope"
+
+We can then configure ``gcloud-rest-datastore`` to serialize/deserialize from
+this custom entity class with:
+
+.. code-block:: python
+
+    class MyCustomDatastore(gcloud.rest.datastore.Datastore):
+        entity_result_kind.entity_kind = MyEntityKind
+
+The full list of classes which may be overridden in this way is:
+
+.. code-block:: python
+
+    class MyVeryCustomDatastore(gcloud.rest.datastore.Datastore):
+        datastore_operation_kind = DatastoreOperation
+        entity_result_kind = EntityResult
+        entity_result_kind.entity_kind = Entity
+        entity_result_kind.entity_kind.key_kind = Key
+        key_kind = Key
+        key_kind.path_element_kind = PathElement
+        query_result_batch_kind = QueryResultBatch
+        query_result_batch_kind.entity_result_kind = EntityResult
+        value_kind = Value
+        value_kind.key_kind = Key
+
+    class MyVeryCustomQuery(gcloud.rest.datastore.Query):
+        value_kind = Value
+
+    class MyVeryCustomGQLQuery(gcloud.rest.datastore.GQLQuery):
+        value_kind = Value
+
+You can then drop-in the ``MyVeryCustomDatastore`` class anywhere where you
+previously used ``Datastore`` and do the same for ``Query`` and ``GQLQuery``.
+
+To override any sub-key, you'll need to override any parents which use it. For
+example, if you want to use a custom Key kind and be able to use queries with
+it, you will need to implement your own ``Value``, ``Query``, and ``GQLQuery``
+classes and wire them up to the rest of the custom classes:
+
+.. code-block:: python
+
+    class MyKey(gcloud.rest.datastore.Key):
+        pass
+
+    class MyValue(gcloud.rest.datastore.Value):
+        key_kind = MyKey
+
+    class MyEntity(gcloud.rest.datastore.Entity):
+        key_kind = MyKey
+        value_kind = MyValue
+
+    class MyEntityResult(gcloud.rest.datastore.EntityResult):
+        entity_kind = MyEntity
+
+    class MyQueryResultBatch(gcloud.rest.datastore.QueryResultBatch):
+        entity_result_kind = MyEntityResult
+
+    class MyDatastore(gcloud.rest.datastore.Datastore):
+        key_kind = MyKey
+        entity_result_kind = MyEntityResult
+        query_result_batch = MyQueryResultBatch
+        value_kind = MyValue
+
+    class MyQuery(gcloud.rest.datastore.Query):
+        value_kind = MyValue
+
+    class MyGQLQuery(gcloud.rest.datastore.GQLQuery):
+        value_kind = MyValue
+
+Contributing
+------------
+
+Please see our `contributing guide`_.
+
+.. _contributing guide: https://github.com/talkiq/gcloud-rest/blob/master/.github/CONTRIBUTING.rst
+
+.. |pypi| image:: https://img.shields.io/pypi/v/gcloud-rest-datastore.svg?style=flat-square
+    :alt: Latest PyPI Version
+    :target: https://pypi.org/project/gcloud-rest-datastore/
+
+.. |pythons| image:: https://img.shields.io/pypi/pyversions/gcloud-rest-datastore.svg?style=flat-square
+    :alt: Python Version Support
+    :target: https://pypi.org/project/gcloud-rest-datastore/

datastore/gcloud/__init__.py

@@ -0,0 +1,6 @@
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)

datastore/gcloud/rest/__init__.py

@@ -0,0 +1,6 @@
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)