feat(datastore): add gcloud-rest-datastore (#35)

.circleci/config.yml

@@ -4,7 +4,8 @@ orbs:
   linter: thekevjames/linter@0.1
 
 jobs:
-  test:
+  # TODO: Remove old_nox when everything is split into modules
+  old_nox:
     docker:
       - image: thekevjames/nox:2019.5.30
     environment:
@@ -14,6 +15,19 @@ jobs:
       - checkout
       - run: nox
 
+  nox:
+    docker:
+      - image: thekevjames/nox:2019.5.30
+    environment:
+      GOOGLE_APPLICATION_CREDENTIALS: /key.json
+    parameters:
+      folder:
+        type: string
+    steps:
+      - run: echo ${GOOGLE_SERVICE_PUBLIC} | base64 -d > ${GOOGLE_APPLICATION_CREDENTIALS}
+      - checkout
+      - run: nox -f <<parameters.folder>>/noxfile.py
+
   pypi:
     docker:
       - image: python:3.7.3-slim
@@ -23,6 +37,13 @@ jobs:
       - deploy:
           name: upload to pypi
           command: |
+            # TODO: Simplify when everything is split into modules
+            # The check for the dash in the tag name won't be necessary
+            if [[ $CIRCLE_TAG =~ "-" ]]
+            then
+              export PROJECT=$(echo "${CIRCLE_TAG}" | sed 's/-.*//')
+              cd "${PROJECT}"
+            fi
             python setup.py sdist bdist_wheel
             twine upload dist/*
 
@@ -41,13 +62,27 @@ jobs:
       - deploy:
           name: create GitHub release
           command: |
-            export PREV_RELEASE=$(git tag --sort=version:refname | tail -n2 | head -n1)
-            [ "${PREV_RELEASE}" = "${CIRCLE_TAG}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD)
-            [ -z "${PREV_RELEASE}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD)
+            # TODO: Simplify when everything is split into modules
+            # The check for the dash in the tag name won't be necessary
+            if [[ $CIRCLE_TAG =~ "-" ]]
+            then
+              export PROJECT=$(echo "${CIRCLE_TAG}" | sed 's/-.*//')
+              export PREV_RELEASE=$(git tag --sort=version:refname | grep ${PROJECT} | tail -n2 | head -n1)
+              [ "${PREV_RELEASE}" = "${CIRCLE_TAG}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD)
+              [ -z "${PREV_RELEASE}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD)
+
+              git log ${PREV_RELEASE}..${CIRCLE_TAG} --pretty=format:'- %s' > release-description.md
+              ./bin/linux/amd64/github-release release -t "${CIRCLE_TAG}"
+              cat release-description.md | grep ${PROJECT} | ./bin/linux/amd64/github-release edit -t ${CIRCLE_TAG} -d -
+            else
+              export PREV_RELEASE=$(git tag --sort=version:refname | tail -n2 | head -n1)
+              [ "${PREV_RELEASE}" = "${CIRCLE_TAG}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD)
+              [ -z "${PREV_RELEASE}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD)
 
-            git log ${PREV_RELEASE}..${CIRCLE_TAG} --pretty=format:'- %s' > release-description.md
-            ./bin/linux/amd64/github-release release -t "${CIRCLE_TAG}"
-            cat release-description.md | ./bin/linux/amd64/github-release edit -t ${CIRCLE_TAG} -d -
+              git log ${PREV_RELEASE}..${CIRCLE_TAG} --pretty=format:'- %s' > release-description.md
+              ./bin/linux/amd64/github-release release -t "${CIRCLE_TAG}"
+              cat release-description.md | ./bin/linux/amd64/github-release edit -t ${CIRCLE_TAG} -d -
+            fi
 
 workflows:
   run-jobs:
@@ -78,7 +113,14 @@ workflows:
             tags:
               only: /.*/
 
-      - test:
+      - old_nox:
+          filters:
+            tags:
+              only: /.*/
+
+      - nox:
+          name: test-datastore
+          folder: datastore
           filters:
             tags:
               only: /.*/
@@ -89,23 +131,27 @@ workflows:
             branches:
               ignore: /.*/
             tags:
-              only: /[0-9]+\.[0-9]+\.[0-9]+/
+              # TODO make project name mandatory when split into modules
+              only: /([a-z]+-)?[0-9]+\.[0-9]+\.[0-9]+/
           requires:
             - lint-py27
             - lint-py35
             - lint-py36
             - lint-py37
-            - test
+            - old_nox
+            - test-datastore
       - github:
           context: org-global
           filters:
             branches:
               ignore: /.*/
             tags:
-              only: /[0-9]+\.[0-9]+\.[0-9]+/
+              # TODO make project name mandatory when split into modules
+              only: /([a-z]+-)?[0-9]+\.[0-9]+\.[0-9]+/
           requires:
             - lint-py27
             - lint-py35
             - lint-py36
             - lint-py37
-            - test
+            - old_nox
+            - test-datastore

.github/CONTRIBUTING.rst

@@ -5,14 +5,29 @@ Thanks for contributing to ``gcloud-rest``! We appreciate contributions of any
 size and hope to make it easy for you to dive in. Here's the thousand-foot
 overview of how we've set this project up.
 
+Project Setup
+-------------
+
+Our vision is that each module (``auth``, ``core``, ``kms``, etc.) should be
+released in its own PyPI package.
+
+Some modules are still bundled together in the ``gcloud-rest`` package. If you
+are making changes to one of those modules, please consider creating another
+PR to create a separate package for it. You can get inspiration from the
+`datastore module`_.
+
 Testing
 -------
 
-Tests are run with `nox`_. See ``noxfile.py`` for the scaffolding and the
-``tests/unit`` and ``tests/integration`` folders for the actual test code.
+Tests are run with `nox`_. See each project's ``noxfile.py`` for the scaffolding
+and the ``tests/unit`` and ``tests/integration`` folders for the actual test
+code.
+
+You can get nox with ``pip install nox`` and run a specific project's tests with
+``nox -f <project-folder>/noxfile.py``.
 
-You can get nox with ``pip install nox`` and run the project's tests with
-``nox``.
+Modules in the ``gcloud-rest`` package are tested together, using the root
+``noxfile.py``. You can run their tests by running ``nox``.
 
 Local Development
 ~~~~~~~~~~~~~~~~~
@@ -72,6 +87,31 @@ in CI against all changesets.
 You can also run ``pre-commit`` in an ad-hoc fashion by calling
 ``pre-commit run --all-files``.
 
+CircleCI will also make sure that the code is correct in Python 2.7.
+To check that locally, run ``pre-commit run -c .pre-commit-config.py27.yaml --all-files``
+in a Python 2.7 environment.
+
+It may be useful to have two Python virtual environments and run pre-commit in both.
+
+Python 2.7:
+
+.. code-block:: console
+
+    $ virtualenv venv-27
+    $ source venv-27/bin/activate
+    $ pip install pre-commit
+    $ pre-commit run -c .pre-commit-config.py27.yaml --all-files
+
+Python 3.7:
+
+.. code-block:: console
+
+    $ python3 -m venv venv-37
+    $ source venv-37/bin/activate
+    $ pip install pre-commit
+    $ pre-commit run --all-files
+
+
 Other than the above enforced standards, we like code that is easy-to-read for
 any new or returning contributors with relevant comments where appropriate.
 
@@ -82,6 +122,7 @@ If you are a maintainer looking to release a new version, see our
 `Release documentation`_.
 
 .. _conventional changelog: https://github.com/conventional-changelog/conventional-changelog
+.. _datastore module: https://github.com/talkiq/gcloud-rest/blob/master/datastore
 .. _nox: https://nox.readthedocs.io/en/latest/
 .. _pre-commit: http://pre-commit.com/
 .. _Pull Request: https://github.com/talkiq/gcloud-rest/pull/new/master

.gitignore

@@ -1,10 +1,10 @@
-**/*.pyc
+**/*.egg-info/*
+**/.idea/*
+**/.nox/*
+**/.pytest_cache/*
 **/__pycache__/*
-*.egg-info/*
-.cache/*
-.coverage*
-.idea/*
-.nox/*
-build/*
-dist/*
-venv/*
+**/dist/*
+**/venv*/*
+*.pyc
+*/.coverage*
+*/build/*

.pre-commit-config.py27.yaml

@@ -39,7 +39,7 @@ repos:
         - -d locally-disabled
         - -d missing-docstring
         - -d too-few-public-methods
-        - -d too-many-arguments
+        - -d too-many-arguments # TODO stop ignoring this and fix the code
 -   repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.1.6
     hooks:

.pre-commit-config.yaml

@@ -41,6 +41,7 @@ repos:
         - -d too-few-public-methods
         - -d too-many-arguments
         - -d useless-object-inheritance  # necessary for Python 2 compatibility
+        - -d too-many-arguments # TODO stop ignoring this and fix the code
 -   repo: https://github.com/Lucas-C/pre-commit-hooks
     rev: v1.1.6
     hooks:

README.rst

@@ -11,6 +11,14 @@ support for ``asyncio``.
 
 |pypi| |circleci| |coverage| |pythons|
 
+Some of the client libraries live in separate packages:
+
+- |pypids| `Google Cloud Datastore`_ (`Datastore README`_)
+
+Note that all the modules will eventually be released separately, and the
+``gcloud-rest`` package will be deprecated.
+
+
 Installation
 ------------
 
@@ -21,8 +29,13 @@ Installation
 Usage
 -----
 
-This project currently exposes interfaces to ``CloudTasks``, ``KMS``, and
-``Storage``.
+This project currently exposes interfaces to ``CloudTasks``, ``Datastore``,
+``KMS``, and ``Storage``.
+
+For the libraries that are packaged separately, you can find usage examples
+in the corresponding README file.
+
+For the others, keep reading.
 
 Storage (see `bucket.py`_):
 
@@ -117,6 +130,9 @@ TaskManager (for ``CloudTasks``, see `manager.py`_):
     tm = TaskManager('my-project', 'taskqueue-name', worker_method)
     tm.find_tasks_forever()
 
+.. _Google Cloud Datastore: https://pypi.org/project/gcloud-rest-datastore/
+.. _Datastore README: https://github.com/talkiq/gcloud-rest/blob/master/datastore/README.rst
+
 .. _bucket.py: https://github.com/talkiq/gcloud-rest/blob/master/gcloud/rest/storage/bucket.py
 .. _client.py: https://github.com/talkiq/gcloud-rest/blob/master/gcloud/rest/kms/client.py
 .. _manager.py: https://github.com/talkiq/gcloud-rest/blob/master/gcloud/rest/taskqueue/manager.py
@@ -139,3 +155,7 @@ TaskManager (for ``CloudTasks``, see `manager.py`_):
 .. |pythons| image:: https://img.shields.io/pypi/pyversions/gcloud-rest.svg?style=flat-square
     :alt: Python Version Support
     :target: https://pypi.org/project/gcloud-rest/
+
+.. |pypids| image:: https://img.shields.io/pypi/v/gcloud-rest-datastore.svg?style=flat-square
+    :alt: Latest PyPI Version
+    :target: https://pypi.org/project/gcloud-rest-datastore/

datastore/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 TalkIQ
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datastore/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.rst LICENSE requirements.txt
+recursive-include gcloud *.py
+recursive-include tests *.py
+global-exclude *.pyc __pycache__