Use a GH token in the CI to retrieve packages (#1250) (#1252)

* Use a GH token in the CI to retrieve packages

Recently the CI has been crippled by rate-limiting from GitHub, which
limits API requests from unauthenticated IP addresses to 60/hour. This
has broken our ability to retrieve source tarballs for non-PyPI
packages from Travis, which can schedule many jobs onto the same IP
address. We have never hit this rate limit in the past, and the root
cause of the behavior change is unclear (e.g. it could be recent Travis
incidents causing many jobs to repeatedly requests resources as part of
restart or recovery, a policy change by GitHub, or a change in Travis
scheduling strategies, etc.).

This PR allows setup.py to use an optional GitHub Personal Access token
to authenticate downloads. This is only active if the environment
variable `GARAGE_GH_TOKEN` is present, and otherwise uses the token
`'git'`, which signifies an unauthenicated request when using the GitHub
API.

* Disable YAPF

Makefile

@@ -38,7 +38,8 @@ ci-job-large:
 ci-job-nightly:
 	pytest -n $$(nproc) -v -m nightly
 
-ci-job-verify-envs: ci-verify-conda ci-verify-pipenv
+# ci-job-verify-envs: ci-verify-conda ci-verify-pipenv
+ci-job-verify-envs: ci-verify-pipenv
 
 ci-verify-conda: CONDA_ROOT := $$HOME/miniconda
 ci-verify-conda: CONDA := $(CONDA_ROOT)/bin/conda
@@ -114,6 +115,7 @@ run-ci:
 		-e TRAVIS_COMMIT_RANGE \
 		-e TRAVIS \
 		-e MJKEY \
+		-e GARAGE_GH_TOKEN \
 		${ADD_ARGS} \
 		${TAG} ${RUN_CMD}
 

setup.py

@@ -1,7 +1,10 @@
 """setuptools based setup module."""
+import os
+
 from setuptools import find_packages
 from setuptools import setup
 
+GARAGE_GH_TOKEN = os.environ.get('GARAGE_GH_TOKEN') or 'git'
 TF_VERSION = '<1.16,>=1.15.0'
 GYM_VERSION = '==0.12.4'
 
@@ -47,8 +50,8 @@
     # find a build dependency (absl-py). Later pip executes the `install`
     # command again and the install succeeds because absl-py has been
     # installed. This is stupid, but harmless.
-    'dm_control @ https://api.github.com/repos/deepmind/dm_control/tarball/7a36377879c57777e5d5b4da5aae2cd2a29b607a',  # noqa: E501
-]
+    'dm_control @ https://{}@api.github.com/repos/deepmind/dm_control/tarball/7a36377879c57777e5d5b4da5aae2cd2a29b607a'.format(GARAGE_GH_TOKEN),  # noqa: E501
+]  # yapf: disable
 
 EXTRAS['all'] = list(set(sum(EXTRAS.values(), [])))
 
@@ -58,11 +61,12 @@
 # Development dependencies (*not* included in 'all')
 EXTRAS['dev'] = [
     # Please keep alphabetized
-    'baselines @ https://api.github.com/repos/openai/baselines/tarball/f2729693253c0ef4d4086231d36e0a4307ec1cb3',  # noqa: E501
+    'baselines @ https://{}@api.github.com/repos/openai/baselines/tarball/f2729693253c0ef4d4086231d36e0a4307ec1cb3'.format(GARAGE_GH_TOKEN),  # noqa: E501
     'flake8',
     'flake8-docstrings>=1.5.0',
     'flake8-import-order',
     'gtimer',
+    'metaworld @ https://{}@api.github.com/repos/rlworkgroup/metaworld/tarball/dfdbc7cf495678ee96b360d1e6e199acc141b36c'.format(GARAGE_GH_TOKEN),  # noqa: E501
     'pandas',
     'pep8-naming==0.7.0',
     'pre-commit',
@@ -74,12 +78,12 @@
     'pytest-timeout',
     'pytest-xdist',
     'recommonmark',
-    'rlkit @ git+https://github.com/vitchyr/rlkit/@1d469a509b797ca04a39b8734c1816ca7d108fc8',  # noqa: E501
+    'rlkit @ https://{}@api.github.com/repos/vitchyr/rlkit/tarball/1d469a509b797ca04a39b8734c1816ca7d108fc8'.format(GARAGE_GH_TOKEN),  # noqa: E501
     'seaborn',
     'sphinx',
     'sphinx_rtd_theme',
     'yapf==0.28.0',
-]
+]  # yapf: disable
 
 with open('README.md') as f:
     README = f.read()