From b7f16e4bba62759c6ab3182bee56ab8bd5c68a96 Mon Sep 17 00:00:00 2001
From: Ryan Julian <ryanjulian@users.noreply.github.com>
Date: Wed, 25 Mar 2020 13:02:58 -0700
Subject: [PATCH] Use a GH token in the CI to retrieve packages (#1250) (#1252)

* Use a GH token in the CI to retrieve packages

Recently the CI has been crippled by rate-limiting from GitHub, which
limits API requests from unauthenticated IP addresses to 60/hour. This
has broken our ability to retrieve source tarballs for non-PyPI
packages from Travis, which can schedule many jobs onto the same IP
address. We have never hit this rate limit in the past, and the root
cause of the behavior change is unclear (e.g. it could be recent Travis
incidents causing many jobs to repeatedly requests resources as part of
restart or recovery, a policy change by GitHub, or a change in Travis
scheduling strategies, etc.).

This PR allows setup.py to use an optional GitHub Personal Access token
to authenticate downloads. This is only active if the environment
variable `GARAGE_GH_TOKEN` is present, and otherwise uses the token
`'git'`, which signifies an unauthenicated request when using the GitHub
API.

* Disable YAPF
---
 Makefile |  4 +++-
 setup.py | 14 +++++++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 6e8bffc5c4..96eb6d8c52 100644
--- a/Makefile
+++ b/Makefile
@@ -38,7 +38,8 @@ ci-job-large:
 ci-job-nightly:
 	pytest -n $$(nproc) -v -m nightly
 
-ci-job-verify-envs: ci-verify-conda ci-verify-pipenv
+# ci-job-verify-envs: ci-verify-conda ci-verify-pipenv
+ci-job-verify-envs: ci-verify-pipenv
 
 ci-verify-conda: CONDA_ROOT := $$HOME/miniconda
 ci-verify-conda: CONDA := $(CONDA_ROOT)/bin/conda
@@ -114,6 +115,7 @@ run-ci:
 		-e TRAVIS_COMMIT_RANGE \
 		-e TRAVIS \
 		-e MJKEY \
+		-e GARAGE_GH_TOKEN \
 		${ADD_ARGS} \
 		${TAG} ${RUN_CMD}
 
diff --git a/setup.py b/setup.py
index ab27c7d81b..23bcf09629 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,10 @@
 """setuptools based setup module."""
+import os
+
 from setuptools import find_packages
 from setuptools import setup
 
+GARAGE_GH_TOKEN = os.environ.get('GARAGE_GH_TOKEN') or 'git'
 TF_VERSION = '<1.16,>=1.15.0'
 GYM_VERSION = '==0.12.4'
 
@@ -47,8 +50,8 @@
     # find a build dependency (absl-py). Later pip executes the `install`
     # command again and the install succeeds because absl-py has been
     # installed. This is stupid, but harmless.
-    'dm_control @ https://api.github.com/repos/deepmind/dm_control/tarball/7a36377879c57777e5d5b4da5aae2cd2a29b607a',  # noqa: E501
-]
+    'dm_control @ https://{}@api.github.com/repos/deepmind/dm_control/tarball/7a36377879c57777e5d5b4da5aae2cd2a29b607a'.format(GARAGE_GH_TOKEN),  # noqa: E501
+]  # yapf: disable
 
 EXTRAS['all'] = list(set(sum(EXTRAS.values(), [])))
 
@@ -58,11 +61,12 @@
 # Development dependencies (*not* included in 'all')
 EXTRAS['dev'] = [
     # Please keep alphabetized
-    'baselines @ https://api.github.com/repos/openai/baselines/tarball/f2729693253c0ef4d4086231d36e0a4307ec1cb3',  # noqa: E501
+    'baselines @ https://{}@api.github.com/repos/openai/baselines/tarball/f2729693253c0ef4d4086231d36e0a4307ec1cb3'.format(GARAGE_GH_TOKEN),  # noqa: E501
     'flake8',
     'flake8-docstrings>=1.5.0',
     'flake8-import-order',
     'gtimer',
+    'metaworld @ https://{}@api.github.com/repos/rlworkgroup/metaworld/tarball/dfdbc7cf495678ee96b360d1e6e199acc141b36c'.format(GARAGE_GH_TOKEN),  # noqa: E501
     'pandas',
     'pep8-naming==0.7.0',
     'pre-commit',
@@ -74,12 +78,12 @@
     'pytest-timeout',
     'pytest-xdist',
     'recommonmark',
-    'rlkit @ git+https://github.com/vitchyr/rlkit/@1d469a509b797ca04a39b8734c1816ca7d108fc8',  # noqa: E501
+    'rlkit @ https://{}@api.github.com/repos/vitchyr/rlkit/tarball/1d469a509b797ca04a39b8734c1816ca7d108fc8'.format(GARAGE_GH_TOKEN),  # noqa: E501
     'seaborn',
     'sphinx',
     'sphinx_rtd_theme',
     'yapf==0.28.0',
-]
+]  # yapf: disable
 
 with open('README.md') as f:
     README = f.read()