ref(py3): pre-commit upgrade + sweeping rerun + __future__ removals

.github/workflows/backend-lint.yml

@@ -10,7 +10,7 @@ jobs:
   lint:
     name: backend lint
     runs-on: ubuntu-16.04
-    timeout-minutes: 3
+    timeout-minutes: 10
 
     steps:
       - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -12,24 +12,25 @@ exclude: >
         ^src/sentry/data/
     )
 repos:
--   repo: https://github.com/ambv/black
-    rev: 19.10b0
+-   repo: local
     hooks:
+    # Configuration for black exists in pyproject.toml,
+    # but we let pre-commit take care of the file filtering.
     - id: black
+      name: black
+      entry: black
+      language: python
       types: [python]
-      exclude: (migrations/)
--   repo: local
-    hooks:
+    # Configuration for flake8 exists in setup.cfg,
+    # but we let pre-commit take care of the file filtering.
     - id: flake8
       name: flake8
       entry: flake8
       language: python
-      files: \.py$
+      types: [python]
       log_file: '.artifacts/flake8.pycodestyle.log'
-      additional_dependencies:
-      - sentry-flake8==0.4.0
 -   repo: git://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v3.4.0
     hooks:
     - id: check-case-conflict
     - id: check-executables-have-shebangs

Makefile

@@ -79,8 +79,7 @@ setup-git: ensure-venv setup-git-config
 	@echo "--> Installing git hooks"
 	mkdir -p .git/hooks && cd .git/hooks && ln -sf ../../config/hooks/* ./
 	@python3 -c '' || (echo 'Please run `make setup-pyenv` to install the required Python 3 version.'; exit 1)
-	@# pre-commit loosely pins virtualenv, which has caused problems in the past.
-	$(PIP) install "pre-commit==1.18.2" "virtualenv==20.0.32"
+	$(PIP) install -r requirements-pre-commit.txt
 	@pre-commit install --install-hooks
 	@echo ""
 

bin/dump-command-help

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+
 import os
 import six
 import click

bin/find-good-catalogs

@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 
+
 import os
-import json
+import json  # noqa: b317
 import click
 
 from babel.messages.pofile import read_po
@@ -11,7 +12,7 @@ MINIMUM = 80
 
 
 def is_translated(msg):
-    if isinstance(msg.string, basestring):
+    if isinstance(msg.string, bytes):
         return bool(msg.string)
     for item in msg.string:
         if not item:

bin/lint

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import absolute_import
 
 import os
 import sys

bin/load-mocks

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import absolute_import
 
 from sentry.runner import configure
 
@@ -63,7 +62,6 @@ from sentry.models import (
 )
 from sentry.signals import mocks_loaded
 from sentry.similarity import features
-from sentry.snuba.models import QueryAggregations
 from sentry.utils import loremipsum
 from sentry.utils.hashlib import md5_text
 from sentry.utils.samples import create_sample_event as _create_sample_event
@@ -164,7 +162,7 @@ def generate_tombstones(project, user):
         GroupTombstone.objects.create(
             previous_group_id=prev_group_id,
             actor_id=user.id,
-            **{name: getattr(group, name) for name in TOMBSTONE_FIELDS_FROM_GROUP}
+            **{name: getattr(group, name) for name in TOMBSTONE_FIELDS_FROM_GROUP},
         )
         prev_group_id += 1
 
@@ -260,7 +258,10 @@ def create_sample_time_series(event, release=None):
             int(count * 1.1),
         )
         tsdb.incr(
-            tsdb.models.project_total_forwarded, project.id, now, int(count * 1.1),
+            tsdb.models.project_total_forwarded,
+            project.id,
+            now,
+            int(count * 1.1),
         )
         tsdb.incr_multi(
             (

bin/merge-catalogs

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+
 import os
 import click
 

bin/mock-event

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# isort:skip_file
+
 from sentry.runner import configure
 
 configure()
@@ -28,7 +28,7 @@ def main(project, sample_type):
     if not project.first_event:
         project.update(first_event=timezone.now())
 
-    print("> Created event {}".format(event.event_id))
+    sys.stdout.write("> Created event {}\n".format(event.event_id))
 
 
 if __name__ == "__main__":

bin/mock-user

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# isort:skip_file
+
 from sentry.runner import configure
 
 configure()

config/hooks/post-merge

@@ -9,12 +9,13 @@ trap "rm -f ${files_changed_upstream}" EXIT
 
 git diff-tree -r --name-only --no-commit-id ORIG_HEAD HEAD > "$files_changed_upstream"
 
-grep -E --quiet 'requirements.*\.txt' "$files_changed_upstream" && py="install-py-dev "
-grep -E --quiet 'yarn.lock' "$files_changed_upstream"           && js="install-js-dev "
-grep -E --quiet 'migrations' "$files_changed_upstream"          && migrations="apply-migrations "
+grep -E --quiet 'requirements-pre-commit\.txt' "$files_changed_upstream" && pc="setup-git "
+grep -E --quiet 'requirements-(base|dev)\.txt' "$files_changed_upstream" && py="install-py-dev "
+grep -E --quiet 'yarn\.lock' "$files_changed_upstream"                   && js="install-js-dev "
+grep -E --quiet 'migrations' "$files_changed_upstream"                   && migrations="apply-migrations "
 
-[[ "$py" || "$js" || "$migrations" ]] && needs_update=1
-update_command="make ${py}${js}${migrations}"
+[[ "$pc" || "$py" || "$js" || "$migrations" ]] && needs_update=1
+update_command="make ${pc}${py}${js}${migrations}"
 
 [[ "$needs_update" ]] && cat <<EOF
 

config/hooks/pre-commit

@@ -1,12 +1,11 @@
 #!/usr/bin/env python
-from __future__ import absolute_import
 
 import os
 import sys
 
 from glob import glob
 
-text_type = type(u"")
+text_type = type("")
 
 # git usurbs your bin path for hooks and will always run system python
 if "VIRTUAL_ENV" in os.environ:

conftest.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import os
 import sys
 

docker/sentry.conf.py

@@ -1,5 +1,4 @@
 # flake8: noqa
-from __future__ import absolute_import
 
 # This file is just Python, with a touch of Django which means
 # you can inherit and tweak settings to your hearts content.

examples/oauth2_consumer_webserver/app.py

@@ -1,6 +1,4 @@
-from __future__ import absolute_import
-
-import json
+import json  # noqa
 import os
 import six
 

pyproject.toml

@@ -3,13 +3,6 @@ requires = ["setuptools>=40.2.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.black]
+# File filtering is taken care of in pre-commit.
 line-length=100
-target-version=['py27']
-include='\.py$'
-exclude='''
-(
-      \.venv/
-    | node_modules/
-    | migrations/
-)
-'''
+target-version=['py36']

requirements-pre-commit.txt

@@ -0,0 +1,3 @@
+pre-commit==2.9.3
+black==20.8b1
+sentry-flake8==0.4.0

setup.cfg

@@ -11,10 +11,11 @@ markers =
     snuba: mark a test as requiring snuba
 
 [flake8]
+# File filtering is taken care of in pre-commit.
+# TODO(joshuarli): B102 doesn't apply to Python 3. Could remove it from sentry-flake8 in a future release.
 # E203 false positive, see https://github.com/PyCQA/pycodestyle/issues/373
 # W605 false positive until python3.8: https://github.com/PyCQA/pycodestyle/issues/755
-ignore = F999,E203,E501,E128,E124,E402,W503,W504,W605,E731,C901,B007,B306,B009,B010
-exclude = .venv/.git,*/migrations/*,node_modules/*,src/sentry/static/sentry/vendor/*,docs/*,src/south/*,examples/*
+ignore = B102,F999,E203,E501,E128,E124,E402,W503,W504,W605,E731,C901,B007,B306,B009,B010
 
 # XXX: E501 is ignored, which disables line length checking.
 # Currently, the black formatter doesn't wrap long strings: https://github.com/psf/black/issues/182#issuecomment-385325274

setup.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import absolute_import
 
 import os
 import sys
@@ -81,7 +80,7 @@ def run(self):
 
 
 def get_requirements(env):
-    with open(u"requirements-{}.txt".format(env)) as fp:
+    with open("requirements-{}.txt".format(env)) as fp:
         return [x.strip() for x in fp.read().split("\n") if not x.startswith("#")]
 
 

src/bitfield/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from bitfield.models import Bit, BitHandler, BitField  # NOQA
 
 default_app_config = "bitfield.apps.BitFieldAppConfig"

src/bitfield/admin.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import six
 from django.contrib.admin import FieldListFilter
 from django.contrib.admin.options import IncorrectLookupParameters

src/bitfield/apps.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from django.apps import AppConfig
 
 

src/bitfield/compat.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 __all__ = ("bitand", "bitor")
 
 

src/bitfield/models.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import six
 
 from django.db.models.fields import BigIntegerField

src/bitfield/query.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from bitfield.types import Bit, BitHandler
 
 from django.db.models.lookups import Exact

src/bitfield/types.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import six
 
 from six import string_types

src/sentry/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import os
 import os.path
 

src/sentry/__main__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from .runner import main
 
 main()

src/sentry/admin.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from django import forms
 from django.conf import settings
 from django.conf.urls import url
@@ -40,7 +38,7 @@ class OptionAdmin(admin.ModelAdmin):
     search_fields = ("key",)
 
     def value_repr(self, instance):
-        return u'<pre style="display:inline-block;white-space:pre-wrap;">{}</pre>'.format(
+        return '<pre style="display:inline-block;white-space:pre-wrap;">{}</pre>'.format(
             escape(saferepr(instance.value))
         )
 

src/sentry/adoption/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from .manager import AdoptionManager
 
 manager = AdoptionManager()

src/sentry/adoption/manager.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from collections import defaultdict, namedtuple
 
 FEATURE_LOCATION = {"language", "integration", "code", "web", "admin", "api"}

src/sentry/analytics/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from django.conf import settings
 
 from sentry import options

src/sentry/analytics/base.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 __all__ = ("Analytics",)
 
 import six

src/sentry/analytics/event.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import, print_function
 from sentry.utils.compat import map
 
 __all__ = ("Attribute", "Event", "Map")
@@ -56,13 +55,13 @@ def extract(self, value):
         for attr in self.attributes:
             nv = items.pop(attr.name, None)
             if attr.required and nv is None:
-                raise ValueError(u"{} is required (cannot be None)".format(attr.name))
+                raise ValueError("{} is required (cannot be None)".format(attr.name))
 
             data[attr.name] = attr.extract(nv)
 
         if items:
             raise ValueError(
-                u"Unknown attributes: {}".format(", ".join(map(six.text_type, six.iterkeys(items))))
+                "Unknown attributes: {}".format(", ".join(map(six.text_type, six.iterkeys(items))))
             )
 
         return data
@@ -89,11 +88,11 @@ def __init__(self, type=None, datetime=None, **items):
         for attr in self.attributes:
             nv = items.pop(attr.name, None)
             if attr.required and nv is None:
-                raise ValueError(u"{} is required (cannot be None)".format(attr.name))
+                raise ValueError("{} is required (cannot be None)".format(attr.name))
             data[attr.name] = attr.extract(nv)
 
         if items:
-            raise ValueError(u"Unknown attributes: {}".format(", ".join(six.iterkeys(items))))
+            raise ValueError("Unknown attributes: {}".format(", ".join(six.iterkeys(items))))
 
         self.data = data
 

src/sentry/analytics/event_manager.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 __all__ = ("default_manager", "EventManager")
 
 

src/sentry/analytics/events/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import, print_function
-
 from sentry.utils.imports import import_submodules
 
 import_submodules(globals(), __name__, __path__)

src/sentry/analytics/events/advanced_search_feature_gated.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 from sentry import analytics