Blacken .py files in blib2to3 (#1011)

* Blacken .py files in blib2to3

This is in preparation for adding type annotations to blib2to3 in
order to compiling it with mypyc (#1009, which I can rebase on top of
this).

To enforce that it stays blackened, I just cargo-culted the existing
test code used for validating formatting. It feels pretty clunky now,
though, so I can abstract the common logic out into a helper if that
seems better. (But error messages might be less clear then?)

* Tidy up the tests

blib2to3/__init__.py

@@ -1 +1 @@
-#empty
+# empty

blib2to3/pgen2/conv.py

@@ -70,8 +70,7 @@ def parse_graminit_h(self, filename):
             lineno += 1
             mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line)
             if not mo and line.strip():
-                print("%s(%s): can't parse %s" % (filename, lineno,
-                                                  line.strip()))
+                print("%s(%s): can't parse %s" % (filename, lineno, line.strip()))
             else:
                 symbol, number = mo.groups()
                 number = int(number)
@@ -118,49 +117,48 @@ def parse_graminit_c(self, filename):
         lineno = 0
 
         # Expect the two #include lines
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         assert line == '#include "pgenheaders.h"\n', (lineno, line)
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         assert line == '#include "grammar.h"\n', (lineno, line)
 
         # Parse the state definitions
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         allarcs = {}
         states = []
         while line.startswith("static arc "):
             while line.startswith("static arc "):
-                mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$",
-                              line)
+                mo = re.match(r"static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$", line)
                 assert mo, (lineno, line)
                 n, m, k = list(map(int, mo.groups()))
                 arcs = []
                 for _ in range(k):
-                    lineno, line = lineno+1, next(f)
+                    lineno, line = lineno + 1, next(f)
                     mo = re.match(r"\s+{(\d+), (\d+)},$", line)
                     assert mo, (lineno, line)
                     i, j = list(map(int, mo.groups()))
                     arcs.append((i, j))
-                lineno, line = lineno+1, next(f)
+                lineno, line = lineno + 1, next(f)
                 assert line == "};\n", (lineno, line)
                 allarcs[(n, m)] = arcs
-                lineno, line = lineno+1, next(f)
+                lineno, line = lineno + 1, next(f)
             mo = re.match(r"static state states_(\d+)\[(\d+)\] = {$", line)
             assert mo, (lineno, line)
             s, t = list(map(int, mo.groups()))
             assert s == len(states), (lineno, line)
             state = []
             for _ in range(t):
-                lineno, line = lineno+1, next(f)
+                lineno, line = lineno + 1, next(f)
                 mo = re.match(r"\s+{(\d+), arcs_(\d+)_(\d+)},$", line)
                 assert mo, (lineno, line)
                 k, n, m = list(map(int, mo.groups()))
                 arcs = allarcs[n, m]
                 assert k == len(arcs), (lineno, line)
                 state.append(arcs)
             states.append(state)
-            lineno, line = lineno+1, next(f)
+            lineno, line = lineno + 1, next(f)
             assert line == "};\n", (lineno, line)
-            lineno, line = lineno+1, next(f)
+            lineno, line = lineno + 1, next(f)
         self.states = states
 
         # Parse the dfas
@@ -169,9 +167,8 @@ def parse_graminit_c(self, filename):
         assert mo, (lineno, line)
         ndfas = int(mo.group(1))
         for i in range(ndfas):
-            lineno, line = lineno+1, next(f)
-            mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$',
-                          line)
+            lineno, line = lineno + 1, next(f)
+            mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$', line)
             assert mo, (lineno, line)
             symbol = mo.group(2)
             number, x, y, z = list(map(int, mo.group(1, 3, 4, 5)))
@@ -180,29 +177,29 @@ def parse_graminit_c(self, filename):
             assert x == 0, (lineno, line)
             state = states[z]
             assert y == len(state), (lineno, line)
-            lineno, line = lineno+1, next(f)
+            lineno, line = lineno + 1, next(f)
             mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line)
             assert mo, (lineno, line)
             first = {}
             rawbitset = eval(mo.group(1))
             for i, c in enumerate(rawbitset):
                 byte = ord(c)
                 for j in range(8):
-                    if byte & (1<<j):
-                        first[i*8 + j] = 1
+                    if byte & (1 << j):
+                        first[i * 8 + j] = 1
             dfas[number] = (state, first)
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         assert line == "};\n", (lineno, line)
         self.dfas = dfas
 
         # Parse the labels
         labels = []
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         mo = re.match(r"static label labels\[(\d+)\] = {$", line)
         assert mo, (lineno, line)
         nlabels = int(mo.group(1))
         for i in range(nlabels):
-            lineno, line = lineno+1, next(f)
+            lineno, line = lineno + 1, next(f)
             mo = re.match(r'\s+{(\d+), (0|"\w+")},$', line)
             assert mo, (lineno, line)
             x, y = mo.groups()
@@ -212,44 +209,44 @@ def parse_graminit_c(self, filename):
             else:
                 y = eval(y)
             labels.append((x, y))
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         assert line == "};\n", (lineno, line)
         self.labels = labels
 
         # Parse the grammar struct
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         assert line == "grammar _PyParser_Grammar = {\n", (lineno, line)
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         mo = re.match(r"\s+(\d+),$", line)
         assert mo, (lineno, line)
         ndfas = int(mo.group(1))
         assert ndfas == len(self.dfas)
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         assert line == "\tdfas,\n", (lineno, line)
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         mo = re.match(r"\s+{(\d+), labels},$", line)
         assert mo, (lineno, line)
         nlabels = int(mo.group(1))
         assert nlabels == len(self.labels), (lineno, line)
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         mo = re.match(r"\s+(\d+)$", line)
         assert mo, (lineno, line)
         start = int(mo.group(1))
         assert start in self.number2symbol, (lineno, line)
         self.start = start
-        lineno, line = lineno+1, next(f)
+        lineno, line = lineno + 1, next(f)
         assert line == "};\n", (lineno, line)
         try:
-            lineno, line = lineno+1, next(f)
+            lineno, line = lineno + 1, next(f)
         except StopIteration:
             pass
         else:
             assert 0, (lineno, line)
 
     def finish_off(self):
         """Create additional useful structures.  (Internal)."""
-        self.keywords = {} # map from keyword strings to arc labels
-        self.tokens = {}   # map from numeric token values to arc labels
+        self.keywords = {}  # map from keyword strings to arc labels
+        self.tokens = {}  # map from numeric token values to arc labels
         for ilabel, (type, value) in enumerate(self.labels):
             if type == token.NAME and value is not None:
                 self.keywords[value] = ilabel

blib2to3/pgen2/driver.py

@@ -28,13 +28,7 @@
 
 
 class Driver(object):
-
-    def __init__(
-        self,
-        grammar,
-        convert=None,
-        logger=None,
-    ):
+    def __init__(self, grammar, convert=None, logger=None):
         self.grammar = grammar
         if logger is None:
             logger = logging.getLogger(__name__)
@@ -73,8 +67,9 @@ def parse_tokens(self, tokens, debug=False):
             if type == token.OP:
                 type = grammar.opmap[value]
             if debug:
-                self.logger.debug("%s %r (prefix=%r)",
-                                  token.tok_name[type], value, prefix)
+                self.logger.debug(
+                    "%s %r (prefix=%r)", token.tok_name[type], value, prefix
+                )
             if type == token.INDENT:
                 indent_columns.append(len(value))
                 _prefix = prefix + value
@@ -96,8 +91,7 @@ def parse_tokens(self, tokens, debug=False):
                 column = 0
         else:
             # We never broke out -- EOF is too soon (how can this happen???)
-            raise parse.ParseError("incomplete input",
-                                   type, value, (prefix, start))
+            raise parse.ParseError("incomplete input", type, value, (prefix, start))
         return p.rootnode
 
     def parse_stream_raw(self, stream, debug=False):
@@ -117,8 +111,7 @@ def parse_file(self, filename, encoding=None, debug=False):
     def parse_string(self, text, debug=False):
         """Parse a string and return the syntax tree."""
         tokens = tokenize.generate_tokens(
-            io.StringIO(text).readline,
-            grammar=self.grammar
+            io.StringIO(text).readline, grammar=self.grammar
         )
         return self.parse_tokens(tokens, debug)
 
@@ -130,24 +123,24 @@ def _partially_consume_prefix(self, prefix, column):
         for char in prefix:
             current_line += char
             if wait_for_nl:
-                if char == '\n':
+                if char == "\n":
                     if current_line.strip() and current_column < column:
-                        res = ''.join(lines)
-                        return res, prefix[len(res):]
+                        res = "".join(lines)
+                        return res, prefix[len(res) :]
 
                     lines.append(current_line)
                     current_line = ""
                     current_column = 0
                     wait_for_nl = False
-            elif char in ' \t':
+            elif char in " \t":
                 current_column += 1
-            elif char == '\n':
+            elif char == "\n":
                 # unexpected empty line
                 current_column = 0
             else:
                 # indent is finished
                 wait_for_nl = True
-        return ''.join(lines), current_line
+        return "".join(lines), current_line
 
 
 def _generate_pickle_name(gt, cache_dir=None):
@@ -161,8 +154,7 @@ def _generate_pickle_name(gt, cache_dir=None):
         return name
 
 
-def load_grammar(gt="Grammar.txt", gp=None,
-                 save=True, force=False, logger=None):
+def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None):
     """Load the grammar (maybe from a pickle)."""
     if logger is None:
         logger = logging.getLogger(__name__)
@@ -219,11 +211,11 @@ def main(*args):
     """
     if not args:
         args = sys.argv[1:]
-    logging.basicConfig(level=logging.INFO, stream=sys.stdout,
-                        format='%(message)s')
+    logging.basicConfig(level=logging.INFO, stream=sys.stdout, format="%(message)s")
     for gt in args:
         load_grammar(gt, save=True, force=True)
     return True
 
+
 if __name__ == "__main__":
     sys.exit(int(not main()))

blib2to3/pgen2/grammar.py

@@ -90,7 +90,9 @@ def __init__(self):
 
     def dump(self, filename):
         """Dump the grammar tables to a pickle file."""
-        with tempfile.NamedTemporaryFile(dir=os.path.dirname(filename), delete=False) as f:
+        with tempfile.NamedTemporaryFile(
+            dir=os.path.dirname(filename), delete=False
+        ) as f:
             pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL)
         os.replace(f.name, filename)
 
@@ -109,8 +111,14 @@ def copy(self):
         Copy the grammar.
         """
         new = self.__class__()
-        for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
-                          "tokens", "symbol2label"):
+        for dict_attr in (
+            "symbol2number",
+            "number2symbol",
+            "dfas",
+            "keywords",
+            "tokens",
+            "symbol2label",
+        ):
             setattr(new, dict_attr, getattr(self, dict_attr).copy())
         new.labels = self.labels[:]
         new.states = self.states[:]
@@ -121,6 +129,7 @@ def copy(self):
     def report(self):
         """Dump the grammar tables to standard output, for debugging."""
         from pprint import pprint
+
         print("s2n")
         pprint(self.symbol2number)
         print("n2s")

blib2to3/pgen2/literals.py

@@ -5,16 +5,19 @@
 
 import regex as re
 
-simple_escapes = {"a": "\a",
-                  "b": "\b",
-                  "f": "\f",
-                  "n": "\n",
-                  "r": "\r",
-                  "t": "\t",
-                  "v": "\v",
-                  "'": "'",
-                  '"': '"',
-                  "\\": "\\"}
+simple_escapes = {
+    "a": "\a",
+    "b": "\b",
+    "f": "\f",
+    "n": "\n",
+    "r": "\r",
+    "t": "\t",
+    "v": "\v",
+    "'": "'",
+    '"': '"',
+    "\\": "\\",
+}
+
 
 def escape(m):
     all, tail = m.group(0, 1)
@@ -37,16 +40,18 @@ def escape(m):
             raise ValueError("invalid octal string escape ('\\%s')" % tail) from None
     return chr(i)
 
+
 def evalString(s):
     assert s.startswith("'") or s.startswith('"'), repr(s[:1])
     q = s[0]
-    if s[:3] == q*3:
-        q = q*3
-    assert s.endswith(q), repr(s[-len(q):])
-    assert len(s) >= 2*len(q)
-    s = s[len(q):-len(q)]
+    if s[:3] == q * 3:
+        q = q * 3
+    assert s.endswith(q), repr(s[-len(q) :])
+    assert len(s) >= 2 * len(q)
+    s = s[len(q) : -len(q)]
     return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s)
 
+
 def test():
     for i in range(256):
         c = chr(i)