python: reformat and addition of format checks

.clang-format

@@ -10,13 +10,17 @@ AllowShortLoopsOnASingleLine : false
 BinPackParameters : false
 AllowAllParametersOfDeclarationOnNextLine : false
 AlignTrailingComments : true
-ColumnLimit : 80
-PenaltyBreakBeforeFirstCallParameter : 100
+ColumnLimit : 88
+
+# do not put all arguments on one line unless it's the same line as the call
+PenaltyBreakBeforeFirstCallParameter : 10000000
 PenaltyReturnTypeOnItsOwnLine : 65000
 PenaltyBreakString : 10
 
 # These improve formatting results but require clang 3.6/7 or higher
-# BreakBeforeBinaryOperators : NonAssignment
-# AlignAfterOpenBracket: true
-# BinPackArguments : false
-# AlignOperands : true
+BreakBeforeBinaryOperators : NonAssignment
+AlignAfterOpenBracket: true
+BinPackArguments : false
+AlignOperands : true
+BreakBeforeTernaryOperators : true
+AllowAllParametersOfDeclarationOnNextLine : false

.travis.yml

@@ -3,60 +3,84 @@ services: docker
 dist: trusty
 language: c
 
-matrix:
+jobs:
   include:
+    - stage: 'style checks'
+      name: 'python format'
+      language: 'python'
+      python: '3.6'
+      install: pip install --upgrade black
+      script: ./scripts/check-format
+    - stage: 'style checks'
+      name: 'python lint'
+      language: 'python'
+      python: '3.6'
+      install: pip install --upgrade pylint
+      script: pylint --rcfile=src/bindings/python/.pylintrc src/bindings/python/flux
     - name: "Ubuntu: no configure flags"
+      stage: test
       compiler: gcc
       env:
        - PYTHON_VERSION=2.7
     - name: "Ubuntu: py3.6 distcheck"
+      stage: test
       compiler: gcc
       env:
        - DISTCHECK=t
        - PYTHON_VERSION=3.6
     - name: "Ubuntu: gcc-8 --with-flux-security/caliper, distcheck"
+      stage: test
       compiler: gcc-8
       env:
        - CC=gcc-8
        - CXX=g++-8
        - ARGS="--with-flux-security --enable-caliper"
        - DISTCHECK=t
        - PYTHON_VERSION=2.7
-    - name: "Ubuntu: clang-6.0 chain-lint --with-flux-security --enable-pylint"
+    - name: "Ubuntu: clang-6.0 chain-lint --with-flux-security"
+      stage: test
       compiler: clang-6.0
       env:
        - CC=clang-6.0
        - CXX=clang++-6.0
        - chain_lint=t
-       - ARGS="--with-flux-security --enable-pylint"
+       - ARGS="--with-flux-security"
        - PYTHON_VERSION=2.7
     - name: "Ubuntu: COVERAGE=t, --with-flux-security --enable-caliper"
+      stage: test
       compiler: gcc
       env:
        - COVERAGE=t
        - ARGS="--with-flux-security --enable-caliper"
        - PYTHON_VERSION=2.7
     - name: "Ubuntu: TEST_INSTALL docker-deploy"
+      stage: test
       compiler: gcc
       env:
        - ARGS="--with-flux-security --enable-caliper"
        - TEST_INSTALL=t
        - DOCKER_TAG=t
        - PYTHON_VERSION=2.7
     - name: "Centos 7: --with-flux-security --enable-caliper docker-deploy"
+      stage: test
       compiler: gcc
       env:
        - ARGS="--with-flux-security --enable-caliper --prefix=/usr"
        - IMG=centos7-base
        - DOCKER_TAG=t
        - PYTHON_VERSION=2.7
     - name: "Centos 7: py3.4 --with-flux-security"
+      stage: test
       compiler: gcc
       env:
        - ARGS="--with-flux-security --prefix=/usr"
        - IMG=centos7-base
        - PYTHON_VERSION=3.4
 
+stages:
+  - 'style checks'
+  - test
+
 env:
   global:
    - TAP_DRIVER_QUIET=1
@@ -87,7 +111,7 @@ before_install:
      export TAGNAME="${DOCKERREPO}:${IMG}-${TRAVIS_TAG:-latest}"
      echo "Tagging new image $TAGNAME"
   fi
- 
+
 script:
  - |
   src/test/docker/docker-run-checks.sh -j2 \

scripts/check-format

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+if command -v black 2>&1 > /dev/null ; then
+  find src t -name '*.py' -print0 | xargs -0 black --check
+else
+  echo "black not found, failing"
+  exit 1
+fi

scripts/format

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if command -v black 2>&1 > /dev/null ; then
+  find src t -name '*.py' -print0 | xargs -0 black
+else
+  echo "black not found, python left unformatted"
+fi

src/bindings/python/.pylintrc

@@ -44,7 +44,7 @@ symbols=no
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=missing-docstring, too-few-public-methods, too-many-arguments, star-args, arguments-differ, no-member, import-error, no-name-in-module, useless-object-inheritance, fixme
+disable=missing-docstring, too-few-public-methods, too-many-arguments, star-args, arguments-differ, no-member, import-error, no-name-in-module, useless-object-inheritance, fixme, bad-continuation
 
 
 [REPORTS]
@@ -175,7 +175,7 @@ docstring-min-length=-1
 [FORMAT]
 
 # Maximum number of characters on a single line.
-max-line-length=80
+max-line-length=88
 
 # Regexp for a line that is allowed to be longer than the limit.
 ignore-long-lines=^\s*(# )?<?https?://\S+>?$

src/bindings/python/flux/__init__.py

@@ -5,12 +5,8 @@
 # pylint: disable=invalid-name
 def Flux(*args, **kwargs):
     import flux.core.handle
+
     return flux.core.handle.Flux(*args, **kwargs)
 
-__all__ = ['core',
-           'kvs',
-           'jsc',
-           'rpc',
-           'mrpc',
-           'constants',
-           'Flux', ]
+
+__all__ = ["core", "kvs", "jsc", "rpc", "mrpc", "constants", "Flux"]

src/bindings/python/flux/core/handle.py

@@ -7,6 +7,7 @@
 from flux.core.inner import raw
 from _flux._core import ffi, lib
 
+
 class Flux(Wrapper):
     """
   The general Flux handle class, create one of these to connect to the
@@ -17,12 +18,14 @@ class Flux(Wrapper):
 
     def __init__(self, url=ffi.NULL, flags=0, handle=None):
         super(Flux, self).__init__(
-            ffi, lib,
+            ffi,
+            lib,
             handle=handle,
             match=ffi.typeof(lib.flux_open).result,
             filter_match=False,
-            prefixes=['flux_', 'FLUX_',],
-            destructor=raw.flux_close,)
+            prefixes=["flux_", "FLUX_"],
+            destructor=raw.flux_close,
+        )
 
         if handle is None:
             self.handle = raw.flux_open(url, flags)
@@ -39,7 +42,7 @@ def log(self, level, fstring):
         """
         # Short-circuited because variadics can't be wrapped cleanly
         if isinstance(fstring, six.text_type):
-            fstring = fstring.encode('utf-8')
+            fstring = fstring.encode("utf-8")
         lib.flux_log(self.handle, level, fstring)
 
     def send(self, message, flags=0):
@@ -48,43 +51,39 @@ def send(self, message, flags=0):
             message = message.handle
         return self.flux_send(message, flags)
 
-    def recv(self,
-             type_mask=raw.FLUX_MSGTYPE_ANY,
-             match_tag=raw.FLUX_MATCHTAG_NONE,
-             topic_glob=None,
-             flags=0):
+    def recv(
+        self,
+        type_mask=raw.FLUX_MSGTYPE_ANY,
+        match_tag=raw.FLUX_MATCHTAG_NONE,
+        topic_glob=None,
+        flags=0,
+    ):
         """
         Receive a message, returns a flux.Message containing the result or None
         """
-        match = ffi.new('struct flux_match *', {
-            'typemask': type_mask,
-            'matchtag': match_tag,
-            'topic_glob': topic_glob if topic_glob is not None else ffi.NULL,
-        })
+        match = ffi.new(
+            "struct flux_match *",
+            {
+                "typemask": type_mask,
+                "matchtag": match_tag,
+                "topic_glob": topic_glob if topic_glob is not None else ffi.NULL,
+            },
+        )
         handle = self.flux_recv(match[0], flags)
         if handle is not None:
             return Message(handle=handle)
         return None
 
-    def rpc_send(self, topic,
-                 payload=None,
-                 nodeid=raw.FLUX_NODEID_ANY,
-                 flags=0):
+    def rpc_send(self, topic, payload=None, nodeid=raw.FLUX_NODEID_ANY, flags=0):
         """ Create and send an RPC in one step """
         with RPC(self, topic, payload, nodeid, flags) as rpc:
             return rpc.get()
 
-    def rpc_create(self, topic,
-                   payload=None,
-                   nodeid=raw.FLUX_NODEID_ANY,
-                   flags=0):
+    def rpc_create(self, topic, payload=None, nodeid=raw.FLUX_NODEID_ANY, flags=0):
         """ Create a new RPC object """
         return RPC(self, topic, payload, nodeid, flags)
 
-    def mrpc_create(self, topic,
-                    payload=None,
-                    rankset="any",
-                    flags=0):
+    def mrpc_create(self, topic, payload=None, rankset="any", flags=0):
         """
         Create a new MRPC object. Messages are sent on MRPC creation. Responses
         are accessible by iterating on the returned MRPC object. For example:
@@ -112,23 +111,27 @@ def event_send(self, topic, payload=None):
     def event_recv(self, topic=None):
         return self.recv(type_mask=raw.FLUX_MSGTYPE_EVENT, topic_glob=topic)
 
-    def msg_watcher_create(self, callback,
-                           type_mask=raw.FLUX_MSGTYPE_ANY,
-                           topic_glob='*',
-                           args=None,
-                           match_tag=raw.FLUX_MATCHTAG_NONE):
+    def msg_watcher_create(
+        self,
+        callback,
+        type_mask=raw.FLUX_MSGTYPE_ANY,
+        topic_glob="*",
+        args=None,
+        match_tag=raw.FLUX_MATCHTAG_NONE,
+    ):
         from flux.message import MessageWatcher
-        return MessageWatcher(self, type_mask, callback, topic_glob,
-                              match_tag, args)
+
+        return MessageWatcher(self, type_mask, callback, topic_glob, match_tag, args)
 
     def timer_watcher_create(self, after, callback, repeat=0.0, args=None):
         from flux.core.watchers import TimerWatcher
+
         return TimerWatcher(self, after, callback, repeat=repeat, args=args)
 
     def barrier(self, name, nprocs):
         self.flux_barrier(name, nprocs)
 
     def get_rank(self):
-        rank = ffi.new('uint32_t [1]')
+        rank = ffi.new("uint32_t [1]")
         self.flux_get_rank(rank)
         return rank[0]

src/bindings/python/flux/core/inner.py

@@ -9,12 +9,8 @@ class Core(Wrapper):
 
     def __init__(self):
         """Set up the wrapper interface for functions prefixed with flux_"""
-        super(Core, self).__init__(ffi,
-                                   lib,
-                                   prefixes=[
-                                       'flux_',
-                                       'FLUX_',
-                                   ])
+        super(Core, self).__init__(ffi, lib, prefixes=["flux_", "FLUX_"])
+
 
 # keeping this for compatibility
 # pylint: disable=invalid-name

src/bindings/python/flux/core/trampoline.py

@@ -8,8 +8,7 @@ def mod_main_trampoline(name, int_handle, args):
     flux_instance = Flux(handle=lib.unpack_long(int_handle))
     user_mod = None
     try:
-        user_mod = importlib.import_module(
-            'flux.modules.' + name, 'flux.modules')
+        user_mod = importlib.import_module("flux.modules." + name, "flux.modules")
     except ImportError:  # check user paths for the module
         user_mod = importlib.import_module(name)
 

src/bindings/python/flux/core/watchers.py

@@ -1,7 +1,7 @@
 import abc
 from flux.core.inner import raw, ffi
 
-__all__ = ['TimerWatcher', 'FDWatcher']
+__all__ = ["TimerWatcher", "FDWatcher"]
 
 
 class Watcher(object):
@@ -35,16 +35,16 @@ def destroy(self):
             raw.flux_watcher_destroy(self.handle)
             self.handle = None
 
-@ffi.callback('flux_watcher_f')
+
+@ffi.callback("flux_watcher_f")
 def timeout_handler_wrapper(unused1, unused2, revents, opaque_handle):
     del unused1, unused2  # unused arguments
     watcher = ffi.from_handle(opaque_handle)
     watcher.callback(watcher.flux_handle, watcher, revents, watcher.args)
 
 
 class TimerWatcher(Watcher):
-
-    def __init__(self, flux_handle, after, callback, repeat=0, args=None, ):
+    def __init__(self, flux_handle, after, callback, repeat=0, args=None):
         self.flux_handle = flux_handle
         self.after = after
         self.repeat = repeat
@@ -58,20 +58,20 @@ def __init__(self, flux_handle, after, callback, repeat=0, args=None, ):
                 float(after),
                 float(repeat),
                 timeout_handler_wrapper,
-                self.wargs))
+                self.wargs,
+            )
+        )
 
 
-@ffi.callback('flux_watcher_f')
+@ffi.callback("flux_watcher_f")
 def fd_handler_wrapper(unused1, unused2, revents, opaque_handle):
     del unused1, unused2  # unused arguments
     watcher = ffi.from_handle(opaque_handle)
     fd_int = raw.fd_watcher_get_fd(watcher.handle)
-    watcher.callback(watcher.flux_handle, watcher,
-                     fd_int, revents, watcher.args)
+    watcher.callback(watcher.flux_handle, watcher, fd_int, revents, watcher.args)
 
 
 class FDWatcher(Watcher):
-
     def __init__(self, flux_handle, fd_int, events, callback, args=None):
         self.flux_handle = flux_handle
         self.fd_int = fd_int
@@ -86,4 +86,6 @@ def __init__(self, flux_handle, fd_int, events, callback, args=None):
                 self.fd_int,
                 self.events,
                 fd_handler_wrapper,
-                self.wargs))
+                self.wargs,
+            )
+        )