Upgrade linter versions to the newest versions that still install und…

…er both Python 2 and Python 3.7 (#444) * Upgrade to newer versions of lint tools. Pin to versions compatible with both Py2 and 3. * Apply isort fixes. * Fix bandit and pylint. Add exclusion for pylint false positive. * Use pylint version compatible with Python 3.7
kensho-technologies · Aug 2, 2019 · 1edf355 · 1edf355
1 parent 1aac2a7
commit 1edf355
Show file tree

Hide file tree

Showing 13 changed files with 395 additions and 282 deletions.
diff --git a/Pipfile b/Pipfile
@@ -4,29 +4,33 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-bandit = "==1.5.1"
-coveralls = "==1.5.1"
 # The Cython dependency of pymssql is not correctly resolved on all systems
 # so it is explicitly included here.
 cython = "==0.29.2"
-flake8 = "==3.6.0"
-flake8-print = "==3.1.0"
-flake8-quotes = "==1.0.0"
+
+# Test requirements that are not otherwise necessary when using the package.
 frozendict = "==1.2"
-isort = "==4.3.4"
 mysqlclient = "==1.3.14"
 neo4j = ">=1.7.4,<2"
-parameterized = "==0.6.1"
 psycopg2 = "==2.7.5"
-pydocstyle = "==2.1.1"
-pylint = "==1.8.2"
 pymssql = "==2.1.4"
 pyorient = "==1.5.5"
-pytest = "==4.1.0"
-pytest-cov = "==2.6.1"
 redis = ">=3.2.1,<4"
 redisgraph = ">=1.7,<1.9"
-snapshottest = "==0.5.0"
+
+# Linters and other development tools
+bandit = ">=1.5.1,<2"
+coveralls = ">=1.5.1,<2"
+flake8 = ">=3.6.0,<4"
+flake8-print = ">=3.1.0,<4"
+flake8-quotes = ">=1.0.0,<2"
+isort = ">=4.3.4,<5"
+parameterized = ">=0.6.1,<1"
+pydocstyle = ">=2.1.1,<3"
+pylint = "==1.9.4"  # v1.9.5 is marked as "python_requires < 3.7", update this after we're Py3+ only
+pytest = ">=4.1.0,<5"
+pytest-cov = ">=2.6.1,<3"
+snapshottest = ">=0.5.1,<1"
 
 [packages]
 arrow = ">=0.10.0,<1"

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/Pipfile.py2.lock b/Pipfile.py2.lock
diff --git a/graphql_compiler/__init__.py b/graphql_compiler/__init__.py
@@ -1,27 +1,22 @@
 # Copyright 2017-present Kensho Technologies, LLC.
 """Commonly-used functions and data types from this package."""
 from .compiler import (  # noqa
-    CompilationResult,
-    OutputMetadata,
-    compile_graphql_to_gremlin,
-    compile_graphql_to_match,
-    compile_graphql_to_sql,
-    compile_graphql_to_cypher,
+    CompilationResult, OutputMetadata, compile_graphql_to_cypher, compile_graphql_to_gremlin,
+    compile_graphql_to_match, compile_graphql_to_sql
 )
-from .query_formatting import insert_arguments_into_query  # noqa
-from .query_formatting.graphql_formatting import pretty_print_graphql  # noqa
 from .exceptions import (  # noqa
     GraphQLCompilationError, GraphQLError, GraphQLInvalidArgumentError, GraphQLParsingError,
     GraphQLValidationError
 )
+from .query_formatting import insert_arguments_into_query  # noqa
+from .query_formatting.graphql_formatting import pretty_print_graphql  # noqa
 from .schema import (  # noqa
     DIRECTIVES, EXTENDED_META_FIELD_DEFINITIONS, GraphQLDate, GraphQLDateTime, GraphQLDecimal,
     insert_meta_fields_into_existing_schema, is_meta_field
 )
 from .schema_generation.graphql_schema import get_graphql_schema_from_schema_graph
-from .schema_generation.orientdb.schema_graph_builder import (
-    get_orientdb_schema_graph
-)
+from .schema_generation.orientdb.schema_graph_builder import get_orientdb_schema_graph
+
 
 __package_name__ = 'graphql-compiler'
 __version__ = '1.11.0'

diff --git a/graphql_compiler/compiler/__init__.py b/graphql_compiler/compiler/__init__.py
@@ -1,10 +1,7 @@
 # Copyright 2017-present Kensho Technologies, LLC.
-from .common import (  # noqa
-    CompilationResult,
-    compile_graphql_to_cypher,
-    compile_graphql_to_gremlin,
-    compile_graphql_to_match,
-    compile_graphql_to_sql,
+from .common import (  # noqa; noqa
+    CYPHER_LANGUAGE, GREMLIN_LANGUAGE, MATCH_LANGUAGE, SQL_LANGUAGE, CompilationResult,
+    compile_graphql_to_cypher, compile_graphql_to_gremlin, compile_graphql_to_match,
+    compile_graphql_to_sql
 )
-from .common import CYPHER_LANGUAGE, GREMLIN_LANGUAGE, MATCH_LANGUAGE, SQL_LANGUAGE  # noqa
 from .compiler_frontend import OutputMetadata  # noqa
diff --git a/graphql_compiler/compiler/ir_lowering_cypher/__init__.py b/graphql_compiler/compiler/ir_lowering_cypher/__init__.py
@@ -1,14 +1,14 @@
 # Copyright 2019-present Kensho Technologies, LLC.
-from ..ir_sanity_checks import sanity_check_ir_blocks_from_frontend
+from ..cypher_query import convert_to_cypher_query
 from ..ir_lowering_common.common import (
     lower_context_field_existence, merge_consecutive_filter_clauses,
     optimize_boolean_expression_comparisons
 )
+from ..ir_sanity_checks import sanity_check_ir_blocks_from_frontend
 from .ir_lowering import (
     insert_explicit_type_bounds, move_filters_in_optional_locations_to_global_operations,
     remove_mark_location_after_optional_backtrack, replace_local_fields_with_context_fields
 )
-from ..cypher_query import convert_to_cypher_query
 
 
 ##############

diff --git a/graphql_compiler/compiler/ir_lowering_gremlin/__init__.py b/graphql_compiler/compiler/ir_lowering_gremlin/__init__.py
@@ -1,11 +1,13 @@
 # Copyright 2018-present Kensho Technologies, LLC.
-from .ir_lowering import (lower_coerce_type_block_type_data, lower_coerce_type_blocks,
-                          lower_folded_outputs_and_context_fields,
-                          rewrite_filters_in_optional_blocks)
+from ..ir_lowering_common.common import (
+    lower_context_field_existence, merge_consecutive_filter_clauses,
+    optimize_boolean_expression_comparisons
+)
 from ..ir_sanity_checks import sanity_check_ir_blocks_from_frontend
-from ..ir_lowering_common.common import (lower_context_field_existence,
-                                         merge_consecutive_filter_clauses,
-                                         optimize_boolean_expression_comparisons)
+from .ir_lowering import (
+    lower_coerce_type_block_type_data, lower_coerce_type_blocks,
+    lower_folded_outputs_and_context_fields, rewrite_filters_in_optional_blocks
+)
 
 
 ##############

diff --git a/graphql_compiler/compiler/ir_lowering_match/__init__.py b/graphql_compiler/compiler/ir_lowering_match/__init__.py
@@ -2,29 +2,31 @@
 import six
 
 from ..blocks import Filter
-from ..ir_lowering_common.common import (extract_optional_location_root_info,
-                                         extract_simple_optional_location_info,
-                                         lower_context_field_existence,
-                                         merge_consecutive_filter_clauses,
-                                         optimize_boolean_expression_comparisons,
-                                         remove_end_optionals)
-from .ir_lowering import (lower_backtrack_blocks,
-                          lower_folded_coerce_types_into_filter_blocks,
-                          lower_has_substring_binary_compositions,
-                          remove_backtrack_blocks_from_fold,
-                          rewrite_binary_composition_inside_ternary_conditional,
-                          truncate_repeated_single_step_traversals,
-                          truncate_repeated_single_step_traversals_in_sub_queries)
+from ..ir_lowering_common.common import (
+    extract_optional_location_root_info, extract_simple_optional_location_info,
+    lower_context_field_existence, merge_consecutive_filter_clauses,
+    optimize_boolean_expression_comparisons, remove_end_optionals
+)
 from ..ir_sanity_checks import sanity_check_ir_blocks_from_frontend
-from .between_lowering import lower_comparisons_to_between
-from .optional_traversal import (collect_filters_to_first_location_occurrence,
-                                 convert_optional_traversals_to_compound_match_query,
-                                 lower_context_field_expressions, prune_non_existent_outputs)
 from ..match_query import convert_to_match_query
-from ..workarounds import (orientdb_class_with_while, orientdb_eval_scheduling,
-                           orientdb_query_execution)
+from ..workarounds import (
+    orientdb_class_with_while, orientdb_eval_scheduling, orientdb_query_execution
+)
+from .between_lowering import lower_comparisons_to_between
+from .ir_lowering import (
+    lower_backtrack_blocks, lower_folded_coerce_types_into_filter_blocks,
+    lower_has_substring_binary_compositions, remove_backtrack_blocks_from_fold,
+    rewrite_binary_composition_inside_ternary_conditional, truncate_repeated_single_step_traversals,
+    truncate_repeated_single_step_traversals_in_sub_queries
+)
+from .optional_traversal import (
+    collect_filters_to_first_location_occurrence,
+    convert_optional_traversals_to_compound_match_query, lower_context_field_expressions,
+    prune_non_existent_outputs
+)
 from .utils import construct_where_filter_predicate
 
+
 ##############
 # Public API #
 ##############

diff --git a/graphql_compiler/compiler/ir_lowering_sql/__init__.py b/graphql_compiler/compiler/ir_lowering_sql/__init__.py
@@ -2,12 +2,13 @@
 
 import six
 
-from .sql_tree import SqlNode, SqlQueryTree
 from .. import blocks
 from ...compiler import expressions
 from ...compiler.helpers import Location
 from ..ir_lowering_sql import constants
 from ..metadata import LocationInfo
+from .sql_tree import SqlNode, SqlQueryTree
+
 
 ##############
 # Public API #

diff --git a/graphql_compiler/cost_estimation/cardinality_estimator.py b/graphql_compiler/cost_estimation/cardinality_estimator.py
@@ -115,16 +115,23 @@ def _estimate_children_per_parent(
     )
     child_name_from_location = query_metadata.get_location_info(child_location).type.name
     if child_name_from_edge != child_name_from_location:
+        # False-positive bug in pylint: https://github.com/PyCQA/pylint/issues/3039
+        # pylint: disable=old-division
         edge_counts *= (
             float(lookup_class_counts(child_name_from_location)) /
             lookup_class_counts(child_name_from_edge)
         )
+        # pylint: enable=old-division
 
     # Count the number of parents, over which we assume the edges are uniformly distributed.
     parent_counts = lookup_class_counts(parent_name_from_edge)
 
+    # False-positive bug in pylint: https://github.com/PyCQA/pylint/issues/3039
+    # pylint: disable=old-division
+    #
     # TODO(evan): edges are not necessarily uniformly distributed, so record more statistics
     child_counts_per_parent = float(edge_counts) / parent_counts
+    # pylint: enable=old-division
 
     # Recursion always starts with depth = 0, so we should treat the parent result set itself as a
     # child result set to be expanded (so add 1 to child_counts).

diff --git a/graphql_compiler/tests/integration_tests/integration_backend_config.py b/graphql_compiler/tests/integration_tests/integration_backend_config.py
@@ -4,8 +4,8 @@
 from .. import test_backend
 
 
-DEFAULT_ROOT_PASSWORD = u'root'
-MSSQL_ROOT_PASSWORD = u'Root-secure1'  # mssql has stricter root password restrictions
+DEFAULT_ROOT_PASSWORD = u'root'  # nosec
+MSSQL_ROOT_PASSWORD = u'Root-secure1'  # mssql has stricter root password restrictions  # nosec
 
 SQL_BACKENDS = {
     test_backend.POSTGRES,

diff --git a/graphql_compiler/tests/test_data_tools/neo4j_graph.py b/graphql_compiler/tests/test_data_tools/neo4j_graph.py
@@ -6,7 +6,7 @@
 NEO4J_SERVER = 'localhost'
 NEO4J_PORT = 7687
 NEO4J_USER = 'neo4j'
-NEO4J_PASSWORD = 'root'
+NEO4J_PASSWORD = 'root'  # nosec
 
 
 class Neo4jClient(object):

diff --git a/graphql_compiler/tests/test_data_tools/orientdb_graph.py b/graphql_compiler/tests/test_data_tools/orientdb_graph.py
@@ -7,7 +7,7 @@
 ORIENTDB_SERVER = 'localhost'
 ORIENTDB_PORT = 2424
 ORIENTDB_USER = 'root'
-ORIENTDB_PASSWORD = 'root'
+ORIENTDB_PASSWORD = 'root'  # nosec
 
 
 def get_orientdb_url(database_name):