diff --git a/pyoozie/model.py b/pyoozie/model.py index 9d2badc..c65d6a9 100644 --- a/pyoozie/model.py +++ b/pyoozie/model.py @@ -252,7 +252,7 @@ class _OozieArtifact(object): REQUIRED_KEYS = {} # type: typing.Dict[unicode, typing.Callable] - SUPPORTED_KEYS = {'toString': None} # type: typing.Dict[unicode, typing.Callable] + SUPPORTED_KEYS = {'toString': None} # type: typing.Dict[unicode, typing.Optional[typing.Callable]] def __init__(self, oozie_client, details, parent=None): self._client = oozie_client diff --git a/pyoozie/tags.py b/pyoozie/tags.py index 12f61d7..c5eb4db 100644 --- a/pyoozie/tags.py +++ b/pyoozie/tags.py @@ -1,17 +1,17 @@ # Copyright (c) 2017 "Shopify inc." All rights reserved. # Use of this source code is governed by a MIT-style license that can be found in the LICENSE file. from __future__ import unicode_literals + import abc import collections import copy import datetime +import enum import itertools import re import string # pylint: disable=deprecated-module -import uuid - -import enum import typing # pylint: disable=unused-import +import uuid import six import yattag @@ -496,7 +496,7 @@ def _xml(self, doc, tag, text): return doc -class _AbstractWorkflowEntity(typing.Iterable): +class _AbstractWorkflowEntity(collections.Iterable): """An abstract object representing an Oozie workflow action that can be serialized to XML.""" # pylint: disable=abstract-method @@ -504,15 +504,15 @@ class _AbstractWorkflowEntity(typing.Iterable): def __init__( self, - xml_tag=None, # type: typing.Optional[typing.Text] + xml_tag, # type: typing.Text name=None, # type: typing.Optional[typing.Text] on_error=None # type: typing.Optional[_AbstractWorkflowEntity] ): # type: (...) -> None - self.__xml_tag = xml_tag + self.__xml_tag = xml_tag or 'unknown' self.__name = name if name else uuid.uuid4().hex[:8] self.__on_error = copy.deepcopy(on_error) - self.__identifier = self.create_identifier(xml_tag) + self.__identifier = self.create_identifier(self.__xml_tag) def xml_tag(self): # type: () -> typing.Text @@ -526,8 +526,14 @@ def create_identifier(self, xml_tag): # type: (typing.Text) -> typing.Text return validate_xml_id('{tag}-{name}'.format(tag=xml_tag, name=self.__name)) - def _xml_and_get_on_error(self, doc, tag, text, on_next, on_error): - # type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc + def _xml_and_get_on_error( + self, + doc, # type: yattag.doc.Doc + tag, # type: yattag.doc.Doc.tag + text, # type: yattag.doc.Doc.Text + on_next, # type: typing.Text + on_error # type: typing.Optional[typing.Text] + ): if self.__on_error: self.__on_error._xml(doc, tag, text, on_next, on_error) return self.__on_error.identifier() if self.__on_error else ( @@ -535,8 +541,14 @@ def _xml_and_get_on_error(self, doc, tag, text, on_next, on_error): ) @abc.abstractmethod - def _xml(self, doc, tag, text, on_next, on_error): - # type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc + def _xml( + self, + doc, # type: yattag.doc.Doc + tag, # type: yattag.doc.Doc.tag + text, # type: yattag.doc.Doc.Text + on_next, # type: typing.Text + on_error # type: typing.Optional[typing.Text] + ): raise NotImplementedError() def __iter__(self): @@ -564,8 +576,14 @@ def __init__(self, message, name=None): super(Kill, self).__init__(xml_tag='kill', name=name) self.message = message - def _xml(self, doc, tag, text, on_next, on_error): - # type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc + def _xml( + self, + doc, # type: yattag.doc.Doc + tag, # type: yattag.doc.Doc.tag + text, # type: yattag.doc.Doc.Text + on_next, # type: typing.Text + on_error # type: typing.Optional[typing.Text] + ): with tag(self.xml_tag(), name=self.identifier()): with tag('message'): doc.text(self.message) @@ -600,8 +618,14 @@ def credential(self): # type: () -> typing.Optional[typing.Text] return self.__credential - def _xml(self, doc, tag, text, on_next, on_error): - # type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc + def _xml( + self, + doc, # type: yattag.doc.Doc + tag, # type: yattag.doc.Doc.tag + text, # type: yattag.doc.Doc.Text + on_next, # type: typing.Text + on_error # type: typing.Optional[typing.Text] + ): _on_error = self._xml_and_get_on_error(doc, tag, text, on_next, on_error) attributes = { @@ -638,8 +662,14 @@ def __init__( self.__default = copy.deepcopy(default) self.__choices = copy.deepcopy(choices) - def _xml(self, doc, tag, text, on_next, on_error): - # type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc + def _xml( + self, + doc, # type: yattag.doc.Doc + tag, # type: yattag.doc.Doc.tag + text, # type: yattag.doc.Doc.Text + on_next, # type: typing.Text + on_error # type: typing.Optional[typing.Text] + ): _on_error = self._xml_and_get_on_error(doc, tag, text, on_next, on_error) # Write switch/case @@ -671,14 +701,20 @@ class Serial(_AbstractWorkflowEntity): def __init__(self, *entities, **kwargs): # type: (*_AbstractWorkflowEntity, **_AbstractWorkflowEntity) -> None - super(Serial, self).__init__(on_error=kwargs.get(str('on_error'))) + super(Serial, self).__init__(xml_tag='unknown', on_error=kwargs.get(str('on_error'))) self.__entities = tuple(copy.deepcopy(entities)) # type: typing.Tuple[_AbstractWorkflowEntity, ...] def identifier(self): # type: () -> typing.Text - return self.__entities[0].identifier() if self.__entities else None + return self.__entities[0].identifier() - def _xml(self, doc, tag, text, on_next, on_error): - # type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc + def _xml( + self, + doc, # type: yattag.doc.Doc + tag, # type: yattag.doc.Doc.tag + text, # type: yattag.doc.Doc.Text + on_next, # type: typing.Text + on_error # type: typing.Optional[typing.Text] + ): _on_error = self._xml_and_get_on_error(doc, tag, text, on_next, on_error) entity_nextidentifier = zip(self.__entities, itertools.chain( (a.identifier() for a in self.__entities[1:]), @@ -714,8 +750,14 @@ def __init__(self, *entities, **kwargs): assert entities, 'At least 1 entity required' self.__entities = frozenset(copy.deepcopy(entities)) # type: typing.FrozenSet[_AbstractWorkflowEntity] - def _xml(self, doc, tag, text, on_next, on_error): - # type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc + def _xml( + self, + doc, # type: yattag.doc.Doc + tag, # type: yattag.doc.Doc.tag + text, # type: yattag.doc.Doc.Text + on_next, # type: typing.Text + on_error # type: typing.Optional[typing.Text] + ): _on_error = self._xml_and_get_on_error(doc, tag, text, on_next, on_error) with tag(self.xml_tag(), name=self.identifier()): for entity in self.__entities: diff --git a/setup.py b/setup.py index cfee621..ea6817e 100755 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def get_version(): 'mock', 'pycodestyle == 2.2.0', 'pylint>=1.7.1,<1.8', - 'pytest-cov', + 'pytest-cov>=2.4.0,<2.6', # pinned, see https://github.com/z4r/python-coveralls/issues/66 'pytest-randomly', 'pytest>=3.0', 'requests-mock', @@ -54,7 +54,10 @@ def get_version(): 'xmltodict', 'pywebhdfs>=0.4.1', ], - 'test: python_version >= "3.3"': [ + 'test: python_version ~= "3.4"': [ # mypy 0.700 dropped python 3.4 support + 'mypy == 0.670', + ], + 'test: python_version >= "3.5"': [ 'mypy', ], 'docs': [