Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix CI errors on master #73

Merged
merged 7 commits into from Apr 11, 2019
Merged

Fix CI errors on master #73

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f715d6e
fix type errors
Apr 10, 2019
9054831
fix pytest-cov issue
Apr 10, 2019
28dc9a5
fix 3.7-dev type issue
Apr 10, 2019
16f2832
pin pytest-cov to specific version
Apr 10, 2019
7ec4e95
try to fix mypy 3.4 error
Apr 10, 2019
6168299
Revert "pin pytest-cov to specific version"
Apr 10, 2019
ed27975
Make xml_tag required
cfournie Apr 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyoozie/model.py
View file
Open in desktop
Expand Up @@ -252,7 +252,7 @@ class _OozieArtifact(object):

REQUIRED_KEYS = {} # type: typing.Dict[unicode, typing.Callable]

SUPPORTED_KEYS = {'toString': None} # type: typing.Dict[unicode, typing.Callable]
SUPPORTED_KEYS = {'toString': None} # type: typing.Dict[unicode, typing.Optional[typing.Callable]]

def __init__(self, oozie_client, details, parent=None):
self._client = oozie_client
Expand Down
84 changes: 63 additions & 21 deletions pyoozie/tags.py
View file
Open in desktop
@@ -1,17 +1,17 @@
# Copyright (c) 2017 "Shopify inc." All rights reserved.
# Use of this source code is governed by a MIT-style license that can be found in the LICENSE file.
from __future__ import unicode_literals

import abc
import collections
import copy
import datetime
import enum
import itertools
import re
import string # pylint: disable=deprecated-module
import uuid

import enum
import typing # pylint: disable=unused-import
import uuid

import six
import yattag
Expand Down Expand Up @@ -496,7 +496,7 @@ def _xml(self, doc, tag, text):
return doc


class _AbstractWorkflowEntity(typing.Iterable):
class _AbstractWorkflowEntity(collections.Iterable):
"""An abstract object representing an Oozie workflow action that can be serialized to XML."""
# pylint: disable=abstract-method

Expand All @@ -509,10 +509,10 @@ def __init__(
on_error=None # type: typing.Optional[_AbstractWorkflowEntity]
):
# type: (...) -> None
self.__xml_tag = xml_tag
self.__xml_tag = xml_tag or 'unknown'
self.__name = name if name else uuid.uuid4().hex[:8]
self.__on_error = copy.deepcopy(on_error)
self.__identifier = self.create_identifier(xml_tag)
self.__identifier = self.create_identifier(self.__xml_tag)

def xml_tag(self):
# type: () -> typing.Text
Expand All @@ -526,17 +526,29 @@ def create_identifier(self, xml_tag):
# type: (typing.Text) -> typing.Text
return validate_xml_id('{tag}-{name}'.format(tag=xml_tag, name=self.__name))

def _xml_and_get_on_error(self, doc, tag, text, on_next, on_error):
# type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc
def _xml_and_get_on_error(
self,
doc, # type: yattag.doc.Doc
tag, # type: yattag.doc.Doc.tag
text, # type: yattag.doc.Doc.Text
on_next, # type: typing.Text
on_error # type: typing.Optional[typing.Text]
):
if self.__on_error:
self.__on_error._xml(doc, tag, text, on_next, on_error)
return self.__on_error.identifier() if self.__on_error else (
on_error if on_error else on_next
)

@abc.abstractmethod
def _xml(self, doc, tag, text, on_next, on_error):
# type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc
def _xml(
self,
doc, # type: yattag.doc.Doc
tag, # type: yattag.doc.Doc.tag
text, # type: yattag.doc.Doc.Text
on_next, # type: typing.Text
on_error # type: typing.Optional[typing.Text]
):
raise NotImplementedError()

def __iter__(self):
Expand Down Expand Up @@ -564,8 +576,14 @@ def __init__(self, message, name=None):
super(Kill, self).__init__(xml_tag='kill', name=name)
self.message = message

def _xml(self, doc, tag, text, on_next, on_error):
# type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc
def _xml(
self,
doc, # type: yattag.doc.Doc
tag, # type: yattag.doc.Doc.tag
text, # type: yattag.doc.Doc.Text
on_next, # type: typing.Text
on_error # type: typing.Optional[typing.Text]
):
with tag(self.xml_tag(), name=self.identifier()):
with tag('message'):
doc.text(self.message)
Expand Down Expand Up @@ -600,8 +618,14 @@ def credential(self):
# type: () -> typing.Optional[typing.Text]
return self.__credential

def _xml(self, doc, tag, text, on_next, on_error):
# type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc
def _xml(
self,
doc, # type: yattag.doc.Doc
tag, # type: yattag.doc.Doc.tag
text, # type: yattag.doc.Doc.Text
on_next, # type: typing.Text
on_error # type: typing.Optional[typing.Text]
):
_on_error = self._xml_and_get_on_error(doc, tag, text, on_next, on_error)

attributes = {
Expand Down Expand Up @@ -638,8 +662,14 @@ def __init__(
self.__default = copy.deepcopy(default)
self.__choices = copy.deepcopy(choices)

def _xml(self, doc, tag, text, on_next, on_error):
# type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc
def _xml(
self,
doc, # type: yattag.doc.Doc
tag, # type: yattag.doc.Doc.tag
text, # type: yattag.doc.Doc.Text
on_next, # type: typing.Text
on_error # type: typing.Optional[typing.Text]
):
_on_error = self._xml_and_get_on_error(doc, tag, text, on_next, on_error)

# Write switch/case
Expand Down Expand Up @@ -675,10 +705,16 @@ def __init__(self, *entities, **kwargs):
self.__entities = tuple(copy.deepcopy(entities)) # type: typing.Tuple[_AbstractWorkflowEntity, ...]

def identifier(self): # type: () -> typing.Text
return self.__entities[0].identifier() if self.__entities else None
return self.__entities[0].identifier() if self.__entities else ''

def _xml(self, doc, tag, text, on_next, on_error):
# type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc
def _xml(
self,
doc, # type: yattag.doc.Doc
tag, # type: yattag.doc.Doc.tag
text, # type: yattag.doc.Doc.Text
on_next, # type: typing.Text
on_error # type: typing.Optional[typing.Text]
):
_on_error = self._xml_and_get_on_error(doc, tag, text, on_next, on_error)
entity_nextidentifier = zip(self.__entities, itertools.chain(
(a.identifier() for a in self.__entities[1:]),
Expand Down Expand Up @@ -714,8 +750,14 @@ def __init__(self, *entities, **kwargs):
assert entities, 'At least 1 entity required'
self.__entities = frozenset(copy.deepcopy(entities)) # type: typing.FrozenSet[_AbstractWorkflowEntity]

def _xml(self, doc, tag, text, on_next, on_error):
# type: (yattag.doc.Doc, yattag.doc.Doc.tag, yattag.doc.Doc.text, typing.Text, typing.Text) -> yattag.doc.Doc
def _xml(
self,
doc, # type: yattag.doc.Doc
tag, # type: yattag.doc.Doc.tag
text, # type: yattag.doc.Doc.Text
on_next, # type: typing.Text
on_error # type: typing.Optional[typing.Text]
):
_on_error = self._xml_and_get_on_error(doc, tag, text, on_next, on_error)
with tag(self.xml_tag(), name=self.identifier()):
for entity in self.__entities:
Expand Down
7 changes: 5 additions & 2 deletions setup.py
View file
Open in desktop
Expand Up @@ -46,15 +46,18 @@ def get_version():
'mock',
'pycodestyle == 2.2.0',
'pylint>=1.7.1,<1.8',
'pytest-cov',
'pytest-cov>=2.4.0,<2.6', # pinned, see https://github.com/z4r/python-coveralls/issues/66
'pytest-randomly',
'pytest>=3.0',
'requests-mock',
'shopify_python==0.4.1',
'xmltodict',
'pywebhdfs>=0.4.1',
],
'test: python_version >= "3.3"': [
'test: python_version ~= "3.4"': [ # mypy 0.700 dropped python 3.4 support
'mypy == 0.670',
],
'test: python_version >= "3.5"': [
'mypy',
],
'docs': [
Expand Down