diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 4f83ae1d..93454fb8 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -8,7 +8,7 @@ from loader import * from dumper import * -__version__ = '4.1' +__version__ = '3.13' try: from cyaml import * @@ -16,6 +16,44 @@ except ImportError: __with_libyaml__ = False + +#------------------------------------------------------------------------------ +# Warnings control +#------------------------------------------------------------------------------ + +# 'Global' warnings state: +_warnings_enabled = { + 'YAMLLoadWarning': True, +} + +# Get or set global warnings' state +def warnings(settings=None): + if settings is None: + return _warnings_enabled + + if type(settings) is dict: + for key in settings: + if key in _warnings_enabled: + _warnings_enabled[key] = settings[key] + +# Warn when load() is called without Loader=... +class YAMLLoadWarning(RuntimeWarning): + pass + +def load_warning(method): + if _warnings_enabled['YAMLLoadWarning'] is False: + return + + import warnings + + message = "\n\ + *** Calling yaml.%s() without Loader=... is deprecated.\n\ + *** The default Loader is unsafe.\n\ + *** Please read https://msg.pyyaml.org/load for full details." % method + + warnings.warn(message, YAMLLoadWarning, stacklevel=3) + +#------------------------------------------------------------------------------ def scan(stream, Loader=Loader): """ Scan a YAML stream and produce scanning tokens. @@ -61,22 +99,30 @@ def compose_all(stream, Loader=Loader): finally: loader.dispose() -def load(stream, Loader=Loader): +def load(stream, Loader=None): """ Parse the first YAML document in a stream and produce the corresponding Python object. """ + if Loader is None: + load_warning('load') + Loader = FullLoader + loader = Loader(stream) try: return loader.get_single_data() finally: loader.dispose() -def load_all(stream, Loader=Loader): +def load_all(stream, Loader=None): """ Parse all YAML documents in a stream and produce corresponding Python objects. """ + if Loader is None: + load_warning('load_all') + Loader = FullLoader + loader = Loader(stream) try: while loader.check_data(): @@ -84,11 +130,33 @@ def load_all(stream, Loader=Loader): finally: loader.dispose() +def full_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load(stream, FullLoader) + +def full_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load_all(stream, FullLoader) + def safe_load(stream): """ Parse the first YAML document in a stream and produce the corresponding Python object. - Resolve only basic YAML tags. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. """ return load(stream, SafeLoader) @@ -96,10 +164,32 @@ def safe_load_all(stream): """ Parse all YAML documents in a stream and produce corresponding Python objects. - Resolve only basic YAML tags. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. """ return load_all(stream, SafeLoader) +def unsafe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load(stream, UnsafeLoader) + +def unsafe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load_all(stream, UnsafeLoader) + def emit(events, stream=None, Dumper=Dumper, canonical=None, indent=None, width=None, allow_unicode=None, line_break=None): diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 635faac3..ca4bde15 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -1,6 +1,12 @@ -__all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor', - 'ConstructorError'] +__all__ = [ + 'BaseConstructor', + 'SafeConstructor', + 'FullConstructor', + 'UnsafeConstructor', + 'Constructor', + 'ConstructorError' +] from error import * from nodes import * @@ -464,7 +470,7 @@ def construct_undefined(self, node): SafeConstructor.add_constructor(None, SafeConstructor.construct_undefined) -class Constructor(SafeConstructor): +class FullConstructor(SafeConstructor): def construct_python_str(self, node): return self.construct_scalar(node).encode('utf-8') @@ -481,18 +487,22 @@ def construct_python_complex(self, node): def construct_python_tuple(self, node): return tuple(self.construct_sequence(node)) - def find_python_module(self, name, mark): + def find_python_module(self, name, mark, unsafe=False): if not name: raise ConstructorError("while constructing a Python module", mark, "expected non-empty name appended to the tag", mark) - try: - __import__(name) - except ImportError, exc: + if unsafe: + try: + __import__(name) + except ImportError, exc: + raise ConstructorError("while constructing a Python module", mark, + "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark) + if not name in sys.modules: raise ConstructorError("while constructing a Python module", mark, - "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark) + "module %r is not imported" % name.encode('utf-8'), mark) return sys.modules[name] - def find_python_name(self, name, mark): + def find_python_name(self, name, mark, unsafe=False): if not name: raise ConstructorError("while constructing a Python object", mark, "expected non-empty name appended to the tag", mark) @@ -501,11 +511,15 @@ def find_python_name(self, name, mark): else: module_name = '__builtin__' object_name = name - try: - __import__(module_name) - except ImportError, exc: + if unsafe: + try: + __import__(module_name) + except ImportError, exc: + raise ConstructorError("while constructing a Python object", mark, + "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark) + if not module_name in sys.modules: raise ConstructorError("while constructing a Python object", mark, - "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark) + "module %r is not imported" % module_name.encode('utf-8'), mark) module = sys.modules[module_name] if not hasattr(module, object_name): raise ConstructorError("while constructing a Python object", mark, @@ -532,12 +546,16 @@ def construct_python_module(self, suffix, node): class classobj: pass def make_python_instance(self, suffix, node, - args=None, kwds=None, newobj=False): + args=None, kwds=None, newobj=False, unsafe=False): if not args: args = [] if not kwds: kwds = {} cls = self.find_python_name(suffix, node.start_mark) + if not (unsafe or isinstance(cls, type) or isinstance(cls, type(self.classobj))): + raise ConstructorError("while constructing a Python instance", node.start_mark, + "expected a class, but found %r" % type(cls), + node.start_mark) if newobj and isinstance(cls, type(self.classobj)) \ and not args and not kwds: instance = self.classobj() @@ -571,6 +589,64 @@ def construct_python_object(self, suffix, node): state = self.construct_mapping(node, deep=deep) self.set_python_instance_state(instance, state) +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/none', + FullConstructor.construct_yaml_null) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/bool', + FullConstructor.construct_yaml_bool) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/str', + FullConstructor.construct_python_str) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/unicode', + FullConstructor.construct_python_unicode) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/int', + FullConstructor.construct_yaml_int) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/long', + FullConstructor.construct_python_long) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/float', + FullConstructor.construct_yaml_float) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/complex', + FullConstructor.construct_python_complex) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/list', + FullConstructor.construct_yaml_seq) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/tuple', + FullConstructor.construct_python_tuple) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/dict', + FullConstructor.construct_yaml_map) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/name:', + FullConstructor.construct_python_name) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/module:', + FullConstructor.construct_python_module) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object:', + FullConstructor.construct_python_object) + +class UnsafeConstructor(FullConstructor): + def construct_python_object_apply(self, suffix, node, newobj=False): # Format: # !!python/object/apply # (or !!python/object/new) @@ -609,67 +685,25 @@ def construct_python_object_apply(self, suffix, node, newobj=False): def construct_python_object_new(self, suffix, node): return self.construct_python_object_apply(suffix, node, newobj=True) -Constructor.add_constructor( - u'tag:yaml.org,2002:python/none', - Constructor.construct_yaml_null) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/bool', - Constructor.construct_yaml_bool) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/str', - Constructor.construct_python_str) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/unicode', - Constructor.construct_python_unicode) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/int', - Constructor.construct_yaml_int) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/long', - Constructor.construct_python_long) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/float', - Constructor.construct_yaml_float) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/complex', - Constructor.construct_python_complex) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/list', - Constructor.construct_yaml_seq) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/tuple', - Constructor.construct_python_tuple) - -Constructor.add_constructor( - u'tag:yaml.org,2002:python/dict', - Constructor.construct_yaml_map) - -Constructor.add_multi_constructor( - u'tag:yaml.org,2002:python/name:', - Constructor.construct_python_name) + def find_python_module(self, name, mark): + return super(UnsafeConstructor, self).find_python_module(name, mark, unsafe=True) -Constructor.add_multi_constructor( - u'tag:yaml.org,2002:python/module:', - Constructor.construct_python_module) + def find_python_name(self, name, mark): + return super(UnsafeConstructor, self).find_python_name(name, mark, unsafe=True) -Constructor.add_multi_constructor( - u'tag:yaml.org,2002:python/object:', - Constructor.construct_python_object) + def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False): + return super(UnsafeConstructor, self).make_python_instance( + suffix, node, args, kwds, newobj, unsafe=True) -Constructor.add_multi_constructor( +UnsafeConstructor.add_multi_constructor( u'tag:yaml.org,2002:python/object/apply:', - Constructor.construct_python_object_apply) + UnsafeConstructor.construct_python_object_apply) -Constructor.add_multi_constructor( +UnsafeConstructor.add_multi_constructor( u'tag:yaml.org,2002:python/object/new:', - Constructor.construct_python_object_new) + UnsafeConstructor.construct_python_object_new) +# Constructor is same as UnsafeConstructor. Need to leave this in place in case +# people have extended it directly. +class Constructor(UnsafeConstructor): + pass diff --git a/lib/yaml/cyaml.py b/lib/yaml/cyaml.py index 68dcd751..d7a467c9 100644 --- a/lib/yaml/cyaml.py +++ b/lib/yaml/cyaml.py @@ -1,6 +1,8 @@ -__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader', - 'CBaseDumper', 'CSafeDumper', 'CDumper'] +__all__ = [ + 'CBaseLoader', 'CSafeLoader', 'CFullLoader', 'CUnsafeLoader', 'CLoader', + 'CBaseDumper', 'CSafeDumper', 'CDumper' +] from _yaml import CParser, CEmitter @@ -25,6 +27,20 @@ def __init__(self, stream): SafeConstructor.__init__(self) Resolver.__init__(self) +class CFullLoader(CParser, FullConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class CUnsafeLoader(CParser, UnsafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + UnsafeConstructor.__init__(self) + Resolver.__init__(self) + class CLoader(CParser, Constructor, Resolver): def __init__(self, stream): diff --git a/lib/yaml/loader.py b/lib/yaml/loader.py index 293ff467..a79182ea 100644 --- a/lib/yaml/loader.py +++ b/lib/yaml/loader.py @@ -1,5 +1,5 @@ -__all__ = ['BaseLoader', 'SafeLoader', 'Loader'] +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader'] from reader import * from scanner import * @@ -18,6 +18,16 @@ def __init__(self, stream): BaseConstructor.__init__(self) BaseResolver.__init__(self) +class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + FullConstructor.__init__(self) + Resolver.__init__(self) + class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): def __init__(self, stream): @@ -38,3 +48,16 @@ def __init__(self, stream): Constructor.__init__(self) Resolver.__init__(self) +# UnsafeLoader is the same as Loader (which is and was always unsafe on +# untrusted input). Use of either Loader or UnsafeLoader should be rare, since +# FullLoad should be able to load almost all YAML safely. Loader is left intact +# to ensure backwards compatability. +class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) diff --git a/setup.py b/setup.py index cc102d90..f4a1f564 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ NAME = 'PyYAML' -VERSION = '4.1' +VERSION = '3.13' DESCRIPTION = "YAML parser and emitter for Python" LONG_DESCRIPTION = """\ YAML is a data serialization format designed for human readability diff --git a/tests/lib/test_errors.py b/tests/lib/test_errors.py index 7dc9388c..32423c1b 100644 --- a/tests/lib/test_errors.py +++ b/tests/lib/test_errors.py @@ -3,7 +3,7 @@ def test_loader_error(error_filename, verbose=False): try: - list(yaml.load_all(open(error_filename, 'rb'))) + list(yaml.load_all(open(error_filename, 'rb'), yaml.FullLoader)) except yaml.YAMLError, exc: if verbose: print "%s:" % exc.__class__.__name__, exc @@ -14,7 +14,7 @@ def test_loader_error(error_filename, verbose=False): def test_loader_error_string(error_filename, verbose=False): try: - list(yaml.load_all(open(error_filename, 'rb').read())) + list(yaml.load_all(open(error_filename, 'rb').read(), yaml.FullLoader)) except yaml.YAMLError, exc: if verbose: print "%s:" % exc.__class__.__name__, exc @@ -25,7 +25,7 @@ def test_loader_error_string(error_filename, verbose=False): def test_loader_error_single(error_filename, verbose=False): try: - yaml.load(open(error_filename, 'rb').read()) + yaml.load(open(error_filename, 'rb').read(), yaml.FullLoader) except yaml.YAMLError, exc: if verbose: print "%s:" % exc.__class__.__name__, exc diff --git a/tests/lib/test_input_output.py b/tests/lib/test_input_output.py index 3277a886..12e95b14 100644 --- a/tests/lib/test_input_output.py +++ b/tests/lib/test_input_output.py @@ -17,7 +17,7 @@ def _unicode_open(file, encoding, errors='strict'): def test_unicode_input(unicode_filename, verbose=False): data = open(unicode_filename, 'rb').read().decode('utf-8') value = ' '.join(data.split()) - output = yaml.load(_unicode_open(StringIO.StringIO(data.encode('utf-8')), 'utf-8')) + output = yaml.full_load(_unicode_open(StringIO.StringIO(data.encode('utf-8')), 'utf-8')) assert output == value, (output, value) for input in [data, data.encode('utf-8'), codecs.BOM_UTF8+data.encode('utf-8'), @@ -25,9 +25,9 @@ def test_unicode_input(unicode_filename, verbose=False): codecs.BOM_UTF16_LE+data.encode('utf-16-le')]: if verbose: print "INPUT:", repr(input[:10]), "..." - output = yaml.load(input) + output = yaml.full_load(input) assert output == value, (output, value) - output = yaml.load(StringIO.StringIO(input)) + output = yaml.full_load(StringIO.StringIO(input)) assert output == value, (output, value) test_unicode_input.unittest = ['.unicode'] @@ -40,14 +40,14 @@ def test_unicode_input_errors(unicode_filename, verbose=False): codecs.BOM_UTF8+data.encode('utf-16-le')]: try: - yaml.load(input) + yaml.full_load(input) except yaml.YAMLError, exc: if verbose: print exc else: raise AssertionError("expected an exception") try: - yaml.load(StringIO.StringIO(input)) + yaml.full_load(StringIO.StringIO(input)) except yaml.YAMLError, exc: if verbose: print exc diff --git a/tests/lib/test_recursive.py b/tests/lib/test_recursive.py index 6707fd44..312204ea 100644 --- a/tests/lib/test_recursive.py +++ b/tests/lib/test_recursive.py @@ -30,7 +30,7 @@ def test_recursive(recursive_filename, verbose=False): output2 = None try: output1 = yaml.dump(value1) - value2 = yaml.load(output1) + value2 = yaml.load(output1, yaml.FullLoader) output2 = yaml.dump(value2) assert output1 == output2, (output1, output2) finally: