From e1ffe1afaa1de24dd07283accab6e9ecf91f580f Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Fri, 13 Dec 2019 17:30:52 -0600 Subject: [PATCH] increase size of index, line, and column fields (#310) * increase size of index, line, and column fields * use size_t instead of unsigned long long * better test infrastructure for test for large file * only run large file test when env var is set * fix review comments regarding env vars * fix missing import on python 3 * force all tests in CI --- .appveyor.yml | 2 ++ .travis.yml | 4 ++++ ext/_yaml.pyx | 8 ++++---- tests/lib/test_yaml_ext.py | 18 +++++++++++++++++- tests/lib3/test_yaml_ext.py | 18 +++++++++++++++++- 5 files changed, 44 insertions(+), 6 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 1ca6e2c2..8c752749 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,6 +11,8 @@ image: environment: libyaml_repo_url: https://github.com/yaml/libyaml.git libyaml_refspec: 0.2.2 + PYYAML_TEST_GROUP: all + # matrix: # - PYTHON_VER: Python27 # - PYTHON_VER: Python27-x64 diff --git a/.travis.yml b/.travis.yml index a7102d69..b3185f95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,10 @@ language: python cache: pip +env: + global: + - PYYAML_TEST_GROUP=all + matrix: include: - python: 2.7 diff --git a/ext/_yaml.pyx b/ext/_yaml.pyx index b2f409cc..ff4efe80 100644 --- a/ext/_yaml.pyx +++ b/ext/_yaml.pyx @@ -63,13 +63,13 @@ MappingNode = yaml.nodes.MappingNode cdef class Mark: cdef readonly object name - cdef readonly int index - cdef readonly int line - cdef readonly int column + cdef readonly size_t index + cdef readonly size_t line + cdef readonly size_t column cdef readonly buffer cdef readonly pointer - def __init__(self, object name, int index, int line, int column, + def __init__(self, object name, size_t index, size_t line, size_t column, object buffer, object pointer): self.name = name self.index = index diff --git a/tests/lib/test_yaml_ext.py b/tests/lib/test_yaml_ext.py index bdfda3e7..7f9357f6 100644 --- a/tests/lib/test_yaml_ext.py +++ b/tests/lib/test_yaml_ext.py @@ -1,6 +1,6 @@ import _yaml, yaml -import types, pprint +import types, pprint, tempfile, sys, os yaml.PyBaseLoader = yaml.BaseLoader yaml.PySafeLoader = yaml.SafeLoader @@ -233,6 +233,22 @@ def test_c_emitter(data_filename, canonical_filename, verbose=False): test_c_emitter.unittest = ['.data', '.canonical'] test_c_emitter.skip = ['.skip-ext'] +def test_large_file(verbose=False): + SIZE_LINE = 24 + SIZE_ITERATION = 0 + SIZE_FILE = 31 + if sys.maxsize <= 2**32: + return + if os.environ.get('PYYAML_TEST_GROUP', '') != 'all': + return + with tempfile.TemporaryFile() as temp_file: + for i in range(2**(SIZE_FILE-SIZE_ITERATION-SIZE_LINE) + 1): + temp_file.write(('-' + (' ' * (2**SIZE_LINE-4))+ '{}\n')*(2**SIZE_ITERATION)) + temp_file.seek(0) + yaml.load(temp_file, Loader=yaml.CLoader) + +test_large_file.unittest = None + def wrap_ext_function(function): def wrapper(*args, **kwds): _set_up() diff --git a/tests/lib3/test_yaml_ext.py b/tests/lib3/test_yaml_ext.py index 93d397b8..d902214e 100644 --- a/tests/lib3/test_yaml_ext.py +++ b/tests/lib3/test_yaml_ext.py @@ -1,6 +1,6 @@ import _yaml, yaml -import types, pprint +import types, pprint, tempfile, sys, os yaml.PyBaseLoader = yaml.BaseLoader yaml.PySafeLoader = yaml.SafeLoader @@ -233,6 +233,22 @@ def test_c_emitter(data_filename, canonical_filename, verbose=False): test_c_emitter.unittest = ['.data', '.canonical'] test_c_emitter.skip = ['.skip-ext'] +def test_large_file(verbose=False): + SIZE_LINE = 24 + SIZE_ITERATION = 0 + SIZE_FILE = 31 + if sys.maxsize <= 2**32: + return + if os.environ.get('PYYAML_TEST_GROUP', '') != 'all': + return + with tempfile.TemporaryFile() as temp_file: + for i in range(2**(SIZE_FILE-SIZE_ITERATION-SIZE_LINE) + 1): + temp_file.write(bytes(('-' + (' ' * (2**SIZE_LINE-4))+ '{}\n')*(2**SIZE_ITERATION), 'utf-8')) + temp_file.seek(0) + yaml.load(temp_file, Loader=yaml.CLoader) + +test_large_file.unittest = None + def wrap_ext_function(function): def wrapper(*args, **kwds): _set_up()