Merge pull request #734 from lcorbasson/update-python-versions

Update Python versions, test on more platforms and fix Travis CI issues

.travis.yml

@@ -1,19 +1,133 @@
 language: python
+os: linux
 python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
+  - "3.8"
+  - "3.7"
   - "3.6"
-  - "pypy-5.3.1"
+  - "3.5"
+  - "2.7"
+  - "pypy3"
+  - "pypy3.5-6.0"
+  - "pypy3.5-7.0"
+  - "pypy3.6-7.0.0"
+  - "pypy"
+  - "pypy2.7-6.0"
+  - "pypy2.7-7.0.0"
+jobs:
+  include:
+    - os: osx
+      python: "3.7"
+      osx_image: xcode11.2  # Python 3.7.4 running on macOS 10.14.4
+      language: shell       # 'language: python' is an error on Travis CI macOS
+      before_install:
+        - brew install pkg-config
+        - brew install icu4c
+        - export PATH="$PATH:/usr/local/opt/icu4c/bin"
+        - export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:/usr/local/opt/icu4c/lib/pkgconfig"
+        - which uconv
+        - uconv -V
+        - export ICU_VERSION="$(uconv -V | sed -e 's,.*\<ICU \([^ ]*\).*,\1,')"
+        - locale -a
+        - python3 -m pip install --upgrade pip
+        - python3 -m pip install --upgrade virtualenv
+        - virtualenv -p python3 --system-site-packages "$HOME/venv"
+        - source "$HOME/venv/bin/activate"
+        - CFLAGS="-O0" STATIC_DEPS=true python3 -m pip install lxml
+        - python3 --version
+        - python --version
+      env:
+        - HOMEBREW_NO_INSTALL_CLEANUP=1
+        - HOMEBREW_NO_ANALYTICS=1
+      before_cache:
+        - rm -f "$HOME/Library/Caches/pip/log/debug.log"
+      cache:
+        directories:
+          - "$HOME/Library/Caches/pip"
+    - os: windows           # Windows 10.0.17134 N/A Build 17134
+      python: "3.8"
+      language: shell       # 'language: python' is an error on Travis CI Windows
+      before_install:
+        - choco install pkgconfiglite
+        - git clone https://github.com/Microsoft/vcpkg.git
+        - pushd vcpkg
+        - ./bootstrap-vcpkg.bat -disableMetrics
+        - ./vcpkg integrate install
+        - ./vcpkg install icu:x64-windows
+        - cp buildtrees/icu/x64-windows-rel/bin/uconv.exe installed/x64-windows/bin/
+        - popd
+        - export PATH="$PATH:$PWD/vcpkg/installed/x64-windows/bin"
+        - export LD_LIBRARY_PATH="$PATH:$PWD/vcpkg/installed/x64-windows/bin:$PWD/vcpkg/installed/x64-windows/lib"
+        - export PYICU_INCLUDES="$PWD/vcpkg/installed/x64-windows/include"
+        - export PYICU_LFLAGS='/LIBPATH:$PWD/vcpkg/installed/x64-windows/bin'
+#        - wget -q https://github.com/unicode-org/icu/releases/download/release-65-1/icu4c-65_1-Win64-MSVC2017.zip -O icu4c.zip
+#        - unzip -q icu4c.zip -d icu4c
+#        - export PATH="$PATH:$PWD/icu4c/bin64"
+#        - export LD_LIBRARY_PATH="$PATH:$PWD/icu4c/bin64:$PWD/icu4c/lib64"
+#        - export PYICU_INCLUDES="$PWD/icu4c/include"
+#        - export PYICU_LFLAGS='/LIBPATH:$PWD/icu4c/lib64'
+        - uconv -V
+        - export ICU_VERSION="$(uconv -V | sed -e 's,.*\<ICU \([^ ]*\).*,\1,')"
+        - locale -a
+        - choco install python --version 3.8.0
+        - python -m pip install --upgrade pip
+        - python --version
+      env: PATH=/c/Python38:/c/Python38/Scripts:$PATH
+  # allow failure on OSes other than Linux
+  # and on PyPy, which regularly segfaults on Travis CI
+  # because of resource exhaustion
+  allow_failures:
+    - os: osx
+    - os: windows
+    - python: "pypy3"
+    - python: "pypy3.5-6.0"
+    - python: "pypy3.5-7.0"
+    - python: "pypy3.6-7.0.0"
+    - python: "pypy"
+    - python: "pypy2.7-6.0"
+    - python: "pypy2.7-7.0.0"
+  fast_finish: true
 # command to install dependencies
 install:
-    - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; else pip install -r requirements-py2.txt; fi
+  - >
+    if [[ "$TRAVIS_PYTHON_VERSION" == "2"* ]] || [[ "$TRAVIS_PYTHON_VERSION" == "pypy"* ]] && [[ "$TRAVIS_PYTHON_VERSION" != "pypy3"* ]]; then
+      pip install -r requirements-py2.txt;
+    else
+      pip3 install -r requirements-py3.txt;
+    fi
 # command to run tests
-script: nosetests tests
-sudo: false
+script:
+  # pypy2 and pypy3 segfault on Travis CI if running all tests in the same process
+  - >
+    if [[ "$TRAVIS_PYTHON_VERSION" == "pypy" ]]; then
+      nosetests --collect-only -v tests 2>&1 \
+        | grep -e 'ok$' \
+        | while read func class etc; do
+          class="${class//[()]/}";
+          class="${class%.*}:${class##*.}";
+          nosetests -v "$class.$func";
+        done || ( echo "$s" >> "script-failures.log" );
+      if [ -e "script-failures.log" ]; then
+        exit 1;
+      fi;
+    elif [[ "$TRAVIS_PYTHON_VERSION" == "pypy3" ]]; then
+      find tests -type f -name "*.py" | while read s; do
+        ( [ ! -x "$s" ] && nosetests --no-byte-compile -s -v "$s" ) || ( echo "$s" >> "script-failures.log" );
+      done;
+      if [ -e "script-failures.log" ]; then
+        exit 1;
+      fi;
+    else
+      nosetests --no-byte-compile --with-coverage tests;
+    fi
+after_failure:
+  - >
+    if [ -e "script-failures.log" ]; then
+      echo $(cat "script-failures.log");
+    fi
 addons:
   apt:
     packages:
       - language-pack-fr
       - language-pack-de
       - language-pack-ko
+      - pkg-config

agate/data_types/number.py

@@ -98,7 +98,9 @@ def cast(self, d):
 
         try:
             return Decimal(d) * sign
-        except InvalidOperation:
+        # The Decimal class will return an InvalidOperation exception on most Python implementations,
+        # but PyPy3 may return a ValueError if the string is not translatable to ASCII
+        except (InvalidOperation, ValueError):
             pass
 
         raise CastError('Can not parse value "%s" as Decimal.' % d)

agate/table/from_csv.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 
 import io
-
 import six
 
 
@@ -45,44 +44,46 @@ def from_csv(cls, path, column_names=None, column_types=None, row_names=None, sk
 
     close = False
 
-    if hasattr(path, 'read'):
-        f = path
-    else:
-        if six.PY2:
-            f = open(path, 'Urb')
+    try:
+        if hasattr(path, 'read'):
+            f = path
         else:
-            f = io.open(path, encoding=encoding)
+            if six.PY2:
+                f = open(path, 'Urb')
+            else:
+                f = io.open(path, encoding=encoding)
 
-        close = True
+            close = True
 
-    if isinstance(skip_lines, int):
-        while skip_lines > 0:
-            f.readline()
-            skip_lines -= 1
-    else:
-        raise ValueError('skip_lines argument must be an int')
+        if isinstance(skip_lines, int):
+            while skip_lines > 0:
+                f.readline()
+                skip_lines -= 1
+        else:
+            raise ValueError('skip_lines argument must be an int')
 
-    contents = six.StringIO(f.read())
+        contents = six.StringIO(f.read())
 
-    if sniff_limit is None:
-        kwargs['dialect'] = csv.Sniffer().sniff(contents.getvalue())
-    elif sniff_limit > 0:
-        kwargs['dialect'] = csv.Sniffer().sniff(contents.getvalue()[:sniff_limit])
+        if sniff_limit is None:
+            kwargs['dialect'] = csv.Sniffer().sniff(contents.getvalue())
+        elif sniff_limit > 0:
+            kwargs['dialect'] = csv.Sniffer().sniff(contents.getvalue()[:sniff_limit])
 
-    if six.PY2:
-        kwargs['encoding'] = encoding
+        if six.PY2:
+            kwargs['encoding'] = encoding
 
-    reader = csv.reader(contents, header=header, **kwargs)
+        reader = csv.reader(contents, header=header, **kwargs)
 
-    if header:
-        if column_names is None:
-            column_names = next(reader)
-        else:
-            next(reader)
+        if header:
+            if column_names is None:
+                column_names = next(reader)
+            else:
+                next(reader)
 
-    rows = tuple(reader)
+        rows = tuple(reader)
 
-    if close:
-        f.close()
+    finally:
+        if close:
+            f.close()
 
     return Table(rows, column_names, column_types, row_names=row_names)

agate/table/from_fixed.py

@@ -38,28 +38,30 @@ def from_fixed(cls, path, schema_path, column_names=utils.default, column_types=
 
     close_f = False
 
-    if not hasattr(path, 'read'):
-        f = io.open(path, encoding=encoding)
-        close_f = True
-    else:
-        f = path
-
     close_schema_f = False
 
-    if not hasattr(schema_path, 'read'):
-        schema_f = io.open(schema_path, encoding=schema_encoding)
-        close_schema_f = True
-    else:
-        schema_f = path
+    try:
+        if not hasattr(path, 'read'):
+            f = io.open(path, encoding=encoding)
+            close_f = True
+        else:
+            f = path
+
+        if not hasattr(schema_path, 'read'):
+            schema_f = io.open(schema_path, encoding=schema_encoding)
+            close_schema_f = True
+        else:
+            schema_f = path
 
-    reader = fixed.reader(f, schema_f)
-    rows = list(reader)
+        reader = fixed.reader(f, schema_f)
+        rows = list(reader)
 
-    if close_f:
-        f.close()
+    finally:
+        if close_f:
+            f.close()
 
-    if close_schema_f:
-        schema_f.close()
+        if close_schema_f:
+            schema_f.close()
 
     if column_names == utils.default:
         column_names = reader.fieldnames

agate/table/from_json.py

@@ -2,11 +2,13 @@
 
 from collections import OrderedDict
 from decimal import Decimal
+import io
 import json
+import six
 
 
 @classmethod
-def from_json(cls, path, row_names=None, key=None, newline=False, column_types=None, **kwargs):
+def from_json(cls, path, row_names=None, key=None, newline=False, column_types=None, encoding='utf-8', **kwargs):
     """
     Create a new table from a JSON file.
 
@@ -29,33 +31,49 @@ def from_json(cls, path, row_names=None, key=None, newline=False, column_types=N
         If `True` then the file will be parsed as "newline-delimited JSON".
     :param column_types:
         See :meth:`.Table.__init__`.
+    :param encoding:
+        According to RFC4627, JSON text shall be encoded in Unicode; the default encoding is
+        UTF-8. You can override this by using any encoding supported by your Python's open() function
+        if :code:`path` is a filepath. If passing in a file handle, it is assumed you have already opened it with the correct
+        encoding specified.
     """
     from agate.table import Table
 
     if key is not None and newline:
         raise ValueError('key and newline may not be specified together.')
 
-    if newline:
-        js = []
+    close = False
+
+    try:
+        if newline:
+            js = []
+
+            if hasattr(path, 'read'):
+                for line in path:
+                    js.append(json.loads(line, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs))
+            else:
+                f = io.open(path, encoding=encoding)
+                close = True
 
-        if hasattr(path, 'read'):
-            for line in path:
-                js.append(json.loads(line, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs))
-        else:
-            with open(path, 'r') as f:
                 for line in f:
                     js.append(json.loads(line, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs))
-    else:
-        if hasattr(path, 'read'):
-            js = json.load(path, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs)
         else:
-            with open(path, 'r') as f:
+            if hasattr(path, 'read'):
+                js = json.load(path, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs)
+            else:
+                f = io.open(path, encoding=encoding)
+                close = True
+
                 js = json.load(f, object_pairs_hook=OrderedDict, parse_float=Decimal, **kwargs)
 
-    if isinstance(js, dict):
-        if not key:
-            raise TypeError('When converting a JSON document with a top-level dictionary element, a key must be specified.')
+        if isinstance(js, dict):
+            if not key:
+                raise TypeError('When converting a JSON document with a top-level dictionary element, a key must be specified.')
+
+            js = js[key]
 
-        js = js[key]
+    finally:
+        if close:
+            f.close()
 
     return Table.from_object(js, row_names=row_names, column_types=column_types)

docs/install.rst

@@ -45,7 +45,7 @@ Supported platforms
 agate supports the following versions of Python:
 
 * Python 2.7
-* Python 3.4+
+* Python 3.5+
 * `PyPy <http://pypy.org/>`_ versions >= 4.0.0
 
 It is tested primarily on OSX, but due to its minimal dependencies it should work perfectly on both Linux and Windows.

setup.py

@@ -34,9 +34,10 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Scientific/Engineering :: Information Analysis',