From e30d6e83b51d6b7b4967d6555c79b21616617771 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Tue, 21 May 2019 13:18:09 +0200 Subject: [PATCH] Improve encoding of TIFF tags - Pass tagtype from v2 directory to libtiff encoder, instead of autodetecting type. - Use explicit types. E.g. uint32_t for TIFF_LONG to fix issues on platforms with 64bit longs. - Add support for multiple values (arrays). Requires type in v2 directory and values must be passed as a tuple. - Add support for signed types (e.g. TIFFTypes.TIFF_SIGNED_SHORT). --- Tests/test_file_libtiff.py | 62 +++- docs/handbook/image-file-formats.rst | 14 +- src/PIL/TiffImagePlugin.py | 34 +- src/encode.c | 466 +++++++++++++++++---------- src/libImaging/TiffDecode.c | 24 +- src/libImaging/TiffDecode.h | 2 +- 6 files changed, 413 insertions(+), 189 deletions(-) diff --git a/Tests/test_file_libtiff.py b/Tests/test_file_libtiff.py index dedcc179a5e..070fe44c1ee 100644 --- a/Tests/test_file_libtiff.py +++ b/Tests/test_file_libtiff.py @@ -3,6 +3,7 @@ from PIL import features from PIL._util import py3 +from collections import namedtuple from ctypes import c_float import io import logging @@ -235,12 +236,39 @@ def test_additional_metadata(self): TiffImagePlugin.WRITE_LIBTIFF = False def test_custom_metadata(self): + tc = namedtuple("test_case", "value,type,supported_by_default") custom = { - 37000: [4, TiffTags.SHORT], - 37001: [4.2, TiffTags.RATIONAL], - 37002: ["custom tag value", TiffTags.ASCII], - 37003: [u"custom tag value", TiffTags.ASCII], - 37004: [b"custom tag value", TiffTags.BYTE], + 37000 + k: v + for k, v in enumerate( + [ + tc(4, TiffTags.SHORT, True), + tc(123456789, TiffTags.LONG, True), + tc(-4, TiffTags.SIGNED_BYTE, False), + tc(-4, TiffTags.SIGNED_SHORT, False), + tc(-123456789, TiffTags.SIGNED_LONG, False), + tc(TiffImagePlugin.IFDRational(4, 7), TiffTags.RATIONAL, True), + tc(4.25, TiffTags.FLOAT, True), + tc(4.25, TiffTags.DOUBLE, True), + tc("custom tag value", TiffTags.ASCII, True), + tc(u"custom tag value", TiffTags.ASCII, True), + tc(b"custom tag value", TiffTags.BYTE, True), + tc((4, 5, 6), TiffTags.SHORT, True), + tc((123456789, 9, 34, 234, 219387, 92432323), TiffTags.LONG, True), + tc((-4, 9, 10), TiffTags.SIGNED_BYTE, False), + tc((-4, 5, 6), TiffTags.SIGNED_SHORT, False), + tc( + (-123456789, 9, 34, 234, 219387, -92432323), + TiffTags.SIGNED_LONG, + False, + ), + tc((4.25, 5.25), TiffTags.FLOAT, True), + tc((4.25, 5.25), TiffTags.DOUBLE, True), + # array of TIFF_BYTE requires bytes instead of tuple for backwards + # compatibility + tc(bytes([4]), TiffTags.BYTE, True), + tc(bytes((4, 9, 10)), TiffTags.BYTE, True), + ] + ) } libtiff_version = TiffImagePlugin._libtiff_version() @@ -263,8 +291,13 @@ def check_tags(tiffinfo): reloaded = Image.open(out) for tag, value in tiffinfo.items(): reloaded_value = reloaded.tag_v2[tag] - if isinstance(reloaded_value, TiffImagePlugin.IFDRational): - reloaded_value = float(reloaded_value) + if ( + isinstance(reloaded_value, TiffImagePlugin.IFDRational) + and libtiff + ): + # libtiff does not support real RATIONALS + self.assertAlmostEqual(float(reloaded_value), float(value)) + continue if libtiff and isinstance(value, bytes): value = value.decode() @@ -274,12 +307,19 @@ def check_tags(tiffinfo): # Test with types ifd = TiffImagePlugin.ImageFileDirectory_v2() for tag, tagdata in custom.items(): - ifd[tag] = tagdata[0] - ifd.tagtype[tag] = tagdata[1] + ifd[tag] = tagdata.value + ifd.tagtype[tag] = tagdata.type check_tags(ifd) - # Test without types - check_tags({tag: tagdata[0] for tag, tagdata in custom.items()}) + # Test without types. This only works for some types, int for example are + # always encoded as LONG and not SIGNED_LONG. + check_tags( + { + tag: tagdata.value + for tag, tagdata in custom.items() + if tagdata.supported_by_default + } + ) TiffImagePlugin.WRITE_LIBTIFF = False def test_int_dpi(self): diff --git a/docs/handbook/image-file-formats.rst b/docs/handbook/image-file-formats.rst index 3ea8dd53337..cc6102a3728 100644 --- a/docs/handbook/image-file-formats.rst +++ b/docs/handbook/image-file-formats.rst @@ -716,14 +716,20 @@ The :py:meth:`~PIL.Image.Image.save` method can take the following keyword argum :py:class:`~PIL.TiffImagePlugin.ImageFileDirectory_v1` object may be passed in this field. However, this is deprecated. - .. versionadded:: 3.0.0 - - .. note:: + .. versionadded:: 5.4.0 - Only some tags are currently supported when writing using + Previous versions only supported some tags when writing using libtiff. The supported list is found in :py:attr:`~PIL:TiffTags.LIBTIFF_CORE`. + .. versionadded:: 6.1.0 + + Added support for signed types (e.g. ``TIFF_SIGNED_LONG``) and multiple values. + Multiple values for a single tag must be to + :py:class:`~PIL.TiffImagePlugin.ImageFileDirectory_v2` as a tuple and + require a matching type in + :py:attr:`~PIL.TiffImagePlugin.ImageFileDirectory_v2.tagtype` tagtype. + **compression** A string containing the desired compression method for the file. (valid only with libtiff installed) Valid compression diff --git a/src/PIL/TiffImagePlugin.py b/src/PIL/TiffImagePlugin.py index 7596aae655f..7a1ff49e04e 100644 --- a/src/PIL/TiffImagePlugin.py +++ b/src/PIL/TiffImagePlugin.py @@ -99,6 +99,7 @@ Y_RESOLUTION = 283 PLANAR_CONFIGURATION = 284 RESOLUTION_UNIT = 296 +TRANSFERFUNCTION = 301 SOFTWARE = 305 DATE_TIME = 306 ARTIST = 315 @@ -108,6 +109,7 @@ EXTRASAMPLES = 338 SAMPLEFORMAT = 339 JPEGTABLES = 347 +REFERENCEBLACKWHITE = 532 COPYRIGHT = 33432 IPTC_NAA_CHUNK = 33723 # newsphoto properties PHOTOSHOP_CHUNK = 34377 # photoshop properties @@ -1538,9 +1540,21 @@ def _save(im, fp, filename): except io.UnsupportedOperation: pass + # optional types for non core tags + types = {} # STRIPOFFSETS and STRIPBYTECOUNTS are added by the library # based on the data in the strip. - blocklist = [STRIPOFFSETS, STRIPBYTECOUNTS] + # The other tags expect arrays with a certain length (fixed or depending on + # BITSPERSAMPLE, etc), passing arrays with a different length will result in + # segfaults. Block these tags until we add extra validation. + blocklist = [ + COLORMAP, + REFERENCEBLACKWHITE, + STRIPBYTECOUNTS, + STRIPOFFSETS, + TRANSFERFUNCTION, + ] + atts = {} # bits per sample is a single short in the tiff directory, not a list. atts[BITSPERSAMPLE] = bits[0] @@ -1555,15 +1569,19 @@ def _save(im, fp, filename): ): # Libtiff can only process certain core items without adding # them to the custom dictionary. - # Support for custom items has only been been added - # for int, float, unicode, string and byte values + # Custom items are supported for int, float, unicode, string and byte + # values. Other types and tuples require a tagtype. if tag not in TiffTags.LIBTIFF_CORE: if TiffTags.lookup(tag).type == TiffTags.UNDEFINED: continue - if ( - distutils.version.StrictVersion(_libtiff_version()) - < distutils.version.StrictVersion("4.0") - ) or not ( + if distutils.version.StrictVersion( + _libtiff_version() + ) < distutils.version.StrictVersion("4.0"): + continue + + if tag in ifd.tagtype: + types[tag] = ifd.tagtype[tag] + elif not ( isinstance(value, (int, float, str, bytes)) or (not py3 and isinstance(value, unicode)) # noqa: F821 ): @@ -1586,7 +1604,7 @@ def _save(im, fp, filename): if im.mode in ("I;16B", "I;16"): rawmode = "I;16N" - a = (rawmode, compression, _fp, filename, atts) + a = (rawmode, compression, _fp, filename, atts, types) e = Image._getencoder(im.mode, "libtiff", a, im.encoderconfig) e.setimage(im.im, (0, 0) + im.size) while True: diff --git a/src/encode.c b/src/encode.c index 6832f90c8bb..7f8cc479a23 100644 --- a/src/encode.c +++ b/src/encode.c @@ -613,6 +613,310 @@ PyImaging_ZipEncoderNew(PyObject* self, PyObject* args) #endif +/* -------------------------------------------------------------------- */ +/* LibTiff */ +/* -------------------------------------------------------------------- */ + +#ifdef HAVE_LIBTIFF + +#include "TiffDecode.h" + +#include + +PyObject* +PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) +{ + ImagingEncoderObject* encoder; + + char* mode; + char* rawmode; + char* compname; + char* filename; + Py_ssize_t fp; + + PyObject *tags, *types; + PyObject *key, *value; + Py_ssize_t pos = 0; + int key_int, status, is_core_tag, is_var_length, num_core_tags, i; + TIFFDataType type = TIFF_NOTYPE; + // This list also exists in TiffTags.py + const int core_tags[] = { + 256, 257, 258, 259, 262, 263, 266, 269, 274, 277, 278, 280, 281, 340, + 341, 282, 283, 284, 286, 287, 296, 297, 321, 338, 32995, 32998, 32996, + 339, 32997, 330, 531, 530 + }; + + Py_ssize_t d_size; + PyObject *keys, *values; + + + if (! PyArg_ParseTuple(args, "sssisOO", &mode, &rawmode, &compname, &fp, &filename, &tags, &types)) { + return NULL; + } + + if (!PyDict_Check(tags)) { + PyErr_SetString(PyExc_ValueError, "Invalid tags dictionary"); + return NULL; + } else { + d_size = PyDict_Size(tags); + TRACE(("dict size: %d\n", (int)d_size)); + keys = PyDict_Keys(tags); + values = PyDict_Values(tags); + for (pos=0;posstate, filename, fp)) { + Py_DECREF(encoder); + PyErr_SetString(PyExc_RuntimeError, "tiff codec initialization failed"); + return NULL; + } + + num_core_tags = sizeof(core_tags) / sizeof(int); + for (pos = 0; pos < d_size; pos++) { + key = PyList_GetItem(keys, pos); + key_int = (int)PyInt_AsLong(key); + value = PyList_GetItem(values, pos); + status = 0; + is_core_tag = 0; + is_var_length = 0; + type = TIFF_NOTYPE; + + for (i=0; i= TIFF_BYTE && type_int <= TIFF_DOUBLE) { + type = (TIFFDataType)type_int; + } + } + } + + + if (type == TIFF_NOTYPE) { + // Autodetect type. Types should not be changed for backwards + // compatibility. + if (PyInt_Check(value)) { + type = TIFF_LONG; + } else if (PyFloat_Check(value)) { + type = TIFF_DOUBLE; + } else if (PyBytes_Check(value)) { + type = TIFF_ASCII; + } + } + + if (PyBytes_Check(value) && + (type == TIFF_BYTE || type == TIFF_UNDEFINED)) { + // For backwards compatibility + type = TIFF_ASCII; + } + + if (PyTuple_Check(value)) { + Py_ssize_t len; + len = PyTuple_Size(value); + + is_var_length = 1; + + if (!len) { + continue; + } + + if (type == TIFF_NOTYPE) { + // Autodetect type based on first item. Types should not be + // changed for backwards compatibility. + if (PyInt_Check(PyTuple_GetItem(value,0))) { + type = TIFF_LONG; + } else if (PyFloat_Check(PyTuple_GetItem(value,0))) { + type = TIFF_FLOAT; + } + } + } + + if (!is_core_tag) { + // Register field for non core tags. + if (ImagingLibTiffMergeFieldInfo(&encoder->state, type, key_int, is_var_length)) { + continue; + } + } + + if (is_var_length) { + Py_ssize_t len,i; + TRACE(("Setting from Tuple: %d \n", key_int)); + len = PyTuple_Size(value); + + if (type == TIFF_BYTE) { + UINT8 *av; + /* malloc check ok, calloc checks for overflow */ + av = calloc(len, sizeof(UINT8)); + if (av) { + for (i=0;istate, (ttag_t) key_int, len, av); + free(av); + } + } else if (type == TIFF_SHORT) { + UINT16 *av; + /* malloc check ok, calloc checks for overflow */ + av = calloc(len, sizeof(UINT16)); + if (av) { + for (i=0;istate, (ttag_t) key_int, len, av); + free(av); + } + } else if (type == TIFF_LONG) { + UINT32 *av; + /* malloc check ok, calloc checks for overflow */ + av = calloc(len, sizeof(UINT32)); + if (av) { + for (i=0;istate, (ttag_t) key_int, len, av); + free(av); + } + } else if (type == TIFF_SBYTE) { + INT8 *av; + /* malloc check ok, calloc checks for overflow */ + av = calloc(len, sizeof(INT8)); + if (av) { + for (i=0;istate, (ttag_t) key_int, len, av); + free(av); + } + } else if (type == TIFF_SSHORT) { + INT16 *av; + /* malloc check ok, calloc checks for overflow */ + av = calloc(len, sizeof(INT16)); + if (av) { + for (i=0;istate, (ttag_t) key_int, len, av); + free(av); + } + } else if (type == TIFF_SLONG) { + INT32 *av; + /* malloc check ok, calloc checks for overflow */ + av = calloc(len, sizeof(INT32)); + if (av) { + for (i=0;istate, (ttag_t) key_int, len, av); + free(av); + } + } else if (type == TIFF_FLOAT) { + FLOAT32 *av; + /* malloc check ok, calloc checks for overflow */ + av = calloc(len, sizeof(FLOAT32)); + if (av) { + for (i=0;istate, (ttag_t) key_int, len, av); + free(av); + } + } else if (type == TIFF_DOUBLE) { + FLOAT64 *av; + /* malloc check ok, calloc checks for overflow */ + av = calloc(len, sizeof(FLOAT64)); + if (av) { + for (i=0;istate, (ttag_t) key_int, len, av); + free(av); + } + } + } else { + if (type == TIFF_SHORT) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + (UINT16)PyInt_AsLong(value)); + } else if (type == TIFF_LONG) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + (UINT32)PyInt_AsLong(value)); + } else if (type == TIFF_SSHORT) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + (INT16)PyInt_AsLong(value)); + } else if (type == TIFF_SLONG) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + (INT32)PyInt_AsLong(value)); + } else if (type == TIFF_FLOAT) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + (FLOAT32)PyFloat_AsDouble(value)); + } else if (type == TIFF_DOUBLE) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + (FLOAT64)PyFloat_AsDouble(value)); + } else if (type == TIFF_BYTE) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + (UINT8)PyInt_AsLong(value)); + } else if (type == TIFF_SBYTE) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + (INT8)PyInt_AsLong(value)); + } else if (type == TIFF_ASCII) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + PyBytes_AsString(value)); + } else if (type == TIFF_RATIONAL) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) key_int, + (FLOAT64)PyFloat_AsDouble(value)); + } else { + TRACE(("Unhandled type for key %d : %s \n", + key_int, + PyBytes_AsString(PyObject_Str(value)))); + } + } + if (!status) { + TRACE(("Error setting Field\n")); + Py_DECREF(encoder); + PyErr_SetString(PyExc_RuntimeError, "Error setting from dictionary"); + return NULL; + } + } + + encoder->encode = ImagingLibTiffEncode; + + return (PyObject*) encoder; +} + +#endif + /* -------------------------------------------------------------------- */ /* JPEG */ /* -------------------------------------------------------------------- */ @@ -787,168 +1091,6 @@ PyImaging_JpegEncoderNew(PyObject* self, PyObject* args) #endif -/* -------------------------------------------------------------------- */ -/* LibTiff */ -/* -------------------------------------------------------------------- */ - -#ifdef HAVE_LIBTIFF - -#include "TiffDecode.h" - -#include - -PyObject* -PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) -{ - ImagingEncoderObject* encoder; - - char* mode; - char* rawmode; - char* compname; - char* filename; - Py_ssize_t fp; - - PyObject *dir; - PyObject *key, *value; - Py_ssize_t pos = 0; - int key_int, status, is_core_tag, number_of_tags, i; - // This list also exists in TiffTags.py - const int tags[] = { - 256, 257, 258, 259, 262, 263, 266, 269, 274, 277, 278, 280, 281, 340, - 341, 282, 283, 284, 286, 287, 296, 297, 321, 338, 32995, 32998, 32996, - 339, 32997, 330, 531, 530 - }; - - Py_ssize_t d_size; - PyObject *keys, *values; - - - if (! PyArg_ParseTuple(args, "sssisO", &mode, &rawmode, &compname, &fp, &filename, &dir)) { - return NULL; - } - - if (!PyDict_Check(dir)) { - PyErr_SetString(PyExc_ValueError, "Invalid Dictionary"); - return NULL; - } else { - d_size = PyDict_Size(dir); - TRACE(("dict size: %d\n", (int)d_size)); - keys = PyDict_Keys(dir); - values = PyDict_Values(dir); - for (pos=0;posstate, filename, fp)) { - Py_DECREF(encoder); - PyErr_SetString(PyExc_RuntimeError, "tiff codec initialization failed"); - return NULL; - } - - number_of_tags = sizeof(tags) / sizeof(int); - for (pos = 0; pos < d_size; pos++) { - key = PyList_GetItem(keys, pos); - key_int = (int)PyInt_AsLong(key); - value = PyList_GetItem(values, pos); - status = 0; - is_core_tag = 0; - for (i=0; istate, TIFF_LONG, key_int)) { - status = ImagingLibTiffSetField(&encoder->state, - (ttag_t) PyInt_AsLong(key), - PyInt_AsLong(value)); - } - } else if (PyFloat_Check(value)) { - TRACE(("Setting from Float: %d, %f \n", key_int, PyFloat_AsDouble(value))); - if (is_core_tag || !ImagingLibTiffMergeFieldInfo(&encoder->state, TIFF_DOUBLE, key_int)) { - status = ImagingLibTiffSetField(&encoder->state, - (ttag_t) PyInt_AsLong(key), - (double)PyFloat_AsDouble(value)); - } - } else if (PyBytes_Check(value)) { - TRACE(("Setting from Bytes: %d, %s \n", key_int, PyBytes_AsString(value))); - if (is_core_tag || !ImagingLibTiffMergeFieldInfo(&encoder->state, TIFF_ASCII, key_int)) { - status = ImagingLibTiffSetField(&encoder->state, - (ttag_t) PyInt_AsLong(key), - PyBytes_AsString(value)); - } - } else if (PyTuple_Check(value)) { - Py_ssize_t len,i; - float *floatav; - int *intav; - TRACE(("Setting from Tuple: %d \n", key_int)); - len = PyTuple_Size(value); - if (len) { - if (PyInt_Check(PyTuple_GetItem(value,0))) { - TRACE((" %d elements, setting as ints \n", (int)len)); - /* malloc check ok, calloc checks for overflow */ - intav = calloc(len, sizeof(int)); - if (intav) { - for (i=0;istate, - (ttag_t) PyInt_AsLong(key), - len, intav); - free(intav); - } - } else if (PyFloat_Check(PyTuple_GetItem(value,0))) { - TRACE((" %d elements, setting as floats \n", (int)len)); - /* malloc check ok, calloc checks for overflow */ - floatav = calloc(len, sizeof(float)); - if (floatav) { - for (i=0;istate, - (ttag_t) PyInt_AsLong(key), - len, floatav); - free(floatav); - } - } else { - TRACE(("Unhandled type in tuple for key %d : %s \n", - key_int, - PyBytes_AsString(PyObject_Str(value)))); - } - } - } else { - TRACE(("Unhandled type for key %d : %s \n", - key_int, - PyBytes_AsString(PyObject_Str(value)))); - } - if (!status) { - TRACE(("Error setting Field\n")); - Py_DECREF(encoder); - PyErr_SetString(PyExc_RuntimeError, "Error setting from dictionary"); - return NULL; - } - } - - encoder->encode = ImagingLibTiffEncode; - - return (PyObject*) encoder; -} - -#endif /* -------------------------------------------------------------------- */ /* JPEG 2000 */ diff --git a/src/libImaging/TiffDecode.c b/src/libImaging/TiffDecode.c index a68f73993ff..e73467fabaa 100644 --- a/src/libImaging/TiffDecode.c +++ b/src/libImaging/TiffDecode.c @@ -517,15 +517,33 @@ int ImagingLibTiffEncodeInit(ImagingCodecState state, char *filename, int fp) { } -int ImagingLibTiffMergeFieldInfo(ImagingCodecState state, TIFFDataType field_type, int key){ +int ImagingLibTiffMergeFieldInfo(ImagingCodecState state, TIFFDataType field_type, int key, int is_var_length){ + // Refer to libtiff docs (http://www.simplesystems.org/libtiff/addingtags.html) TIFFSTATE *clientstate = (TIFFSTATE *)state->context; char field_name[10]; uint32 n; int status = 0; - const TIFFFieldInfo info[] = { - { key, 0, 1, field_type, FIELD_CUSTOM, 1, 0, field_name } + // custom fields added with ImagingLibTiffMergeFieldInfo are only used for + // decoding, ignore readcount; + int readcount = 0; + // we support writing single value, or a variable number of values + int writecount = 1; + // whether the first value should encode the number of values. + int passcount = 0; + + TIFFFieldInfo info[] = { + { key, readcount, writecount, field_type, FIELD_CUSTOM, 1, passcount, field_name } }; + + if (is_var_length) { + info[0].field_writecount = -1; + } + + if (is_var_length && field_type != TIFF_ASCII) { + info[0].field_passcount = 1; + } + n = sizeof(info) / sizeof(info[0]); // Test for libtiff 4.0 or later, excluding libtiff 3.9.6 and 3.9.7 diff --git a/src/libImaging/TiffDecode.h b/src/libImaging/TiffDecode.h index e29a6c88f4b..6cb7e53a2b5 100644 --- a/src/libImaging/TiffDecode.h +++ b/src/libImaging/TiffDecode.h @@ -45,7 +45,7 @@ typedef struct { extern int ImagingLibTiffInit(ImagingCodecState state, int fp, int offset); extern int ImagingLibTiffEncodeInit(ImagingCodecState state, char *filename, int fp); -extern int ImagingLibTiffMergeFieldInfo(ImagingCodecState state, TIFFDataType field_type, int key); +extern int ImagingLibTiffMergeFieldInfo(ImagingCodecState state, TIFFDataType field_type, int key, int is_var_length); extern int ImagingLibTiffSetField(ImagingCodecState state, ttag_t tag, ...);