FileEncoding.java

package org.freebsd.file;

import java.util.Arrays;

/**
 * Tries to guess the encoding of the byte sequence.
 *
 * Orignial code taken from https://github.com/file/file/blob/master/src/encoding.c
 */
public class FileEncoding
{

    private String type = "text/plain";
    private String code = "unknown";
    private String code_mime = "binary";

    public String getCodeMime()
    {
        return code_mime;
    }

    public String getType()
    {
        return type;
    }

    public String getCode()
    {
        return code;
    }

    /**
     * Try to determine whether text is in some character code we can
     * identify. It also identifies EBCDIC by converting it to ISO-8859-1.
     *
     * @return true if it could guess an encoding.
     */
    public boolean guessFileEncoding( byte[] buf )
    {
        int nbytes = buf.length;
        int ucs_type;

        if ( looks_ascii( buf, nbytes ) )
        {
            if ( looks_utf7( buf, nbytes ) )
            {
                code = "UTF-7 Unicode";
                code_mime = "utf-7";
            }
            else
            {
                code = "ASCII";
                code_mime = "us-ascii";
            }
        }
        else if ( looks_utf8_with_BOM( buf, nbytes ) )
        {
            code = "UTF-8 Unicode (with BOM)";
            code_mime = "utf-8";
        }
        else if ( looks_utf8( buf, nbytes ) > 1 )
        {
            code = "UTF-8 Unicode";
            code_mime = "utf-8";
        }
        else if ( ( ucs_type = looks_ucs16( buf, nbytes ) ) != 0 )
        {
            if ( ucs_type == 1 )
            {
                code = "Little-endian UTF-16 Unicode";
                code_mime = "utf-16le";
            }
            else
            {
                code = "Big-endian UTF-16 Unicode";
                code_mime = "utf-16be";
            }
        }
        else if ( looks_latin1( buf, nbytes ) )
        {
            code = "ISO-8859";
            code_mime = "iso-8859-1";
        }
        else if ( looks_extended( buf, nbytes ) )
        {
            code = "Non-ISO extended-ASCII";
            code_mime = "unknown-8bit";
        }
        else
        {
            byte[] nbuf = from_ebcdic( buf, nbytes );

            if ( looks_ascii( nbuf, nbytes ) )
            {
                code = "EBCDIC";
                code_mime = "ebcdic";
            }
            else if ( looks_latin1( nbuf, nbytes ) )
            {
                code = "International EBCDIC";
                code_mime = "ebcdic";
            }
            else
            { /* Doesn't look like text at all */
                type = "binary";
                return false;
            }
        }

        return true;
    }

    /*
     * This table reflects a particular philosophy about what constitutes
     * "text," and there is room for disagreement about it.
     *
     * Version 3.31 of the file command considered a file to be ASCII if
     * each of its characters was approved by either the isascii() or
     * isalpha() function.  On most systems, this would mean that any
     * file consisting only of characters in the range 0x00 ... 0x7F
     * would be called ASCII text, but many systems might reasonably
     * consider some characters outside this range to be alphabetic,
     * so the file command would call such characters ASCII.  It might
     * have been more accurate to call this "considered textual on the
     * local system" than "ASCII."
     *
     * It considered a file to be "International language text" if each
     * of its characters was either an ASCII printing character (according
     * to the real ASCII standard, not the above test), a character in
     * the range 0x80 ... 0xFF, or one of the following control characters:
     * backspace, tab, line feed, vertical tab, form feed, carriage return,
     * escape.  No attempt was made to determine the language in which files
     * of this type were written.
     *
     *
     * The table below considers a file to be ASCII if all of its characters
     * are either ASCII printing characters (again, according to the X3.4
     * standard, not isascii()) or any of the following controls: bell,
     * backspace, tab, line feed, form feed, carriage return, esc, nextline.
     *
     * I include bell because some programs (particularly shell scripts)
     * use it literally, even though it is rare in normal text.  I exclude
     * vertical tab because it never seems to be used in real text.  I also
     * include, with hesitation, the X3.64/ECMA-43 control nextline (0x85),
     * because that's what the dd EBCDIC->ASCII table maps the EBCDIC newline
     * character to.  It might be more appropriate to include it in the 8859
     * set instead of the ASCII set, but it's got to be included in *something*
     * we recognize or EBCDIC files aren't going to be considered textual.
     * Some old Unix source files use SO/SI (^N/^O) to shift between Greek
     * and Latin characters, so these should possibly be allowed.  But they
     * make a real mess on VT100-style displays if they're not paired properly,
     * so we are probably better off not calling them text.
     *
     * A file is considered to be ISO-8859 text if its characters are all
     * either ASCII, according to the above definition, or printing characters
     * from the ISO-8859 8-bit extension, characters 0xA0 ... 0xFF.
     *
     * Finally, a file is considered to be international text from some other
     * character code if its characters are all either ISO-8859 (according to
     * the above definition) or characters in the range 0x80 ... 0x9F, which
     * ISO-8859 considers to be control characters but the IBM PC and Macintosh
     * consider to be printing characters.
     */
    private static final byte F = 0; /* character never appears in text */
    private static final byte T = 1; /* character appears in plain ASCII text */
    private static final byte I = 2; /* character appears in ISO-8859 text */
    private static final byte X = 3; /* character appears in non-ISO extended ASCII (Mac, IBM PC) */

    private byte[] text_chars = {
            /*                  BEL BS HT LF VT FF CR    */
        F, F, F, F, F, F, F, T, T, T, T, T, T, T, F, F, /* 0x0X */
            /*                              ESC          */
        F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F, /* 0x1X */
        T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x2X */
        T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x3X */
        T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x4X */
        T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x5X */
        T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x6X */
        T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F, /* 0x7X */
            /*            NEL                            */
        X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X, /* 0x8X */
        X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 0x9X */
        I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xaX */
        I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xbX */
        I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xcX */
        I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xdX */
        I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xeX */
        I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I /* 0xfX */
    };

    private boolean looks_ascii( byte[] buf, int nbytes )
    {
        for ( int i = 0; i < nbytes; i++ )
        {
            if ( text_chars[ unsignedByte( buf[ i ] ) ] != T )
            {
                return false;
            }
        }
        return true;
    }

    private boolean looks_latin1( byte[] buf, int nbytes )
    {
        for ( int i = 0; i < nbytes; i++ )
        {
            int t = text_chars[ unsignedByte( buf[ i ] ) ];

            if ( t != T && t != I )
            {
                return false;
            }
        }

        return true;
    }

    private boolean looks_extended( byte[] buf, int nbytes )
    {
        for ( int i = 0; i < nbytes; i++ )
        {
            int t = text_chars[ unsignedByte( buf[ i ] ) ];
            if ( t != T && t != I && t != X )
            {
                return false;
            }
        }
        return true;
    }

    /*
     * Decide whether some text looks like UTF-8. Returns:
     *
     *     -1: invalid UTF-8
     *      0: uses odd control characters, so doesn't look like text
     *      1: 7-bit text
     *      2: definitely UTF-8 text (valid high-bit set bytes)
     */
    protected int looks_utf8( byte[] buf, int nbytes )
    {
        boolean gotone = false;

        for ( int i = 0; i < nbytes; i++ )
        {
            if ( ( unsignedByte( buf[ i ] ) & 0x80 ) == 0 )
            { /* 0xxxxxxx is plain ASCII */
                /*
                * Even if the whole file is valid UTF-8 sequences,
                * still reject it if it uses weird control characters.
                */

                if ( text_chars[ unsignedByte( buf[ i ] ) ] != T )
                {
                    return 0;
                }
            }
            else
            {
                if ( ( unsignedByte( buf[ i ] ) & 0x40 ) == 0 )
                { /* 10xxxxxx never 1st byte */
                    return -1;
                }
                else
                {   /* 11xxxxxx begins UTF-8 */
                    int following;

                    if ( ( unsignedByte( buf[ i ] ) & 0x20 ) == 0 )
                    { /* 110xxxxx */
                        following = 1;
                    }
                    else if ( ( unsignedByte( buf[ i ] ) & 0x10 ) == 0 )
                    { /* 1110xxxx */
                        following = 2;
                    }
                    else if ( ( unsignedByte( buf[ i ] ) & 0x08 ) == 0 )
                    { /* 11110xxx */
                        following = 3;
                    }
                    else if ( ( unsignedByte( buf[ i ] ) & 0x04 ) == 0 )
                    { /* 111110xx */
                        following = 4;
                    }
                    else if ( ( unsignedByte( buf[ i ] ) & 0x02 ) == 0 )
                    { /* 1111110x */
                        following = 5;
                    }
                    else
                    {
                        return -1;
                    }

                    for ( int n = 0; n < following; n++ )
                    {
                        i++;
                        if ( i >= nbytes )
                        {
                            return gotone ? 2 : 1;
                        }

                        if ( ( unsignedByte( buf[ i ] ) & 0x80 ) == 0 || ( unsignedByte( buf[ i ] ) & 0x40 ) > 0 )
                        {
                            return -1;
                        }
                    }
                    gotone = true;
                }
            }
        }

        return gotone ? 2 : 1;
    }

    /*
     * Decide whether some text looks like UTF-8 with BOM. If there is no
     * BOM, return -1; otherwise return the result of looks_utf8 on the
     * rest of the text.
     */
    private boolean looks_utf8_with_BOM( byte[] buf, int nbytes )
    {
        if ( nbytes > 3 && unsignedByte( buf[ 0 ] ) == 0xef && unsignedByte( buf[ 1 ] ) == 0xbb && unsignedByte( buf[ 2 ] ) == 0xbf )
        {
            return looks_utf8( Arrays.copyOfRange( buf, 3, nbytes ), nbytes - 3 ) > 0;
        }
        return false;
    }

    private boolean looks_utf7( byte[] buf, int nbytes )
    {
        if ( nbytes > 4 && buf[ 0 ] == '+' && buf[ 1 ] == '/' && buf[ 2 ] == 'v' )
        {
            switch ( buf[ 3 ] )
            {
                case '8':
                case '9':
                case '+':
                case '/':
                    return true;
                default:
                    return false;
            }
        }
        return false;
    }

    private int looks_ucs16( byte[] buf, int nbytes )
    {
        int bigend;
        int i;

        char[] ubuf = new char[ nbytes ];

        if ( nbytes < 2 )
        {
            return 0;
        }

        if ( unsignedByte( buf[ 0 ] ) == 0xff && unsignedByte( buf[ 1 ] ) == 0xfe )
        {
            bigend = 0;
        }
        else
        {
            if ( unsignedByte( buf[ 0 ] ) == 0xfe && unsignedByte( buf[ 1 ] ) == 0xff )
            {
                bigend = 1;
            }
            else
            {
                return 0;
            }
        }

        int ulen = 0;

        for ( i = 2; i + 1 < nbytes; i += 2 )
        {
            if ( bigend == 1 )
            {
                ubuf[ ( ulen )++ ] = (char) ( unsignedByte( buf[ i + 1 ] ) + 256 * buf[ i ] );
            }
            else
            {
                ubuf[ ( ulen )++ ] = (char) ( unsignedByte( buf[ i ] ) + 256 * buf[ i + 1 ] );
            }

            if ( ubuf[ ulen - 1 ] == 0xfffe )
            {
                return 0;
            }
            if ( ubuf[ ulen - 1 ] < 128 && text_chars[ubuf[ ulen - 1 ]] != T )
            {
                return 0;
            }
        }

        return 1 + bigend;
    }

    /*
     * This table maps each EBCDIC character to an (8-bit extended) ASCII
     * character, as specified in the rationale for the dd(1) command in
     * draft 11.2 (September, 1991) of the POSIX P1003.2 standard.
     *
     * Unfortunately it does not seem to correspond exactly to any of the
     * five variants of EBCDIC documented in IBM's _Enterprise Systems
     * Architecture/390: Principles of Operation_, SA22-7201-06, Seventh
     * Edition, July, 1999, pp. I-1 - I-4.
     *
     * Fortunately, though, all versions of EBCDIC, including this one, agree
     * on most of the printing characters that also appear in (7-bit) ASCII.
     * Of these, only '|', '!', '~', '^', '[', and ']' are in question at all.
     *
     * Fortunately too, there is general agreement that codes 0x00 through
     * 0x3F represent control characters, 0x41 a nonbreaking space, and the
     * remainder printing characters.
     *
     * This is sufficient to allow us to identify EBCDIC text and to distinguish
     * between old-style and internationalized examples of text.
     */
    private static final char[] ebcdic_to_ascii = { //
        0, 1, 2, 3, 156, 9, 134, 127, 151, 141, 142, 11, 12, 13, 14, 15, //
        16, 17, 18, 19, 157, 133, 8, 135, 24, 25, 146, 143, 28, 29, 30, 31, //
        128, 129, 130, 131, 132, 10, 23, 27, 136, 137, 138, 139, 140, 5, 6, 7, //
        144, 145, 22, 147, 148, 149, 150, 4, 152, 153, 154, 155, 20, 21, 158, 26, //
        ' ', 160, 161, 162, 163, 164, 165, 166, 167, 168, 213, '.', '<', '(', '+', '|', //
        '&', 169, 170, 171, 172, 173, 174, 175, 176, 177, '!', '$', '*', ')', ';', '~', //
        '-', '/', 178, 179, 180, 181, 182, 183, 184, 185, 203, ',', '%', '_', '>', '?', //
        186, 187, 188, 189, 190, 191, 192, 193, 194, '`', ':', '#', '@', '\'', '=', '"', //
        195, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 196, 197, 198, 199, 200, 201, //
        202, 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', '^', 204, 205, 206, 207, 208, //
        209, 229, 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 210, 211, 212, '[', 214, 215, //
        216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, ']', 230, 231, //
        '{', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 232, 233, 234, 235, 236, 237, //
        '}', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 238, 239, 240, 241, 242, 243, //
        '\\', 159, 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 244, 245, 246, 247, 248, 249, //
        '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 250, 251, 252, 253, 254, 255 //
    };

    /*
     * The following EBCDIC-to-ASCII table may relate more closely to reality,
     * or at least to modern reality.  It comes from
     *
     *   http://ftp.s390.ibm.com/products/oe/bpxqp9.html
     *
     * and maps the characters of EBCDIC code page 1047 (the code used for
     * Unix-derived software on IBM's 390 systems) to the corresponding
     * characters from ISO 8859-1.
     *
     * If this table is used instead of the above one, some of the special
     * cases for the NEL character can be taken out of the code.
     */
    private static final char[] ebcdic_1047_to_8859 = { //
        0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, //
        0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, //
        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, //
        0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, //
        0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, //
        0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E, //
        0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, //
        0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, //
        0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, //
        0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, //
        0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0x5B, 0xDE, 0xAE, //
        0xAC, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0xDD, 0xA8, 0xAF, 0x5D, 0xB4, 0xD7, //
        0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, //
        0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF, //
        0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, //
        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F //
    };

    /*
     * Copy buf[0 ... nbytes-1] into out[], translating EBCDIC to ASCII.
     */
    private byte[] from_ebcdic( byte[] buf, int nbytes )
    {
        byte[] out = new byte[ nbytes ];
        for ( int i = 0; i < nbytes; i++ )
        {
            out[ i ] = (byte) ebcdic_to_ascii[ unsignedByte( buf[ i ] ) ];
        }
        return out;
    }

    private int unsignedByte( byte value )
    {
        return value & 0xFF;
    }
}