From 625cedf140f383f3f5c4f992906a4bb6b809dfd7 Mon Sep 17 00:00:00 2001 From: Gary Gregory Date: Wed, 4 Dec 2019 10:54:41 -0500 Subject: [PATCH] [CODEC-272] Add RandomAccessFile digest methods #31. - This is a slightly different version from https://github.com/apache/commons-codec/pull/31/ - Refactor updateDigest(MessageDigest,RandomAccessFile) into an new private updateDigest(MessageDigest,FileChannel) as possible public candidate. - Do NOT seek to 0 on a RandomAccessFile before calling updateDigest(): We do not do this for ByteBuffer input, so do not do it here and be consistent to assume that when the caller says 'digest this' then do it from where the input stands (like a stream). - Add methods in the file to keep methods in alphabetical order. - Closes #31. --- src/changes/changes.xml | 1 + .../commons/codec/digest/DigestUtils.java | 259 +++++++++++------- .../commons/codec/digest/DigestUtilsTest.java | 19 ++ .../digest/MessageDigestAlgorithmsTest.java | 24 ++ 4 files changed, 199 insertions(+), 104 deletions(-) diff --git a/src/changes/changes.xml b/src/changes/changes.xml index 8cb83fd4c8..93cff09e27 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -49,6 +49,7 @@ The type attribute can be add,update,fix,remove. Add MurmurHash3.hash128x64 methods to fix sign extension error during seeding in hash128 methods. Add MurmurHash3.hash32x86 methods and IncrementalHash32x86 to fix sign extension error in hash32 methods. Allow repeat calls to MurmurHash3.IncrementalHash32.end() to generate the same value. + Add RandomAccessFile digest methods #31. diff --git a/src/main/java/org/apache/commons/codec/digest/DigestUtils.java b/src/main/java/org/apache/commons/codec/digest/DigestUtils.java index c814e11260..17f0e89bc3 100644 --- a/src/main/java/org/apache/commons/codec/digest/DigestUtils.java +++ b/src/main/java/org/apache/commons/codec/digest/DigestUtils.java @@ -20,9 +20,11 @@ import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; +import java.io.RandomAccessFile; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -115,6 +117,19 @@ public static byte[] digest(final MessageDigest messageDigest, final InputStream return updateDigest(messageDigest, data).digest(); } + /** + * Reads through a RandomAccessFile using non-blocking-io (NIO) and returns the digest for the data + * + * @param messageDigest The MessageDigest to use (e.g. MD5) + * @param data Data to digest + * @return the digest + * @throws IOException On error reading from the stream + * @since 1.14 + */ + public static byte[] digest(final MessageDigest messageDigest, final RandomAccessFile data) throws IOException { + return updateDigest(messageDigest, data).digest(); + } + /** * Returns a MessageDigest for the given algorithm. * @@ -699,6 +714,32 @@ public static byte[] sha3_224(final String data) { return sha3_224(StringUtils.getBytesUtf8(data)); } + /** + * Calculates the SHA3-224 digest and returns the value as a hex string. + * + * @param data + * Data to digest + * @return SHA3-224 digest as a hex string + * @since 1.12 + */ + public static String sha3_224Hex(final byte[] data) { + return Hex.encodeHexString(sha3_224(data)); + } + + /** + * Calculates the SHA3-224 digest and returns the value as a hex string. + * + * @param data + * Data to digest + * @return SHA3-224 digest as a hex string + * @throws IOException + * On error reading from the stream + * @since 1.12 + */ + public static String sha3_224Hex(final InputStream data) throws IOException { + return Hex.encodeHexString(sha3_224(data)); + } + /** * Calculates the SHA3-224 digest and returns the value as a hex string. * @@ -749,6 +790,32 @@ public static byte[] sha3_256(final String data) { return sha3_256(StringUtils.getBytesUtf8(data)); } + /** + * Calculates the SHA3-256 digest and returns the value as a hex string. + * + * @param data + * Data to digest + * @return SHA3-256 digest as a hex string + * @since 1.12 + */ + public static String sha3_256Hex(final byte[] data) { + return Hex.encodeHexString(sha3_256(data)); + } + + /** + * Calculates the SHA3-256 digest and returns the value as a hex string. + * + * @param data + * Data to digest + * @return SHA3-256 digest as a hex string + * @throws IOException + * On error reading from the stream + * @since 1.12 + */ + public static String sha3_256Hex(final InputStream data) throws IOException { + return Hex.encodeHexString(sha3_256(data)); + } + /** * Calculates the SHA3-256 digest and returns the value as a hex string. * @@ -799,6 +866,32 @@ public static byte[] sha3_384(final String data) { return sha3_384(StringUtils.getBytesUtf8(data)); } + /** + * Calculates the SHA3-384 digest and returns the value as a hex string. + * + * @param data + * Data to digest + * @return SHA3-384 digest as a hex string + * @since 1.12 + */ + public static String sha3_384Hex(final byte[] data) { + return Hex.encodeHexString(sha3_384(data)); + } + + /** + * Calculates the SHA3-384 digest and returns the value as a hex string. + * + * @param data + * Data to digest + * @return SHA3-384 digest as a hex string + * @throws IOException + * On error reading from the stream + * @since 1.12 + */ + public static String sha3_384Hex(final InputStream data) throws IOException { + return Hex.encodeHexString(sha3_384(data)); + } + /** * Calculates the SHA3-384 digest and returns the value as a hex string. * @@ -849,6 +942,32 @@ public static byte[] sha3_512(final String data) { return sha3_512(StringUtils.getBytesUtf8(data)); } + /** + * Calculates the SHA3-512 digest and returns the value as a hex string. + * + * @param data + * Data to digest + * @return SHA3-512 digest as a hex string + * @since 1.12 + */ + public static String sha3_512Hex(final byte[] data) { + return Hex.encodeHexString(sha3_512(data)); + } + + /** + * Calculates the SHA3-512 digest and returns the value as a hex string. + * + * @param data + * Data to digest + * @return SHA3-512 digest as a hex string + * @throws IOException + * On error reading from the stream + * @since 1.12 + */ + public static String sha3_512Hex(final InputStream data) throws IOException { + return Hex.encodeHexString(sha3_512(data)); + } + /** * Calculates the SHA3-512 digest and returns the value as a hex string. * @@ -987,54 +1106,6 @@ public static String sha512Hex(final byte[] data) { return Hex.encodeHexString(sha512(data)); } - /** - * Calculates the SHA3-224 digest and returns the value as a hex string. - * - * @param data - * Data to digest - * @return SHA3-224 digest as a hex string - * @since 1.12 - */ - public static String sha3_224Hex(final byte[] data) { - return Hex.encodeHexString(sha3_224(data)); - } - - /** - * Calculates the SHA3-256 digest and returns the value as a hex string. - * - * @param data - * Data to digest - * @return SHA3-256 digest as a hex string - * @since 1.12 - */ - public static String sha3_256Hex(final byte[] data) { - return Hex.encodeHexString(sha3_256(data)); - } - - /** - * Calculates the SHA3-384 digest and returns the value as a hex string. - * - * @param data - * Data to digest - * @return SHA3-384 digest as a hex string - * @since 1.12 - */ - public static String sha3_384Hex(final byte[] data) { - return Hex.encodeHexString(sha3_384(data)); - } - - /** - * Calculates the SHA3-512 digest and returns the value as a hex string. - * - * @param data - * Data to digest - * @return SHA3-512 digest as a hex string - * @since 1.12 - */ - public static String sha3_512Hex(final byte[] data) { - return Hex.encodeHexString(sha3_512(data)); - } - /** * Calculates the SHA-512 digest and returns the value as a hex string. * @@ -1049,62 +1120,6 @@ public static String sha512Hex(final InputStream data) throws IOException { return Hex.encodeHexString(sha512(data)); } - /** - * Calculates the SHA3-224 digest and returns the value as a hex string. - * - * @param data - * Data to digest - * @return SHA3-224 digest as a hex string - * @throws IOException - * On error reading from the stream - * @since 1.12 - */ - public static String sha3_224Hex(final InputStream data) throws IOException { - return Hex.encodeHexString(sha3_224(data)); - } - - /** - * Calculates the SHA3-256 digest and returns the value as a hex string. - * - * @param data - * Data to digest - * @return SHA3-256 digest as a hex string - * @throws IOException - * On error reading from the stream - * @since 1.12 - */ - public static String sha3_256Hex(final InputStream data) throws IOException { - return Hex.encodeHexString(sha3_256(data)); - } - - /** - * Calculates the SHA3-384 digest and returns the value as a hex string. - * - * @param data - * Data to digest - * @return SHA3-384 digest as a hex string - * @throws IOException - * On error reading from the stream - * @since 1.12 - */ - public static String sha3_384Hex(final InputStream data) throws IOException { - return Hex.encodeHexString(sha3_384(data)); - } - - /** - * Calculates the SHA3-512 digest and returns the value as a hex string. - * - * @param data - * Data to digest - * @return SHA3-512 digest as a hex string - * @throws IOException - * On error reading from the stream - * @since 1.12 - */ - public static String sha3_512Hex(final InputStream data) throws IOException { - return Hex.encodeHexString(sha3_512(data)); - } - /** * Calculates the SHA-512 digest and returns the value as a hex string. * @@ -1207,6 +1222,27 @@ public static MessageDigest updateDigest(final MessageDigest digest, final File } } + /** + * Reads through a RandomAccessFile and updates the digest for the data using non-blocking-io (NIO). + * + * TODO Decide if this should be public. + * + * @param digest The MessageDigest to use (e.g. MD5) + * @param data Data to digest + * @return the digest + * @throws IOException On error reading from the stream + * @since 1.14 + */ + private static MessageDigest updateDigest(final MessageDigest digest, final FileChannel data) throws IOException { + final ByteBuffer buffer = ByteBuffer.allocate(STREAM_BUFFER_LENGTH); + while (data.read(buffer) > 0) { + buffer.flip(); + digest.update(buffer); + buffer.clear(); + } + return digest; + } + /** * Reads through an InputStream and updates the digest for the data * @@ -1231,6 +1267,21 @@ public static MessageDigest updateDigest(final MessageDigest digest, final Input return digest; } + + /** + * Reads through a RandomAccessFile and updates the digest for the data using non-blocking-io (NIO) + * + * @param digest The MessageDigest to use (e.g. MD5) + * @param data Data to digest + * @return the digest + * @throws IOException On error reading from the stream + * @since 1.14 + */ + public static MessageDigest updateDigest(final MessageDigest digest, final RandomAccessFile data) + throws IOException { + return updateDigest(digest, data.getChannel()); + } + /** * Updates the given {@link MessageDigest} from a String (converted to bytes using UTF-8). *

diff --git a/src/test/java/org/apache/commons/codec/digest/DigestUtilsTest.java b/src/test/java/org/apache/commons/codec/digest/DigestUtilsTest.java index 6416220553..906d245a2d 100644 --- a/src/test/java/org/apache/commons/codec/digest/DigestUtilsTest.java +++ b/src/test/java/org/apache/commons/codec/digest/DigestUtilsTest.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.security.MessageDigest; import java.util.Random; @@ -47,6 +48,10 @@ public class DigestUtilsTest { private File testFile; + private File testRandomAccessFile; + + private RandomAccessFile testRandomAccessFileWrapper; + private void assumeJava8() { Assume.assumeTrue(SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_1_8)); } @@ -63,6 +68,10 @@ File getTestFile() { return testFile; } + RandomAccessFile getTestRandomAccessFile() { + return testRandomAccessFileWrapper; + } + @Before public void setUp() throws Exception { new Random().nextBytes(testData); @@ -70,6 +79,12 @@ public void setUp() throws Exception { try (final FileOutputStream fos = new FileOutputStream(testFile)) { fos.write(testData); } + + testRandomAccessFile = File.createTempFile(DigestUtilsTest.class.getName(), ".dat"); + try (final FileOutputStream fos = new FileOutputStream(testRandomAccessFile)) { + fos.write(testData); + } + testRandomAccessFileWrapper = new RandomAccessFile(testRandomAccessFile, "rw"); } @After @@ -77,6 +92,10 @@ public void tearDown() { if (!testFile.delete()) { testFile.deleteOnExit(); } + + if (!testRandomAccessFile.delete()) { + testRandomAccessFile.deleteOnExit(); + } } @Test(expected=IllegalArgumentException.class) diff --git a/src/test/java/org/apache/commons/codec/digest/MessageDigestAlgorithmsTest.java b/src/test/java/org/apache/commons/codec/digest/MessageDigestAlgorithmsTest.java index 9295c8aa94..b31de8303b 100644 --- a/src/test/java/org/apache/commons/codec/digest/MessageDigestAlgorithmsTest.java +++ b/src/test/java/org/apache/commons/codec/digest/MessageDigestAlgorithmsTest.java @@ -19,6 +19,7 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.nio.ByteBuffer; @@ -106,6 +107,10 @@ private File getTestFile() { return digestUtilsTest.getTestFile(); } + private RandomAccessFile getTestRandomAccessFile() { + return digestUtilsTest.getTestRandomAccessFile(); + } + @Before public void setUp() throws Exception { digestUtilsTest = new DigestUtilsTest(); @@ -151,6 +156,25 @@ public void testDigestFile() throws IOException { Assert.assertArrayEquals(digestTestData(), DigestUtils.digest(DigestUtils.getDigest(messageDigestAlgorithm),getTestFile())); } + @Test + public void testNonBlockingDigestRandomAccessFile() throws IOException { + Assume.assumeTrue(DigestUtils.isAvailable(messageDigestAlgorithm)); + + final byte[] expected = digestTestData(); + + Assert.assertArrayEquals(expected, + DigestUtils.digest( + DigestUtils.getDigest(messageDigestAlgorithm), getTestRandomAccessFile() + ) + ); + + Assert.assertArrayEquals(expected, + DigestUtils.digest( + DigestUtils.getDigest(messageDigestAlgorithm), getTestRandomAccessFile() + ) + ); + } + @Test public void testDigestInputStream() throws IOException { Assume.assumeTrue(DigestUtils.isAvailable(messageDigestAlgorithm));