From bb08ccc5c14bc01b3ccfa628369f0dae9b964d11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Boutemy?= Date: Sun, 8 Sep 2019 10:20:04 +0200 Subject: [PATCH 1/6] added configureReproducible( int sourceDateEpoch ) API --- .../plexus/archiver/AbstractArchiver.java | 28 +++++++++++++++++++ .../codehaus/plexus/archiver/Archiver.java | 17 +++++++++++ .../archiver/diags/DelgatingArchiver.java | 7 +++++ .../plexus/archiver/diags/NoOpArchiver.java | 6 ++++ .../archiver/diags/TrackingArchiver.java | 5 ++++ .../archiver/zip/AbstractZipArchiver.java | 11 ++++++++ 6 files changed, 74 insertions(+) diff --git a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java index d5f80c17b..a3923b192 100755 --- a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java @@ -22,6 +22,7 @@ import java.lang.reflect.UndeclaredThrowableException; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Calendar; import java.util.Comparator; import java.util.Date; import java.util.HashMap; @@ -1257,4 +1258,31 @@ public String getOverrideGroupName() { return overrideGroupName; } + + @Override + public void configureReproducible( int sourceDateEpoch ) + { + // 1. force last modified date + setLastModifiedDate( convertSourceDateEpochToDate( sourceDateEpoch ) ); + + // 2. sort filenames in each directory when scanning filesystem + setFilenameComparator( new Comparator() + { + @Override + public int compare( String s1, String s2 ) + { + return s1.compareTo( s2 ); + } + } ); + + // 3. ignore file/directory mode from filesystem, since they may vary based on local user umask + // notice: this overrides execute bit on Unix (that is already ignored on Windows) + setFileMode( Archiver.DEFAULT_FILE_MODE ); + setDirectoryMode( Archiver.DEFAULT_DIR_MODE ); + } + + protected Date convertSourceDateEpochToDate( int sourceDateEpoch ) + { + return new Date( sourceDateEpoch * 1000L ); + } } diff --git a/src/main/java/org/codehaus/plexus/archiver/Archiver.java b/src/main/java/org/codehaus/plexus/archiver/Archiver.java index b6d9d8581..dee7b8704 100644 --- a/src/main/java/org/codehaus/plexus/archiver/Archiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/Archiver.java @@ -415,11 +415,15 @@ ResourceIterator getResources() Date getLastModifiedDate(); /** + * Set filename comparator, used to sort file entries when scanning directories since File.list() does not + * guarantee any order. + * * @since 4.2.0 */ void setFilenameComparator( Comparator filenameComparator ); /** +<<<<<<< reproducible * @since 4.2.0 */ void setOverrideUid( int uid ); @@ -458,4 +462,17 @@ ResourceIterator getResources() * @since 4.2.0 */ String getOverrideGroupName(); + + /** + * Configure the archiver to get reproducible archives (see defined in Reproducible Builds: + * a UNIX timestamp, defined as the number of seconds, excluding leap seconds, since 01 Jan 1970 00:00:00 + * UTC. + * @since 4.2.0 + */ + void configureReproducible( int sourceDateEpoch ); } diff --git a/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java b/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java index 7dbad752b..5a93605d1 100644 --- a/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java @@ -398,4 +398,11 @@ public String getOverrideGroupName() { return target.getOverrideGroupName(); } + + @Override + public void configureReproducible( int sourceDateEpoch ) + { + target.configureReproducible( sourceDateEpoch ); + } + } diff --git a/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java b/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java index c07389a54..fe08e30f6 100644 --- a/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java @@ -414,4 +414,10 @@ public String getOverrideGroupName() return null; } + @Override + public void configureReproducible( int sourceDateEpoch ) + { + + } + } diff --git a/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java b/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java index fcb28e3ad..0fbaae058 100644 --- a/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java @@ -468,4 +468,9 @@ public String getOverrideGroupName() { return null; } + + @Override + public void configureReproducible( int sourceDateEpoch ) + { + } } diff --git a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java index bdc2fc93d..9ab4539a9 100755 --- a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java @@ -24,6 +24,8 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.charset.Charset; +import java.util.Calendar; +import java.util.Date; import java.util.Hashtable; import java.util.Stack; import java.util.concurrent.ExecutionException; @@ -833,4 +835,13 @@ protected String getArchiveType() return archiveType; } + @Override + protected Date convertSourceDateEpochToDate( int sourceDateEpoch ) + { + // timestamp of zip entries at zip storage level ignores timezone: see https://github.com/Stuk/jszip/issues/369 + Calendar cal = Calendar.getInstance(); + long zipTime = 1000L * sourceDateEpoch - ( cal.get( Calendar.ZONE_OFFSET ) + cal.get( Calendar.DST_OFFSET ) ); + return new Date( zipTime ); + } + } From 2d3bd821790a2885af74c32ec8524baae0f10d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Boutemy?= Date: Sun, 15 Sep 2019 19:52:41 +0200 Subject: [PATCH 2/6] try at better explanation of zip time calculations --- .../plexus/archiver/AbstractArchiver.java | 1 - .../archiver/zip/AbstractZipArchiver.java | 19 +++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java index a3923b192..c07475ae7 100755 --- a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java @@ -22,7 +22,6 @@ import java.lang.reflect.UndeclaredThrowableException; import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.Calendar; import java.util.Comparator; import java.util.Date; import java.util.HashMap; diff --git a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java index 9ab4539a9..20bcb9a5b 100755 --- a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java @@ -838,10 +838,21 @@ protected String getArchiveType() @Override protected Date convertSourceDateEpochToDate( int sourceDateEpoch ) { - // timestamp of zip entries at zip storage level ignores timezone: see https://github.com/Stuk/jszip/issues/369 - Calendar cal = Calendar.getInstance(); - long zipTime = 1000L * sourceDateEpoch - ( cal.get( Calendar.ZONE_OFFSET ) + cal.get( Calendar.DST_OFFSET ) ); - return new Date( zipTime ); + // timestamp of zip entries at zip storage level ignores timezone: managed in ZipEntry.setTime, + // that turns javaToDosTime: need to revert the operation here to get reproducible + // zip entry time + return new Date( dosToJavaTime( 1000L * sourceDateEpoch ) ); } + /** + * Converts DOS time to Java time (number of milliseconds since epoch). + * + * @see java.util.zip.ZipEntry#setTime + * @see java.util.zip.ZipUtils#dosToJavaTime + */ + private static long dosToJavaTime( long dtime ) + { + Calendar cal = Calendar.getInstance(); + return dtime - ( cal.get( Calendar.ZONE_OFFSET ) + cal.get( Calendar.DST_OFFSET ) ); + } } From 6964b62e9f9407da65d8cc6797be11404f27f611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Boutemy?= Date: Sat, 21 Sep 2019 16:48:20 +0200 Subject: [PATCH 3/6] force root/root uid/gid for reproducible tar --- .../java/org/codehaus/plexus/archiver/AbstractArchiver.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java index c07475ae7..359175bba 100755 --- a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java @@ -1278,6 +1278,12 @@ public int compare( String s1, String s2 ) // notice: this overrides execute bit on Unix (that is already ignored on Windows) setFileMode( Archiver.DEFAULT_FILE_MODE ); setDirectoryMode( Archiver.DEFAULT_DIR_MODE ); + + // 4. ignore uid/gid from filesystem (for tar) + setOverrideUid( 0 ); + setOverrideUserName( "root" ); // is it possible to avoid this, like "tar --numeric-owner"? + setOverrideGid( 0 ); + setOverrideGroupName( "root" ); } protected Date convertSourceDateEpochToDate( int sourceDateEpoch ) From 591322728d439467bfc56d49328918af6dbb520e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Boutemy?= Date: Sat, 5 Oct 2019 19:19:21 +0200 Subject: [PATCH 4/6] replace sourceDateEpoch with a Date --- .../org/codehaus/plexus/archiver/AbstractArchiver.java | 8 ++++---- src/main/java/org/codehaus/plexus/archiver/Archiver.java | 7 ++----- .../codehaus/plexus/archiver/diags/DelgatingArchiver.java | 4 ++-- .../org/codehaus/plexus/archiver/diags/NoOpArchiver.java | 2 +- .../codehaus/plexus/archiver/diags/TrackingArchiver.java | 2 +- .../codehaus/plexus/archiver/zip/AbstractZipArchiver.java | 4 ++-- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java index 359175bba..27b9c2a9f 100755 --- a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java @@ -1259,10 +1259,10 @@ public String getOverrideGroupName() } @Override - public void configureReproducible( int sourceDateEpoch ) + public void configureReproducible( Date outputTimestamp ) { // 1. force last modified date - setLastModifiedDate( convertSourceDateEpochToDate( sourceDateEpoch ) ); + setLastModifiedDate( convertOutputTimestamp( outputTimestamp ) ); // 2. sort filenames in each directory when scanning filesystem setFilenameComparator( new Comparator() @@ -1286,8 +1286,8 @@ public int compare( String s1, String s2 ) setOverrideGroupName( "root" ); } - protected Date convertSourceDateEpochToDate( int sourceDateEpoch ) + protected Date convertOutputTimestamp( Date outputTimestamp ) { - return new Date( sourceDateEpoch * 1000L ); + return outputTimestamp; } } diff --git a/src/main/java/org/codehaus/plexus/archiver/Archiver.java b/src/main/java/org/codehaus/plexus/archiver/Archiver.java index dee7b8704..0917eb965 100644 --- a/src/main/java/org/codehaus/plexus/archiver/Archiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/Archiver.java @@ -468,11 +468,8 @@ ResourceIterator getResources() * Builds). This will require reproducible archive entries order, defined entries timestamp and reproducible * entries Unix mode. * - * @param sourceDateEpoch Value like SOURCE_DATE_EPOCH as - * defined in Reproducible Builds: - * a UNIX timestamp, defined as the number of seconds, excluding leap seconds, since 01 Jan 1970 00:00:00 - * UTC. + * @param outputTimestamp the date to use for archive entries last modified time * @since 4.2.0 */ - void configureReproducible( int sourceDateEpoch ); + void configureReproducible( Date outputTimestamp ); } diff --git a/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java b/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java index 5a93605d1..4935986e7 100644 --- a/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java @@ -400,9 +400,9 @@ public String getOverrideGroupName() } @Override - public void configureReproducible( int sourceDateEpoch ) + public void configureReproducible( Date outputTimestamp ) { - target.configureReproducible( sourceDateEpoch ); + target.configureReproducible( outputTimestamp ); } } diff --git a/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java b/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java index fe08e30f6..8c50768e9 100644 --- a/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java @@ -415,7 +415,7 @@ public String getOverrideGroupName() } @Override - public void configureReproducible( int sourceDateEpoch ) + public void configureReproducible( Date outputTimestamp ) { } diff --git a/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java b/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java index 0fbaae058..d63cabb79 100644 --- a/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java @@ -470,7 +470,7 @@ public String getOverrideGroupName() } @Override - public void configureReproducible( int sourceDateEpoch ) + public void configureReproducible( Date outputTimestamp ) { } } diff --git a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java index 20bcb9a5b..3521c2259 100755 --- a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java @@ -836,12 +836,12 @@ protected String getArchiveType() } @Override - protected Date convertSourceDateEpochToDate( int sourceDateEpoch ) + protected Date convertOutputTimestamp( Date outputTimestamp ) { // timestamp of zip entries at zip storage level ignores timezone: managed in ZipEntry.setTime, // that turns javaToDosTime: need to revert the operation here to get reproducible // zip entry time - return new Date( dosToJavaTime( 1000L * sourceDateEpoch ) ); + return new Date( dosToJavaTime( outputTimestamp.getTime() ) ); } /** From ab1f0a9b44bc46d6db1e8e50ad3cc7cf84ea4eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Boutemy?= Date: Sat, 12 Oct 2019 17:11:36 +0200 Subject: [PATCH 5/6] rename a few parameters and internal method names --- .../plexus/archiver/AbstractArchiver.java | 16 ++++++++++++---- .../org/codehaus/plexus/archiver/Archiver.java | 15 +++++++++------ .../plexus/archiver/diags/DelgatingArchiver.java | 4 ++-- .../plexus/archiver/diags/NoOpArchiver.java | 2 +- .../plexus/archiver/diags/TrackingArchiver.java | 2 +- .../plexus/archiver/zip/AbstractZipArchiver.java | 4 ++-- 6 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java index 27b9c2a9f..7b38a937f 100755 --- a/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/AbstractArchiver.java @@ -1259,10 +1259,10 @@ public String getOverrideGroupName() } @Override - public void configureReproducible( Date outputTimestamp ) + public void configureReproducible( Date lastModifiedDate ) { // 1. force last modified date - setLastModifiedDate( convertOutputTimestamp( outputTimestamp ) ); + setLastModifiedDate( normalizeLastModifiedDate( lastModifiedDate ) ); // 2. sort filenames in each directory when scanning filesystem setFilenameComparator( new Comparator() @@ -1286,8 +1286,16 @@ public int compare( String s1, String s2 ) setOverrideGroupName( "root" ); } - protected Date convertOutputTimestamp( Date outputTimestamp ) + /** + * Normalize last modified time value to get reproducible archive entries, based on + * archive binary format (tar uses UTC timestamp but zip uses local time then requires + * tweaks to make the value reproducible whatever the current timezone is). + * + * @param lastModifiedDate + * @return + */ + protected Date normalizeLastModifiedDate( Date lastModifiedDate ) { - return outputTimestamp; + return lastModifiedDate; } } diff --git a/src/main/java/org/codehaus/plexus/archiver/Archiver.java b/src/main/java/org/codehaus/plexus/archiver/Archiver.java index 0917eb965..3c9442a36 100644 --- a/src/main/java/org/codehaus/plexus/archiver/Archiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/Archiver.java @@ -423,7 +423,6 @@ ResourceIterator getResources() void setFilenameComparator( Comparator filenameComparator ); /** -<<<<<<< reproducible * @since 4.2.0 */ void setOverrideUid( int uid ); @@ -464,12 +463,16 @@ ResourceIterator getResources() String getOverrideGroupName(); /** - * Configure the archiver to get reproducible archives (see Reproducible Builds). This will configure: + *
    + *
  • reproducible archive entries order,
    • + *
  • defined entries timestamp
    • + *
  • and reproducible entries Unix mode.
    • + *
      * - * @param outputTimestamp the date to use for archive entries last modified time + * @param lastModifiedDate the date to use for archive entries last modified time * @since 4.2.0 */ - void configureReproducible( Date outputTimestamp ); + void configureReproducible( Date lastModifiedDate ); } diff --git a/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java b/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java index 4935986e7..e4b8a807f 100644 --- a/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/diags/DelgatingArchiver.java @@ -400,9 +400,9 @@ public String getOverrideGroupName() } @Override - public void configureReproducible( Date outputTimestamp ) + public void configureReproducible( Date lastModifiedDate ) { - target.configureReproducible( outputTimestamp ); + target.configureReproducible( lastModifiedDate ); } } diff --git a/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java b/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java index 8c50768e9..2cca73291 100644 --- a/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/diags/NoOpArchiver.java @@ -415,7 +415,7 @@ public String getOverrideGroupName() } @Override - public void configureReproducible( Date outputTimestamp ) + public void configureReproducible( Date lastModifiedDate ) { } diff --git a/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java b/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java index d63cabb79..c1fb40c43 100644 --- a/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/diags/TrackingArchiver.java @@ -470,7 +470,7 @@ public String getOverrideGroupName() } @Override - public void configureReproducible( Date outputTimestamp ) + public void configureReproducible( Date lastModifiedDate ) { } } diff --git a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java index 3521c2259..9d0275e02 100755 --- a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java @@ -836,12 +836,12 @@ protected String getArchiveType() } @Override - protected Date convertOutputTimestamp( Date outputTimestamp ) + protected Date normalizeLastModifiedDate( Date lastModifiedDate ) { // timestamp of zip entries at zip storage level ignores timezone: managed in ZipEntry.setTime, // that turns javaToDosTime: need to revert the operation here to get reproducible // zip entry time - return new Date( dosToJavaTime( outputTimestamp.getTime() ) ); + return new Date( dosToJavaTime( lastModifiedDate.getTime() ) ); } /** From 4b42ae8efe8dfd6f660964dfb9738882164dfdc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Boutemy?= Date: Sat, 12 Oct 2019 18:18:03 +0200 Subject: [PATCH 6/6] renamed parameter --- .../org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java index 9d0275e02..449ffe886 100755 --- a/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java +++ b/src/main/java/org/codehaus/plexus/archiver/zip/AbstractZipArchiver.java @@ -850,9 +850,9 @@ protected Date normalizeLastModifiedDate( Date lastModifiedDate ) * @see java.util.zip.ZipEntry#setTime * @see java.util.zip.ZipUtils#dosToJavaTime */ - private static long dosToJavaTime( long dtime ) + private static long dosToJavaTime( long dosTime ) { Calendar cal = Calendar.getInstance(); - return dtime - ( cal.get( Calendar.ZONE_OFFSET ) + cal.get( Calendar.DST_OFFSET ) ); + return dosTime - ( cal.get( Calendar.ZONE_OFFSET ) + cal.get( Calendar.DST_OFFSET ) ); } }