Merge pull request #197 from MmAaXx500/zstd

Add zstd support

.drone-local.yml

@@ -111,6 +111,24 @@ steps:
       AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
       AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 
+  - name: rebuild-cache-with-zstd
+    image: drone-cache:MyTestTag
+    pull: if-not-exists
+    settings:
+      archive_format: zstd
+      bucket: drone-cache-bucket
+      cache_key: zstd
+      mount:
+        - vendor
+      rebuild: true
+      region: eu-west-1
+      path_style: true
+      endpoint: minio:9000
+      exit_code: true
+    environment:
+      AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
+      AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
   - name: rebuild-cache-with-filesystem
     image: drone-cache:MyTestTag
     pull: if-not-exists
@@ -162,6 +180,24 @@ steps:
       AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
       AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 
+  - name: restore-cache-with-zstd
+    image: drone-cache:MyTestTag
+    pull: if-not-exists
+    settings:
+      archive_format: zstd
+      bucket: drone-cache-bucket
+      cache_key: zstd
+      mount:
+        - vendor
+      region: eu-west-1
+      restore: true
+      path_style: true
+      endpoint: minio:9000
+      exit_code: true
+    environment:
+      AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
+      AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
   - name: restore-cache-with-filesystem
     image: drone-cache:MyTestTag
     pull: if-not-exists

.drone.yml

@@ -117,6 +117,24 @@ steps:
     environment:
       AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
       AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
+  - name: rebuild-cache-with-zstd
+    image: meltwater/drone-cache:v1.2.2
+    pull: always
+    settings:
+      archive_format: zstd
+      bucket: drone-cache-bucket
+      cache_key: zstd
+      mount:
+        - vendor
+      rebuild: true
+      region: eu-west-1
+      path_style: true
+      endpoint: minio:9000
+      exit_code: true
+    environment:
+      AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
+      AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 
   - name: rebuild-cache-with-filesystem
     image: meltwater/drone-cache:v1.2.2
@@ -169,6 +187,24 @@ steps:
       AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
       AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 
+  - name: restore-cache-with-zstd
+    image: meltwater/drone-cache:v1.2.2
+    pull: always
+    settings:
+      archive_format: zstd
+      bucket: drone-cache-bucket
+      cache_key: zstd
+      mount:
+        - vendor
+      region: eu-west-1
+      restore: true
+      path_style: true
+      endpoint: minio:9000
+      exit_code: true
+    environment:
+      AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
+      AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
   - name: restore-cache-with-filesystem
     image: meltwater/drone-cache:v1.2.2
     pull: always

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## Unreleased
 
 - [#191](https://github.com/meltwater/drone-cache/issues/191) Update examples to reference non-dev images
+- [#197](https://github.com/meltwater/drone-cache/pull/197) Zstd support
 
 ### Added
 

DOCS.md

@@ -332,7 +332,7 @@ cache_key
 : cache key to use for the cache directories
 
 archive_format
-: archive format to use to store the cache directories (`tar`, `gzip`) (default: `tar`)
+: archive format to use to store the cache directories (`tar`, `gzip`, `zstd`) (default: `tar`)
 
 override
 : override already existing cache files (default: `true`)

README.md

@@ -165,9 +165,10 @@ GLOBAL OPTIONS:
    --remote-root value                   remote root directory to contain all the cache files created (default repo.name) [$PLUGIN_REMOTE_ROOT]
    --local-root value                    local root directory to base given mount paths (default pwd [present working directory]) [$PLUGIN_LOCAL_ROOT]
    --override                            override even if cache key already exists in backend (default: true) [$PLUGIN_OVERRIDE]
-   --archive-format value                archive format to use to store the cache directories (tar, gzip) (default: "tar") [$PLUGIN_ARCHIVE_FORMAT]
-   --compression-level value             compression level to use for gzip compression when archive-format specified as gzip
-                                             (check https://godoc.org/compress/flate#pkg-constants for available options) (default: -1) [$PLUGIN_COMPRESSION_LEVEL]
+   --archive-format value                archive format to use to store the cache directories (tar, gzip, zstd) (default: "tar") [$PLUGIN_ARCHIVE_FORMAT]
+   --compression-level value             compression level to use for gzip/zstd compression when archive-format specified as gzip/zstd
+                                             (check https://godoc.org/compress/flate#pkg-constants for available options for gzip
+                                             and https://pkg.go.dev/github.com/klauspost/compress/zstd#EncoderLevelFromZstd for zstd) (default: -1) [$PLUGIN_COMPRESSION_LEVEL]
    --skip-symlinks                       skip symbolic links in archive (default: false) [$PLUGIN_SKIP_SYMLINKS, $SKIP_SYMLINKS]
    --debug                               debug (default: false) [$PLUGIN_DEBUG, $DEBUG]
    --backend.operation-timeout value     timeout value to use for each storage operations (default: 3m0s) [$PLUGIN_BACKEND_OPERATION_TIMEOUT, $BACKEND_OPERATION_TIMEOUT]

archive/archive.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/meltwater/drone-cache/archive/gzip"
 	"github.com/meltwater/drone-cache/archive/tar"
+	"github.com/meltwater/drone-cache/archive/zstd"
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
@@ -14,6 +15,7 @@ import (
 const (
 	Gzip = "gzip"
 	Tar  = "tar"
+	Zstd = "zstd"
 
 	DefaultCompressionLevel = flate.DefaultCompression
 	DefaultArchiveFormat    = Tar
@@ -45,6 +47,8 @@ func FromFormat(logger log.Logger, root string, format string, opts ...Option) A
 		return gzip.New(logger, root, options.skipSymlinks, options.compressionLevel)
 	case Tar:
 		return tar.New(logger, root, options.skipSymlinks)
+	case Zstd:
+		return zstd.New(logger, root, options.skipSymlinks, options.compressionLevel)
 	default:
 		level.Error(logger).Log("msg", "unknown archive format", "format", format)
 		return tar.New(logger, root, options.skipSymlinks) // DefaultArchiveFormat

archive/zstd/zstd.go

@@ -0,0 +1,51 @@
+package zstd
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/klauspost/compress/zstd"
+
+	"github.com/meltwater/drone-cache/archive/tar"
+	"github.com/meltwater/drone-cache/internal"
+
+	"github.com/go-kit/kit/log"
+)
+
+// Archive implements archive for zstd.
+type Archive struct {
+	logger log.Logger
+
+	root             string
+	compressionLevel int
+	skipSymlinks     bool
+}
+
+// New creates an archive that uses the .tar.zst file format.
+func New(logger log.Logger, root string, skipSymlinks bool, compressionLevel int) *Archive {
+	return &Archive{logger, root, compressionLevel, skipSymlinks}
+}
+
+// Create writes content of the given source to an archive, returns written bytes.
+func (a *Archive) Create(srcs []string, w io.Writer) (int64, error) {
+	zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(a.compressionLevel)))
+	if err != nil {
+		return 0, fmt.Errorf("create archive writer, %w", err)
+	}
+
+	defer internal.CloseWithErrLogf(a.logger, zw, "zstd writer")
+
+	return tar.New(a.logger, a.root, a.skipSymlinks).Create(srcs, zw)
+}
+
+// Extract reads content from the given archive reader and restores it to the destination, returns written bytes.
+func (a *Archive) Extract(dst string, r io.Reader) (int64, error) {
+	zr, err := zstd.NewReader(r)
+	if err != nil {
+		return 0, err
+	}
+
+	defer internal.CloseWithErrLogf(a.logger, zr.IOReadCloser(), "zstd reader")
+
+	return tar.New(a.logger, a.root, a.skipSymlinks).Extract(dst, zr)
+}