From 80c9f4f52ff3057f3bad0555d9a023b72a059be4 Mon Sep 17 00:00:00 2001 From: MmAaXx500 Date: Fri, 18 Feb 2022 19:44:01 +0100 Subject: [PATCH] Zstd support --- .drone-local.yml | 36 ++++ .drone.yml | 36 ++++ CHANGELOG.md | 1 + DOCS.md | 2 +- README.md | 7 +- archive/archive.go | 4 + archive/zstd/zstd.go | 51 ++++++ archive/zstd/zstd_test.go | 318 +++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 23 +-- internal/plugin/plugin_test.go | 2 + main.go | 7 +- 12 files changed, 460 insertions(+), 28 deletions(-) create mode 100644 archive/zstd/zstd.go create mode 100644 archive/zstd/zstd_test.go diff --git a/.drone-local.yml b/.drone-local.yml index 4cca669d..ecc07a1e 100644 --- a/.drone-local.yml +++ b/.drone-local.yml @@ -111,6 +111,24 @@ steps: AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + - name: rebuild-cache-with-zstd + image: drone-cache:MyTestTag + pull: if-not-exists + settings: + archive_format: zstd + bucket: drone-cache-bucket + cache_key: zstd + mount: + - vendor + rebuild: true + region: eu-west-1 + path_style: true + endpoint: minio:9000 + exit_code: true + environment: + AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE + AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + - name: rebuild-cache-with-filesystem image: drone-cache:MyTestTag pull: if-not-exists @@ -162,6 +180,24 @@ steps: AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + - name: restore-cache-with-zstd + image: drone-cache:MyTestTag + pull: if-not-exists + settings: + archive_format: zstd + bucket: drone-cache-bucket + cache_key: zstd + mount: + - vendor + region: eu-west-1 + restore: true + path_style: true + endpoint: minio:9000 + exit_code: true + environment: + AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE + AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + - name: restore-cache-with-filesystem image: drone-cache:MyTestTag pull: if-not-exists diff --git a/.drone.yml b/.drone.yml index 245d4f40..6f1087ae 100644 --- a/.drone.yml +++ b/.drone.yml @@ -117,6 +117,24 @@ steps: environment: AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + + - name: rebuild-cache-with-zstd + image: meltwater/drone-cache:v1.2.2 + pull: always + settings: + archive_format: zstd + bucket: drone-cache-bucket + cache_key: zstd + mount: + - vendor + rebuild: true + region: eu-west-1 + path_style: true + endpoint: minio:9000 + exit_code: true + environment: + AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE + AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY - name: rebuild-cache-with-filesystem image: meltwater/drone-cache:v1.2.2 @@ -169,6 +187,24 @@ steps: AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + - name: restore-cache-with-zstd + image: meltwater/drone-cache:v1.2.2 + pull: always + settings: + archive_format: zstd + bucket: drone-cache-bucket + cache_key: zstd + mount: + - vendor + region: eu-west-1 + restore: true + path_style: true + endpoint: minio:9000 + exit_code: true + environment: + AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE + AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + - name: restore-cache-with-filesystem image: meltwater/drone-cache:v1.2.2 pull: always diff --git a/CHANGELOG.md b/CHANGELOG.md index 254c8006..8757f991 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased - [#191](https://github.com/meltwater/drone-cache/issues/191) Update examples to reference non-dev images +- [#197](https://github.com/meltwater/drone-cache/pull/197) Zstd support ### Added diff --git a/DOCS.md b/DOCS.md index 21bb4ff1..c9797357 100644 --- a/DOCS.md +++ b/DOCS.md @@ -332,7 +332,7 @@ cache_key : cache key to use for the cache directories archive_format -: archive format to use to store the cache directories (`tar`, `gzip`) (default: `tar`) +: archive format to use to store the cache directories (`tar`, `gzip`, `zstd`) (default: `tar`) override : override already existing cache files (default: `true`) diff --git a/README.md b/README.md index bd1937f0..7bd787df 100644 --- a/README.md +++ b/README.md @@ -165,9 +165,10 @@ GLOBAL OPTIONS: --remote-root value remote root directory to contain all the cache files created (default repo.name) [$PLUGIN_REMOTE_ROOT] --local-root value local root directory to base given mount paths (default pwd [present working directory]) [$PLUGIN_LOCAL_ROOT] --override override even if cache key already exists in backend (default: true) [$PLUGIN_OVERRIDE] - --archive-format value archive format to use to store the cache directories (tar, gzip) (default: "tar") [$PLUGIN_ARCHIVE_FORMAT] - --compression-level value compression level to use for gzip compression when archive-format specified as gzip - (check https://godoc.org/compress/flate#pkg-constants for available options) (default: -1) [$PLUGIN_COMPRESSION_LEVEL] + --archive-format value archive format to use to store the cache directories (tar, gzip, zstd) (default: "tar") [$PLUGIN_ARCHIVE_FORMAT] + --compression-level value compression level to use for gzip/zstd compression when archive-format specified as gzip/zstd + (check https://godoc.org/compress/flate#pkg-constants for available options for gzip + and https://pkg.go.dev/github.com/klauspost/compress/zstd#EncoderLevelFromZstd for zstd) (default: -1) [$PLUGIN_COMPRESSION_LEVEL] --skip-symlinks skip symbolic links in archive (default: false) [$PLUGIN_SKIP_SYMLINKS, $SKIP_SYMLINKS] --debug debug (default: false) [$PLUGIN_DEBUG, $DEBUG] --backend.operation-timeout value timeout value to use for each storage operations (default: 3m0s) [$PLUGIN_BACKEND_OPERATION_TIMEOUT, $BACKEND_OPERATION_TIMEOUT] diff --git a/archive/archive.go b/archive/archive.go index 67f976f8..6174a297 100644 --- a/archive/archive.go +++ b/archive/archive.go @@ -6,6 +6,7 @@ import ( "github.com/meltwater/drone-cache/archive/gzip" "github.com/meltwater/drone-cache/archive/tar" + "github.com/meltwater/drone-cache/archive/zstd" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" @@ -14,6 +15,7 @@ import ( const ( Gzip = "gzip" Tar = "tar" + Zstd = "zstd" DefaultCompressionLevel = flate.DefaultCompression DefaultArchiveFormat = Tar @@ -45,6 +47,8 @@ func FromFormat(logger log.Logger, root string, format string, opts ...Option) A return gzip.New(logger, root, options.skipSymlinks, options.compressionLevel) case Tar: return tar.New(logger, root, options.skipSymlinks) + case Zstd: + return zstd.New(logger, root, options.skipSymlinks, options.compressionLevel) default: level.Error(logger).Log("msg", "unknown archive format", "format", format) return tar.New(logger, root, options.skipSymlinks) // DefaultArchiveFormat diff --git a/archive/zstd/zstd.go b/archive/zstd/zstd.go new file mode 100644 index 00000000..a92034d0 --- /dev/null +++ b/archive/zstd/zstd.go @@ -0,0 +1,51 @@ +package zstd + +import ( + "fmt" + "io" + + "github.com/klauspost/compress/zstd" + + "github.com/meltwater/drone-cache/archive/tar" + "github.com/meltwater/drone-cache/internal" + + "github.com/go-kit/kit/log" +) + +// Archive implements archive for zstd. +type Archive struct { + logger log.Logger + + root string + compressionLevel int + skipSymlinks bool +} + +// New creates an archive that uses the .tar.zst file format. +func New(logger log.Logger, root string, skipSymlinks bool, compressionLevel int) *Archive { + return &Archive{logger, root, compressionLevel, skipSymlinks} +} + +// Create writes content of the given source to an archive, returns written bytes. +func (a *Archive) Create(srcs []string, w io.Writer) (int64, error) { + zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(a.compressionLevel))) + if err != nil { + return 0, fmt.Errorf("create archive writer, %w", err) + } + + defer internal.CloseWithErrLogf(a.logger, zw, "zstd writer") + + return tar.New(a.logger, a.root, a.skipSymlinks).Create(srcs, zw) +} + +// Extract reads content from the given archive reader and restores it to the destination, returns written bytes. +func (a *Archive) Extract(dst string, r io.Reader) (int64, error) { + zr, err := zstd.NewReader(r) + if err != nil { + return 0, err + } + + defer internal.CloseWithErrLogf(a.logger, zr.IOReadCloser(), "zstd reader") + + return tar.New(a.logger, a.root, a.skipSymlinks).Extract(dst, zr) +} diff --git a/archive/zstd/zstd_test.go b/archive/zstd/zstd_test.go new file mode 100644 index 00000000..22b3a172 --- /dev/null +++ b/archive/zstd/zstd_test.go @@ -0,0 +1,318 @@ +package zstd + +import ( + "compress/flate" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/go-kit/kit/log" + + "github.com/meltwater/drone-cache/archive/tar" + "github.com/meltwater/drone-cache/test" +) + +var ( + testRoot = "testdata" + testRootMounted = "testdata/mounted" + testRootExtracted = "testdata/extracted" +) + +func TestCreate(t *testing.T) { + test.Ok(t, os.MkdirAll(testRootMounted, 0755)) + test.Ok(t, os.MkdirAll(testRootExtracted, 0755)) + t.Cleanup(func() { os.RemoveAll(testRoot) }) + + for _, tc := range []struct { + name string + tzst *Archive + srcs []string + written int64 + err error + }{ + { + name: "empty mount paths", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + srcs: []string{}, + written: 0, + err: nil, + }, + { + name: "non-existing mount paths", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + srcs: []string{ + "iamnotexists", + "metoo", + }, + written: 0, + err: tar.ErrSourceNotReachable, // os.ErrNotExist || os.ErrPermission + }, + { + name: "existing mount paths", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + srcs: exampleFileTree(t, "zstd_create"), + written: 43, // 3 x tmpfile in dir, 1 tmpfile + err: nil, + }, + { + name: "existing mount nested paths", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + srcs: exampleNestedFileTree(t, "tar_create"), + written: 56, // 4 x tmpfile in dir, 1 tmpfile + err: nil, + }, + { + name: "existing mount paths with symbolic links", + tzst: New(log.NewNopLogger(), testRootMounted, false, flate.DefaultCompression), + srcs: exampleFileTreeWithSymlinks(t, "zstd_create_symlink"), + written: 43, + err: nil, + }, + } { + tc := tc // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // Setup + dstDir, dstDirClean := test.CreateTempDir(t, "zstd_create_archives", testRootMounted) + t.Cleanup(dstDirClean) + + extDir, extDirClean := test.CreateTempDir(t, "zstd_create_extracted", testRootExtracted) + t.Cleanup(extDirClean) + + // Run + archivePath := filepath.Join(dstDir, filepath.Clean(tc.name+".tar.zst")) + written, err := create(tc.tzst, tc.srcs, archivePath) + if err != nil { + test.Expected(t, err, tc.err) + return + } + + test.Exists(t, archivePath) + test.Assert(t, written == tc.written, "case %q: written bytes got %d want %v", tc.name, written, tc.written) + + _, err = extract(tc.tzst, archivePath, extDir) + test.Ok(t, err) + test.EqualDirs(t, extDir, testRootMounted, tc.srcs) + }) + } +} + +func TestExtract(t *testing.T) { + test.Ok(t, os.MkdirAll(testRootMounted, 0755)) + test.Ok(t, os.MkdirAll(testRootExtracted, 0755)) + t.Cleanup(func() { os.RemoveAll(testRoot) }) + + // Setup + tzst := New(log.NewNopLogger(), testRootMounted, false, flate.DefaultCompression) + + arcDir, arcDirClean := test.CreateTempDir(t, "zstd_extract_archive") + t.Cleanup(arcDirClean) + + files := exampleFileTree(t, "zstd_extract") + archivePath := filepath.Join(arcDir, "test.tar.zst") + _, err := create(tzst, files, archivePath) + test.Ok(t, err) + + nestedFiles := exampleNestedFileTree(t, "zstd_extract_nested") + nestedArchivePath := filepath.Join(arcDir, "nested_test.tar.zst") + _, err = create(tzst, nestedFiles, nestedArchivePath) + test.Ok(t, err) + + filesWithSymlink := exampleFileTreeWithSymlinks(t, "zstd_extract_symlink") + archiveWithSymlinkPath := filepath.Join(arcDir, "test_with_symlink.tar.zst") + _, err = create(tzst, filesWithSymlink, archiveWithSymlinkPath) + test.Ok(t, err) + + emptyArchivePath := filepath.Join(arcDir, "empty_test.tar.zst") + _, err = create(tzst, []string{}, emptyArchivePath) + test.Ok(t, err) + + badArchivePath := filepath.Join(arcDir, "bad_test.tar.zst") + test.Ok(t, ioutil.WriteFile(badArchivePath, []byte("hello\ndrone\n"), 0644)) + + for _, tc := range []struct { + name string + tzst *Archive + archivePath string + srcs []string + written int64 + err error + }{ + { + name: "non-existing archive", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + archivePath: "iamnotexists", + srcs: []string{}, + written: 0, + err: os.ErrNotExist, + }, + { + name: "non-existing root destination", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + archivePath: emptyArchivePath, + srcs: []string{}, + written: 0, + err: nil, + }, + { + name: "empty archive", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + archivePath: emptyArchivePath, + srcs: []string{}, + written: 0, + err: nil, + }, + { + name: "bad archives", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + archivePath: badArchivePath, + srcs: []string{}, + written: 0, + err: tar.ErrArchiveNotReadable, + }, + { + name: "existing archive", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + archivePath: archivePath, + srcs: files, + written: 43, + err: nil, + }, + { + name: "existing archive with nested files", + tzst: New(log.NewNopLogger(), testRootMounted, true, flate.DefaultCompression), + archivePath: nestedArchivePath, + srcs: nestedFiles, + written: 56, + err: nil, + }, + { + name: "existing archive with symbolic links", + tzst: New(log.NewNopLogger(), testRootMounted, false, flate.DefaultCompression), + archivePath: archiveWithSymlinkPath, + srcs: filesWithSymlink, + written: 43, + err: nil, + }, + } { + tc := tc // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + dstDir, dstDirClean := test.CreateTempDir(t, "zstd_extract_"+tc.name, testRootExtracted) + t.Cleanup(dstDirClean) + + written, err := extract(tc.tzst, tc.archivePath, dstDir) + if err != nil { + test.Expected(t, err, tc.err) + return + } + + test.Assert(t, written == tc.written, "case %q: written bytes got %d want %v", tc.name, written, tc.written) + test.EqualDirs(t, dstDir, testRootMounted, tc.srcs) + }) + } +} + +// Helpers + +func create(a *Archive, srcs []string, dst string) (int64, error) { + pr, pw := io.Pipe() + defer pr.Close() + + var written int64 + go func(w *int64) { + defer pw.Close() + + written, err := a.Create(srcs, pw) + if err != nil { + pw.CloseWithError(err) + } + + *w = written + }(&written) + + content, err := ioutil.ReadAll(pr) + if err != nil { + pr.CloseWithError(err) + return 0, err + } + + if err := ioutil.WriteFile(dst, content, 0644); err != nil { + return 0, err + } + + return written, nil +} + +func extract(a *Archive, src string, dst string) (int64, error) { + pr, pw := io.Pipe() + defer pr.Close() + + f, err := os.Open(src) + if err != nil { + return 0, err + } + + go func() { + defer pw.Close() + + _, err = io.Copy(pw, f) + if err != nil { + pw.CloseWithError(err) + } + }() + + return a.Extract(dst, pr) +} + +// Fixtures + +func exampleFileTree(t *testing.T, name string) []string { + file, fileClean := test.CreateTempFile(t, name, []byte("hello\ndrone!\n"), testRootMounted) // 13 bytes + t.Cleanup(fileClean) + + dir, dirClean := test.CreateTempFilesInDir(t, name, []byte("hello\ngo!\n"), testRootMounted) // 10 bytes + t.Cleanup(dirClean) + + return []string{file, dir} +} + +func exampleNestedFileTree(t *testing.T, name string) []string { + dir, cleanup := test.CreateTempDir(t, name, testRootMounted) + t.Cleanup(cleanup) + + nestedFile, nestedFileClean := test.CreateTempFile(t, name, []byte("hello\ndrone!\n"), dir) // 13 bytes + t.Cleanup(nestedFileClean) + + nestedDir, nestedDirClean := test.CreateTempFilesInDir(t, name, []byte("hello\ngo!\n"), dir) // 10 bytes + t.Cleanup(nestedDirClean) + + nestedDir1, nestedDirClean1 := test.CreateTempDir(t, name, dir) + t.Cleanup(nestedDirClean1) + + nestedDir2, nestedDirClean2 := test.CreateTempDir(t, name, nestedDir1) + t.Cleanup(nestedDirClean2) + + nestedFile1, nestedFileClean1 := test.CreateTempFile(t, name, []byte("hello\ndrone!\n"), nestedDir2) // 13 bytes + t.Cleanup(nestedFileClean1) + + return []string{nestedDir, nestedFile, nestedFile1} +} + +func exampleFileTreeWithSymlinks(t *testing.T, name string) []string { + file, fileClean := test.CreateTempFile(t, name, []byte("hello\ndrone!\n"), testRootMounted) // 13 bytes + t.Cleanup(fileClean) + + symlink := filepath.Join(filepath.Dir(file), name+"_symlink.testfile") + test.Ok(t, os.Symlink(file, symlink)) + t.Cleanup(func() { os.Remove(symlink) }) + + dir, dirClean := test.CreateTempFilesInDir(t, name, []byte("hello\ngo!\n"), testRootMounted) // 10 bytes + t.Cleanup(dirClean) + + return []string{file, dir, symlink} +} diff --git a/go.mod b/go.mod index e3181995..9f750d91 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/go-logfmt/logfmt v0.4.0 // indirect github.com/go-stack/stack v1.8.0 // indirect github.com/google/go-cmp v0.4.0 + github.com/klauspost/compress v1.13.5 github.com/pkg/sftp v1.10.1 github.com/urfave/cli/v2 v2.1.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 diff --git a/go.sum b/go.sum index 4894a1bf..bd480eb4 100644 --- a/go.sum +++ b/go.sum @@ -67,7 +67,6 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -88,8 +87,9 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= @@ -104,7 +104,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -128,12 +127,9 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -156,23 +152,19 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -184,7 +176,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -192,22 +183,17 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -226,7 +212,6 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190917162342-3b4f30a44f3b h1:5PDpbTpVmeVPIQOoxshLbs4ATaIDQrZN5z3nTUtm2+8= golang.org/x/tools v0.0.0-20190917162342-3b4f30a44f3b/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -259,22 +244,18 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= diff --git a/internal/plugin/plugin_test.go b/internal/plugin/plugin_test.go index acb4dba1..6d832a92 100644 --- a/internal/plugin/plugin_test.go +++ b/internal/plugin/plugin_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package plugin @@ -60,6 +61,7 @@ var ( formats = []string{ archive.Gzip, archive.Tar, + archive.Zstd, } ) diff --git a/main.go b/main.go index 0f06c68c..a948836d 100644 --- a/main.go +++ b/main.go @@ -272,14 +272,15 @@ func main() { // RESTORE-KEYS &cli.StringFlag{ Name: "archive-format, arcfmt", - Usage: "archive format to use to store the cache directories (tar, gzip)", + Usage: "archive format to use to store the cache directories (tar, gzip, zstd)", Value: archive.DefaultArchiveFormat, EnvVars: []string{"PLUGIN_ARCHIVE_FORMAT"}, }, &cli.IntFlag{ Name: "compression-level, cpl", - Usage: `compression level to use for gzip compression when archive-format specified as gzip - (check https://godoc.org/compress/flate#pkg-constants for available options)`, + Usage: `compression level to use for gzip/zstd compression when archive-format specified as gzip/zstd + (check https://godoc.org/compress/flate#pkg-constants for available options for gzip + and https://pkg.go.dev/github.com/klauspost/compress/zstd#EncoderLevelFromZstd for zstd)`, Value: archive.DefaultCompressionLevel, EnvVars: []string{"PLUGIN_COMPRESSION_LEVEL"}, },