From 0c030392d5564f36fa5b2a51a93db85eab5fea93 Mon Sep 17 00:00:00 2001 From: Carlos A Becker Date: Wed, 11 May 2022 22:41:47 -0300 Subject: [PATCH 1/7] feat: add extra files to source archives Signed-off-by: Carlos A Becker --- internal/pipe/sourcearchive/source.go | 60 ++++++++++++++++++---- internal/pipe/sourcearchive/source_test.go | 38 +++++++++++++- pkg/config/config.go | 1 + www/docs/customization/source.md | 27 ++++++++++ 4 files changed, 114 insertions(+), 12 deletions(-) diff --git a/internal/pipe/sourcearchive/source.go b/internal/pipe/sourcearchive/source.go index 14fcdc9e3a8..0507c252d7f 100644 --- a/internal/pipe/sourcearchive/source.go +++ b/internal/pipe/sourcearchive/source.go @@ -2,12 +2,18 @@ package sourcearchive import ( + "fmt" + "os" "path/filepath" + "strings" "github.com/apex/log" + "github.com/goreleaser/goreleaser/internal/archivefiles" "github.com/goreleaser/goreleaser/internal/artifact" "github.com/goreleaser/goreleaser/internal/git" "github.com/goreleaser/goreleaser/internal/tmpl" + "github.com/goreleaser/goreleaser/pkg/archive" + "github.com/goreleaser/goreleaser/pkg/config" "github.com/goreleaser/goreleaser/pkg/context" ) @@ -31,20 +37,52 @@ func (Pipe) Run(ctx *context.Context) (err error) { filename := name + "." + ctx.Config.Source.Format path := filepath.Join(ctx.Config.Dist, filename) log.WithField("file", filename).Info("creating source archive") - args := []string{ - "archive", - "-o", path, + + out, err := git.Clean(git.Run(ctx, "ls-files")) + if err != nil { + return fmt.Errorf("could not list source files: %w", err) + } + + prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate) + if err != nil { + return err + } + + af, err := os.Create(path) + if err != nil { + return fmt.Errorf("could not create archive: %w", err) + } + defer af.Close() //nolint:errcheck + + arch, err := archive.New(af, ctx.Config.Source.Format) + if err != nil { + return err } - if ctx.Config.Source.PrefixTemplate != "" { - prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate) - if err != nil { - return err + for _, f := range strings.Split(out, "\n") { + if err := arch.Add(config.File{ + Source: f, + Destination: filepath.Join(prefix, f), + }); err != nil { + return fmt.Errorf("could not add %q to archive: %w", f, err) } - args = append(args, "--prefix", prefix) } - args = append(args, ctx.Git.FullCommit) - out, err := git.Clean(git.Run(ctx, args...)) - log.Debug(out) + files, err := archivefiles.Eval(tmpl.New(ctx), ctx.Config.Source.Files) + if err != nil { + return err + } + for _, f := range files { + if err := arch.Add(f); err != nil { + return fmt.Errorf("could not add %q to archive: %w", f.Source, err) + } + } + + if err := arch.Close(); err != nil { + return fmt.Errorf("could not close archive file: %w", err) + } + if err := af.Close(); err != nil { + return fmt.Errorf("could not close archive file: %w", err) + } + ctx.Artifacts.Add(&artifact.Artifact{ Type: artifact.UploadableSourceArchive, Name: filename, diff --git a/internal/pipe/sourcearchive/source_test.go b/internal/pipe/sourcearchive/source_test.go index 33b7d621d4f..6429612f37e 100644 --- a/internal/pipe/sourcearchive/source_test.go +++ b/internal/pipe/sourcearchive/source_test.go @@ -1,6 +1,7 @@ package sourcearchive import ( + "archive/zip" "os" "path/filepath" "testing" @@ -23,6 +24,8 @@ func TestArchive(t *testing.T) { require.NoError(t, os.WriteFile("README.md", []byte("# my dope fake project"), 0o655)) testlib.GitAdd(t) testlib.GitCommit(t, "feat: first") + require.NoError(t, os.WriteFile("added-later.txt", []byte("this file was added later"), 0o655)) + require.NoError(t, os.WriteFile("ignored.md", []byte("never added"), 0o655)) ctx := context.New(config.Project{ ProjectName: "foo", @@ -31,6 +34,9 @@ func TestArchive(t *testing.T) { Format: format, Enabled: true, PrefixTemplate: "{{ .ProjectName }}-{{ .Version }}/", + Files: []config.File{{ + Source: "*.txt", + }}, }, }) ctx.Git.FullCommit = "HEAD" @@ -49,13 +55,43 @@ func TestArchive(t *testing.T) { artifact.ExtraFormat: format, }, }, *artifacts[0]) - stat, err := os.Stat(filepath.Join(tmp, "dist", "foo-1.0.0."+format)) + path := filepath.Join(tmp, "dist", "foo-1.0.0."+format) + stat, err := os.Stat(path) require.NoError(t, err) require.Greater(t, stat.Size(), int64(100)) + + if format != "zip" { + return + } + + f, err := os.Open(path) + require.NoError(t, err) + z, err := zip.NewReader(f, stat.Size()) + require.NoError(t, err) + + var paths []string + for _, zf := range z.File { + paths = append(paths, zf.Name) + } + require.Equal(t, []string{"foo-1.0.0/README.md", "added-later.txt", "code.txt"}, paths) }) } } +func TestInvalidFormat(t *testing.T) { + ctx := context.New(config.Project{ + Dist: t.TempDir(), + ProjectName: "foo", + Source: config.Source{ + Format: "7z", + Enabled: true, + PrefixTemplate: "{{ .ProjectName }}-{{ .Version }}/", + }, + }) + require.NoError(t, Pipe{}.Default(ctx)) + require.EqualError(t, Pipe{}.Run(ctx), "invalid archive format: 7z") +} + func TestDefault(t *testing.T) { ctx := context.New(config.Project{}) require.NoError(t, Pipe{}.Default(ctx)) diff --git a/pkg/config/config.go b/pkg/config/config.go index 4663575dc45..9891fdce78c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -910,6 +910,7 @@ type Source struct { Format string `yaml:"format,omitempty"` Enabled bool `yaml:"enabled,omitempty"` PrefixTemplate string `yaml:"prefix_template,omitempty"` + Files []File `yaml:"files,omitempty"` } // Project includes all project configuration. diff --git a/www/docs/customization/source.md b/www/docs/customization/source.md index 50b09b52332..b86385c265d 100644 --- a/www/docs/customization/source.md +++ b/www/docs/customization/source.md @@ -23,6 +23,33 @@ source: # String to prepend to each filename in the archive. # Defaults to empty prefix_template: '{{ .ProjectName }}-{{ .Version }}/' + + # Additional files/template/globs you want to add to the archive. + # Defaults are any files matching `LICENSE*`, `README*`, `CHANGELOG*`, + # `license*`, `readme*` and `changelog*`. + files: + - LICENSE.txt + - README_{{.Os}}.md + - CHANGELOG.md + - docs/* + - design/*.png + - templates/**/* + # a more complete example, check the globbing deep dive below + - src: '*.md' + dst: docs + # Strip parent folders when adding files to the archive. + # Default: false + strip_parent: true + # File info. + # Not all fields are supported by all formats available formats. + # Defaults to the file info of the actual file if not provided. + info: + owner: root + group: root + mode: 0644 + # format is `time.RFC3339Nano` + mtime: 2008-01-02T15:04:05Z + ``` !!! tip From 2d937c3e5b6287b06071a2642bffd9fe24455ea6 Mon Sep 17 00:00:00 2001 From: Carlos A Becker Date: Wed, 11 May 2022 23:02:33 -0300 Subject: [PATCH 2/7] fix: docs Signed-off-by: Carlos A Becker --- www/docs/customization/source.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/www/docs/customization/source.md b/www/docs/customization/source.md index b86385c265d..bfec2703804 100644 --- a/www/docs/customization/source.md +++ b/www/docs/customization/source.md @@ -24,9 +24,8 @@ source: # Defaults to empty prefix_template: '{{ .ProjectName }}-{{ .Version }}/' - # Additional files/template/globs you want to add to the archive. - # Defaults are any files matching `LICENSE*`, `README*`, `CHANGELOG*`, - # `license*`, `readme*` and `changelog*`. + # Additional files/template/globs you want to add to the source archive. + # Defaults to empty. files: - LICENSE.txt - README_{{.Os}}.md From 743c7a90e8ecd759a5a1376d41af96b33dcae17e Mon Sep 17 00:00:00 2001 From: Carlos A Becker Date: Wed, 11 May 2022 23:35:44 -0300 Subject: [PATCH 3/7] fix: duplicated files Signed-off-by: Carlos A Becker --- internal/pipe/sourcearchive/source.go | 17 ++++++++++------- internal/pipe/sourcearchive/source_test.go | 8 +++++++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/pipe/sourcearchive/source.go b/internal/pipe/sourcearchive/source.go index 0507c252d7f..9e5787f2985 100644 --- a/internal/pipe/sourcearchive/source.go +++ b/internal/pipe/sourcearchive/source.go @@ -38,7 +38,7 @@ func (Pipe) Run(ctx *context.Context) (err error) { path := filepath.Join(ctx.Config.Dist, filename) log.WithField("file", filename).Info("creating source archive") - out, err := git.Clean(git.Run(ctx, "ls-files")) + out, err := git.Run(ctx, "ls-files") if err != nil { return fmt.Errorf("could not list source files: %w", err) } @@ -58,19 +58,22 @@ func (Pipe) Run(ctx *context.Context) (err error) { if err != nil { return err } + + var ff []config.File for _, f := range strings.Split(out, "\n") { - if err := arch.Add(config.File{ - Source: f, - Destination: filepath.Join(prefix, f), - }); err != nil { - return fmt.Errorf("could not add %q to archive: %w", f, err) + if strings.TrimSpace(f) == "" { + continue } + ff = append(ff, config.File{ + Source: f, + }) } - files, err := archivefiles.Eval(tmpl.New(ctx), ctx.Config.Source.Files) + files, err := archivefiles.Eval(tmpl.New(ctx), append(ff, ctx.Config.Source.Files...)) if err != nil { return err } for _, f := range files { + f.Destination = filepath.Join(prefix, f.Destination) if err := arch.Add(f); err != nil { return fmt.Errorf("could not add %q to archive: %w", f.Source, err) } diff --git a/internal/pipe/sourcearchive/source_test.go b/internal/pipe/sourcearchive/source_test.go index 6429612f37e..6533c2da987 100644 --- a/internal/pipe/sourcearchive/source_test.go +++ b/internal/pipe/sourcearchive/source_test.go @@ -21,6 +21,7 @@ func TestArchive(t *testing.T) { testlib.GitInit(t) require.NoError(t, os.WriteFile("code.txt", []byte("not really code"), 0o655)) + require.NoError(t, os.WriteFile("code.py", []byte("print 1"), 0o655)) require.NoError(t, os.WriteFile("README.md", []byte("# my dope fake project"), 0o655)) testlib.GitAdd(t) testlib.GitCommit(t, "feat: first") @@ -73,7 +74,12 @@ func TestArchive(t *testing.T) { for _, zf := range z.File { paths = append(paths, zf.Name) } - require.Equal(t, []string{"foo-1.0.0/README.md", "added-later.txt", "code.txt"}, paths) + require.Equal(t, []string{ + "foo-1.0.0/README.md", + "foo-1.0.0/added-later.txt", + "foo-1.0.0/code.py", + "foo-1.0.0/code.txt", + }, paths) }) } } From 316c80adec51394e76d560535702a9b2787df390 Mon Sep 17 00:00:00 2001 From: Carlos A Becker Date: Thu, 25 Aug 2022 00:47:54 -0300 Subject: [PATCH 4/7] fix: use git-archive --add-file --- internal/pipe/sourcearchive/source.go | 60 +++++++++------------- internal/pipe/sourcearchive/source_test.go | 9 ++-- pkg/config/config.go | 10 ++-- www/docs/customization/source.md | 17 +----- 4 files changed, 35 insertions(+), 61 deletions(-) diff --git a/internal/pipe/sourcearchive/source.go b/internal/pipe/sourcearchive/source.go index 102713bb1d3..8f3d6c80b06 100644 --- a/internal/pipe/sourcearchive/source.go +++ b/internal/pipe/sourcearchive/source.go @@ -2,17 +2,13 @@ package sourcearchive import ( - "fmt" - "os" "path/filepath" - "strings" "github.com/caarlos0/log" "github.com/goreleaser/goreleaser/internal/archivefiles" "github.com/goreleaser/goreleaser/internal/artifact" "github.com/goreleaser/goreleaser/internal/git" "github.com/goreleaser/goreleaser/internal/tmpl" - "github.com/goreleaser/goreleaser/pkg/archive" "github.com/goreleaser/goreleaser/pkg/config" "github.com/goreleaser/goreleaser/pkg/context" ) @@ -37,54 +33,41 @@ func (Pipe) Run(ctx *context.Context) (err error) { filename := name + "." + ctx.Config.Source.Format path := filepath.Join(ctx.Config.Dist, filename) log.WithField("file", filename).Info("creating source archive") - - out, err := git.Run(ctx, "ls-files") - if err != nil { - return fmt.Errorf("could not list source files: %w", err) + args := []string{ + "archive", + "-o", path, } - prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate) + tpl := tmpl.New(ctx) + prefix, err := tpl.Apply(ctx.Config.Source.PrefixTemplate) if err != nil { return err } - - af, err := os.Create(path) - if err != nil { - return fmt.Errorf("could not create archive: %w", err) - } - defer af.Close() //nolint:errcheck - - arch, err := archive.New(af, ctx.Config.Source.Format) - if err != nil { - return err + if prefix != "" { + args = append(args, "--prefix", prefix) } - var ff []config.File - for _, f := range strings.Split(out, "\n") { - if strings.TrimSpace(f) == "" { - continue - } - ff = append(ff, config.File{ + var files []config.File + for _, f := range ctx.Config.Source.Files { + files = append(files, config.File{ Source: f, }) } - files, err := archivefiles.Eval(tmpl.New(ctx), append(ff, ctx.Config.Source.Files...)) + addFiles, err := archivefiles.Eval(tpl, files) if err != nil { return err } - for _, f := range files { - f.Destination = filepath.Join(prefix, f.Destination) - if err := arch.Add(f); err != nil { - return fmt.Errorf("could not add %q to archive: %w", f.Source, err) + + for _, f := range addFiles { + if isTracked(ctx, f.Source) { + continue } + args = append(args, "--add-file", f.Source) } - if err := arch.Close(); err != nil { - return fmt.Errorf("could not close archive file: %w", err) - } - if err := af.Close(); err != nil { - return fmt.Errorf("could not close archive file: %w", err) - } + args = append(args, ctx.Git.FullCommit) + out, err := git.Clean(git.Run(ctx, args...)) + log.Debug(out) ctx.Artifacts.Add(&artifact.Artifact{ Type: artifact.UploadableSourceArchive, @@ -97,6 +80,11 @@ func (Pipe) Run(ctx *context.Context) (err error) { return err } +func isTracked(ctx *context.Context, path string) bool { + _, err := git.Run(ctx, "ls-files", "--error-unmatch", path) + return err == nil +} + // Default sets the pipe defaults. func (Pipe) Default(ctx *context.Context) error { archive := &ctx.Config.Source diff --git a/internal/pipe/sourcearchive/source_test.go b/internal/pipe/sourcearchive/source_test.go index 6533c2da987..5030aa29f64 100644 --- a/internal/pipe/sourcearchive/source_test.go +++ b/internal/pipe/sourcearchive/source_test.go @@ -35,9 +35,9 @@ func TestArchive(t *testing.T) { Format: format, Enabled: true, PrefixTemplate: "{{ .ProjectName }}-{{ .Version }}/", - Files: []config.File{{ - Source: "*.txt", - }}, + Files: []string{ + "*.txt", + }, }, }) ctx.Git.FullCommit = "HEAD" @@ -75,10 +75,11 @@ func TestArchive(t *testing.T) { paths = append(paths, zf.Name) } require.Equal(t, []string{ + "foo-1.0.0/", "foo-1.0.0/README.md", - "foo-1.0.0/added-later.txt", "foo-1.0.0/code.py", "foo-1.0.0/code.txt", + "foo-1.0.0/added-later.txt", }, paths) }) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 0bab72524f3..6dcde36c472 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -876,11 +876,11 @@ type Publisher struct { // Source configuration. type Source struct { - NameTemplate string `yaml:"name_template,omitempty" json:"name_template,omitempty"` - Format string `yaml:"format,omitempty" json:"format,omitempty"` - Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` - PrefixTemplate string `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"` - Files []File `yaml:"files,omitempty" json:"files,omitempty"` + NameTemplate string `yaml:"name_template,omitempty" json:"name_template,omitempty"` + Format string `yaml:"format,omitempty" json:"format,omitempty"` + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + PrefixTemplate string `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"` + Files []string `yaml:"files,omitempty" json:"files,omitempty"` } // Project includes all project configuration. diff --git a/www/docs/customization/source.md b/www/docs/customization/source.md index bfec2703804..ef1f51815a6 100644 --- a/www/docs/customization/source.md +++ b/www/docs/customization/source.md @@ -25,6 +25,7 @@ source: prefix_template: '{{ .ProjectName }}-{{ .Version }}/' # Additional files/template/globs you want to add to the source archive. + # Will use --add-file of git-archive. # Defaults to empty. files: - LICENSE.txt @@ -33,22 +34,6 @@ source: - docs/* - design/*.png - templates/**/* - # a more complete example, check the globbing deep dive below - - src: '*.md' - dst: docs - # Strip parent folders when adding files to the archive. - # Default: false - strip_parent: true - # File info. - # Not all fields are supported by all formats available formats. - # Defaults to the file info of the actual file if not provided. - info: - owner: root - group: root - mode: 0644 - # format is `time.RFC3339Nano` - mtime: 2008-01-02T15:04:05Z - ``` !!! tip From ba0f1281ec2971808789d026da78b6d0e0c270c8 Mon Sep 17 00:00:00 2001 From: Carlos A Becker Date: Thu, 25 Aug 2022 00:50:38 -0300 Subject: [PATCH 5/7] fix: format Signed-off-by: Carlos A Becker --- internal/pipe/sourcearchive/source.go | 1 + internal/pipe/sourcearchive/source_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/pipe/sourcearchive/source.go b/internal/pipe/sourcearchive/source.go index 8f3d6c80b06..099efa5a101 100644 --- a/internal/pipe/sourcearchive/source.go +++ b/internal/pipe/sourcearchive/source.go @@ -36,6 +36,7 @@ func (Pipe) Run(ctx *context.Context) (err error) { args := []string{ "archive", "-o", path, + "--format", ctx.Config.Source.Format, } tpl := tmpl.New(ctx) diff --git a/internal/pipe/sourcearchive/source_test.go b/internal/pipe/sourcearchive/source_test.go index 5030aa29f64..b28926f7418 100644 --- a/internal/pipe/sourcearchive/source_test.go +++ b/internal/pipe/sourcearchive/source_test.go @@ -96,7 +96,7 @@ func TestInvalidFormat(t *testing.T) { }, }) require.NoError(t, Pipe{}.Default(ctx)) - require.EqualError(t, Pipe{}.Run(ctx), "invalid archive format: 7z") + require.EqualError(t, Pipe{}.Run(ctx), "fatal: Unknown archive format '7z'") } func TestDefault(t *testing.T) { From c3f6866e44810d09993641acf8365baf495ff3b2 Mon Sep 17 00:00:00 2001 From: Carlos A Becker Date: Thu, 25 Aug 2022 00:57:01 -0300 Subject: [PATCH 6/7] refactor: improve code a little bit --- internal/pipe/sourcearchive/source.go | 56 ++++++++++++++++++--------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/internal/pipe/sourcearchive/source.go b/internal/pipe/sourcearchive/source.go index 099efa5a101..8e05bc007d7 100644 --- a/internal/pipe/sourcearchive/source.go +++ b/internal/pipe/sourcearchive/source.go @@ -40,30 +40,20 @@ func (Pipe) Run(ctx *context.Context) (err error) { } tpl := tmpl.New(ctx) - prefix, err := tpl.Apply(ctx.Config.Source.PrefixTemplate) - if err != nil { - return err - } - if prefix != "" { + if ctx.Config.Source.PrefixTemplate != "" { + prefix, err := tpl.Apply(ctx.Config.Source.PrefixTemplate) + if err != nil { + return err + } args = append(args, "--prefix", prefix) } - var files []config.File - for _, f := range ctx.Config.Source.Files { - files = append(files, config.File{ - Source: f, - }) - } - addFiles, err := archivefiles.Eval(tpl, files) + files, err := evalFiles(ctx) if err != nil { return err } - - for _, f := range addFiles { - if isTracked(ctx, f.Source) { - continue - } - args = append(args, "--add-file", f.Source) + for _, f := range files { + args = append(args, "--add-file", f) } args = append(args, ctx.Git.FullCommit) @@ -81,6 +71,36 @@ func (Pipe) Run(ctx *context.Context) (err error) { return err } +// to reuse the archivefiles packages, we do something funky: +// - convert the []string to []config.File +// - eval it in archivefiles +// - convert it back to []string +// +// we also handle files already tracked, as if we add them again, +// they'll get duplicated in the archive. +func evalFiles(ctx *context.Context) ([]string, error) { + var files []config.File + for _, f := range ctx.Config.Source.Files { + files = append(files, config.File{ + Source: f, + }) + } + addFiles, err := archivefiles.Eval(tmpl.New(ctx), files) + if err != nil { + return nil, err + } + + var result []string + for _, f := range addFiles { + if isTracked(ctx, f.Source) { + continue + } + result = append(result, f.Source) + } + return result, nil +} + +// check if file is tracked, and, if it is we should not add it to the archive again. func isTracked(ctx *context.Context, path string) bool { _, err := git.Run(ctx, "ls-files", "--error-unmatch", path) return err == nil From d3039a4a7582874560e6d358de91c28d569e7f34 Mon Sep 17 00:00:00 2001 From: Carlos A Becker Date: Thu, 25 Aug 2022 00:57:45 -0300 Subject: [PATCH 7/7] fix: merge --- internal/pipe/sourcearchive/source.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/pipe/sourcearchive/source.go b/internal/pipe/sourcearchive/source.go index 8e05bc007d7..339d718e447 100644 --- a/internal/pipe/sourcearchive/source.go +++ b/internal/pipe/sourcearchive/source.go @@ -39,9 +39,8 @@ func (Pipe) Run(ctx *context.Context) (err error) { "--format", ctx.Config.Source.Format, } - tpl := tmpl.New(ctx) if ctx.Config.Source.PrefixTemplate != "" { - prefix, err := tpl.Apply(ctx.Config.Source.PrefixTemplate) + prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate) if err != nil { return err }