Skip to content

Commit

Permalink
Merge pull request #1490 from imnitishng/cache-flag/build-volume-cache
Browse files Browse the repository at this point in the history 
Add 'volume' cache format support for --cache flag
  • Loading branch information
@jromero
jromero committed Jul 27, 2022
2 parents 30de289 + 1eb52c0 commit 7ff6918
Show file tree
Hide file tree
Showing 9 changed files with 298 additions and 153 deletions.
8 changes: 4 additions & 4 deletions acceptance/acceptance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -780,8 +780,8 @@ func testAcceptance(
ref, err := name.ParseReference(repoName, name.WeakValidation)
assert.Nil(err)
cacheImage := cache.NewImageCache(ref, dockerCli)
buildCacheVolume := cache.NewVolumeCache(ref, "build", dockerCli)
launchCacheVolume := cache.NewVolumeCache(ref, "launch", dockerCli)
buildCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli)
launchCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli)
cacheImage.Clear(context.TODO())
buildCacheVolume.Clear(context.TODO())
launchCacheVolume.Clear(context.TODO())
Expand Down Expand Up @@ -2423,8 +2423,8 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
imageManager.CleanupImages(origID, repoName, runBefore)
ref, err := name.ParseReference(repoName, name.WeakValidation)
assert.Nil(err)
buildCacheVolume := cache.NewVolumeCache(ref, "build", dockerCli)
launchCacheVolume := cache.NewVolumeCache(ref, "launch", dockerCli)
buildCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli)
launchCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli)
assert.Succeeds(buildCacheVolume.Clear(context.TODO()))
assert.Succeeds(launchCacheVolume.Clear(context.TODO()))
})
Expand Down
8 changes: 4 additions & 4 deletions internal/build/lifecycle_execution.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,18 +124,18 @@ func (l *LifecycleExecution) PrevImageName() string {
func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseFactoryCreator) error {
phaseFactory := phaseFactoryCreator(l)
var buildCache Cache
if l.opts.CacheImage != "" || (l.opts.Cache.CacheType == cache.Build && l.opts.Cache.Format == cache.CacheImage) {
if l.opts.CacheImage != "" || (l.opts.Cache.Build.Format == cache.CacheImage) {
cacheImageName := l.opts.CacheImage
if cacheImageName == "" {
cacheImageName = l.opts.Cache.Source
cacheImageName = l.opts.Cache.Build.Source
}
cacheImage, err := name.ParseReference(cacheImageName, name.WeakValidation)
if err != nil {
return fmt.Errorf("invalid cache image name: %s", err)
}
buildCache = cache.NewImageCache(cacheImage, l.docker)
} else {
buildCache = cache.NewVolumeCache(l.opts.Image, "build", l.docker)
buildCache = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Build, "build", l.docker)
}

l.logger.Debugf("Using build cache volume %s", style.Symbol(buildCache.Name()))
Expand All @@ -146,7 +146,7 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
l.logger.Debugf("Build cache %s cleared", style.Symbol(buildCache.Name()))
}

launchCache := cache.NewVolumeCache(l.opts.Image, "launch", l.docker)
launchCache := cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Launch, "launch", l.docker)

if !l.opts.UseCreator {
if l.platformAPI.LessThan("0.7") {
Expand Down
7 changes: 4 additions & 3 deletions internal/build/lifecycle_execution_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,9 +365,10 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
TrustBuilder: false,
UseCreator: false,
Cache: cache.CacheOpts{
CacheType: cache.Build,
Format: cache.CacheImage,
Source: "%%%",
Build: cache.CacheInfo{
Format: cache.CacheImage,
Source: "%%%",
},
},
Termui: fakeTermui,
}
Expand Down
93 changes: 42 additions & 51 deletions internal/cache/cache_opts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,21 @@ import (
"github.com/pkg/errors"
)

type Cache int
type Format int
type CacheInfo struct {
Format Format
Source string
}
type CacheOpts struct {
CacheType Cache
Format Format
Source string
Build CacheInfo
Launch CacheInfo
}

const (
Build Cache = iota
Launch
)
const (
CacheVolume Format = iota
CacheImage
)

func (c Cache) String() string {
switch c {
case Build:
return "build"
case Launch:
return "launch"
}
return ""
}

func (f Format) String() string {
switch f {
case CacheImage:
Expand All @@ -53,36 +41,46 @@ func (c *CacheOpts) Set(value string) error {
return err
}

cache := &c.Build
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)

if len(parts) != 2 {
return errors.Errorf("invalid field '%s' must be a key=value pair", field)
}
key := strings.ToLower(parts[0])
value := strings.ToLower(parts[1])
if key == "type" {
switch value {
case "build":
cache = &c.Build
case "launch":
cache = &c.Launch
default:
return errors.Errorf("invalid cache type '%s'", value)
}
break
}
}

if len(parts) == 2 {
key := strings.ToLower(parts[0])
value := strings.ToLower(parts[1])
switch key {
case "type":
switch value {
case "build":
c.CacheType = Build
case "launch":
c.CacheType = Launch
default:
return errors.Errorf("invalid cache type '%s'", value)
}
case "format":
switch value {
case "image":
c.Format = CacheImage
default:
return errors.Errorf("invalid cache format '%s'", value)
}
case "name":
c.Source = value
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
if len(parts) != 2 {
return errors.Errorf("invalid field '%s' must be a key=value pair", field)
}
key := strings.ToLower(parts[0])
value := strings.ToLower(parts[1])
switch key {
case "format":
switch value {
case "image":
cache.Format = CacheImage
case "volume":
cache.Format = CacheVolume
default:
return errors.Errorf("invalid cache format '%s'", value)
}
case "name":
cache.Source = value
}
}

Expand All @@ -95,15 +93,8 @@ func (c *CacheOpts) Set(value string) error {

func (c *CacheOpts) String() string {
var cacheFlag string
if c.CacheType.String() != "" {
cacheFlag += fmt.Sprintf("type=%s;", c.CacheType)
}
if c.Format.String() != "" {
cacheFlag += fmt.Sprintf("format=%s;", c.Format)
}
if c.Source != "" {
cacheFlag += fmt.Sprintf("name=%s", c.Source)
}
cacheFlag = fmt.Sprintf("type=build;format=%s;name=%s;", c.Build.Format.String(), c.Build.Source)
cacheFlag += fmt.Sprintf("type=launch;format=%s;name=%s;", c.Launch.Format.String(), c.Launch.Source)
return cacheFlag
}

Expand All @@ -112,7 +103,7 @@ func (c *CacheOpts) Type() string {
}

func populateMissing(c *CacheOpts) error {
if c.Source == "" {
if (c.Build.Source == "" && c.Build.Format == CacheImage) || (c.Launch.Source == "" && c.Launch.Format == CacheImage) {
return errors.Errorf("cache 'name' is required")
}
return nil
Expand Down
100 changes: 83 additions & 17 deletions internal/cache/cache_opts_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,18 @@ func TestMetadata(t *testing.T) {
}

func testCacheOpts(t *testing.T, when spec.G, it spec.S) {
when("cache options are passed", func() {
it("image cache format with complete options", func() {
when("image cache format options are passed", func() {
it("with complete options", func() {
testcases := []CacheOptTestCase{
{
name: "Build cache as Image",
input: "type=build;format=image;name=io.test.io/myorg/my-cache:build",
output: "type=build;format=image;name=io.test.io/myorg/my-cache:build",
output: "type=build;format=image;name=io.test.io/myorg/my-cache:build;type=launch;format=volume;name=;",
},
{
name: "Launch cache as Image",
input: "type=launch;format=image;name=io.test.io/myorg/my-cache:build",
output: "type=launch;format=image;name=io.test.io/myorg/my-cache:build",
output: "type=build;format=volume;name=;type=launch;format=image;name=io.test.io/myorg/my-cache:build;",
},
}

Expand All @@ -48,22 +48,17 @@ func testCacheOpts(t *testing.T, when spec.G, it spec.S) {
}
})

it("image cache format with missing options", func() {
it("with missing options", func() {
successTestCases := []CacheOptTestCase{
{
name: "Build cache as Image missing: type",
input: "format=image;name=io.test.io/myorg/my-cache:build",
output: "type=build;format=image;name=io.test.io/myorg/my-cache:build",
output: "type=build;format=image;name=io.test.io/myorg/my-cache:build;type=launch;format=volume;name=;",
},
{
name: "Build cache as Image missing: format",
input: "type=build;name=io.test.io/myorg/my-cache:build",
output: "type=build;format=volume;name=io.test.io/myorg/my-cache:build",
},
{
name: "Build cache as Image missing: type, format",
input: "name=io.test.io/myorg/my-cache:build",
output: "type=build;format=volume;name=io.test.io/myorg/my-cache:build",
output: "type=build;format=volume;name=io.test.io/myorg/my-cache:build;type=launch;format=volume;name=;",
},
{
name: "Build cache as Image missing: name",
Expand All @@ -72,10 +67,14 @@ func testCacheOpts(t *testing.T, when spec.G, it spec.S) {
shouldFail: true,
},
{
name: "Build cache as Image missing: format, name",
input: "type=build",
output: "cache 'name' is required",
shouldFail: true,
name: "Build cache as Image missing: type, format",
input: "name=io.test.io/myorg/my-cache:build",
output: "type=build;format=volume;name=io.test.io/myorg/my-cache:build;type=launch;format=volume;name=;",
},
{
name: "Build cache as Image missing: format, name",
input: "type=build",
output: "type=build;format=volume;name=;type=launch;format=volume;name=;",
},
{
name: "Build cache as Image missing: type, name",
Expand All @@ -94,6 +93,9 @@ func testCacheOpts(t *testing.T, when spec.G, it spec.S) {
for _, testcase := range successTestCases {
var cacheFlags CacheOpts
t.Logf("Testing cache type: %s", testcase.name)
if testcase.name == "Everything missing" {
print("i am here")
}
err := cacheFlags.Set(testcase.input)

if testcase.shouldFail {
Expand All @@ -106,7 +108,7 @@ func testCacheOpts(t *testing.T, when spec.G, it spec.S) {
}
})

it("image cache format with invalid options", func() {
it("with invalid options", func() {
testcases := []CacheOptTestCase{
{
name: "Invalid cache type",
Expand Down Expand Up @@ -142,4 +144,68 @@ func testCacheOpts(t *testing.T, when spec.G, it spec.S) {
}
})
})

when("volume cache format options are passed", func() {
it("with complete options", func() {
testcases := []CacheOptTestCase{
{
name: "Build cache as Volume",
input: "type=build;format=volume;name=test-build-volume-cache",
output: "type=build;format=volume;name=test-build-volume-cache;type=launch;format=volume;name=;",
},
{
name: "Launch cache as Volume",
input: "type=launch;format=volume;name=test-launch-volume-cache",
output: "type=build;format=volume;name=;type=launch;format=volume;name=test-launch-volume-cache;",
},
}

for _, testcase := range testcases {
var cacheFlags CacheOpts
t.Logf("Testing cache type: %s", testcase.name)
err := cacheFlags.Set(testcase.input)
h.AssertNil(t, err)
h.AssertEq(t, testcase.output, cacheFlags.String())
}
})

it("with missing options", func() {
successTestCases := []CacheOptTestCase{
{
name: "Launch cache as Volume missing: format",
input: "type=launch;name=test-launch-volume",
output: "type=build;format=volume;name=;type=launch;format=volume;name=test-launch-volume;",
},
{
name: "Launch cache as Volume missing: name",
input: "type=launch;format=volume",
output: "type=build;format=volume;name=;type=launch;format=volume;name=;",
},
{
name: "Launch cache as Volume missing: format, name",
input: "type=launch",
output: "type=build;format=volume;name=;type=launch;format=volume;name=;",
},
{
name: "Launch cache as Volume missing: type, name",
input: "format=volume",
output: "type=build;format=volume;name=;type=launch;format=volume;name=;",
},
}

for _, testcase := range successTestCases {
var cacheFlags CacheOpts
t.Logf("Testing cache type: %s", testcase.name)
err := cacheFlags.Set(testcase.input)

if testcase.shouldFail {
h.AssertError(t, err, testcase.output)
} else {
h.AssertNil(t, err)
output := cacheFlags.String()
h.AssertEq(t, testcase.output, output)
}
}
})
})
}
14 changes: 10 additions & 4 deletions internal/cache/volume_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,18 @@ type VolumeCache struct {
volume string
}

func NewVolumeCache(imageRef name.Reference, suffix string, dockerClient client.CommonAPIClient) *VolumeCache {
sum := sha256.Sum256([]byte(imageRef.Name()))
func NewVolumeCache(imageRef name.Reference, cacheType CacheInfo, suffix string, dockerClient client.CommonAPIClient) *VolumeCache {
var volumeName string
if cacheType.Source == "" {
sum := sha256.Sum256([]byte(imageRef.Name()))
vol := paths.FilterReservedNames(fmt.Sprintf("%s-%x", sanitizedRef(imageRef), sum[:6]))
volumeName = fmt.Sprintf("pack-cache-%s.%s", vol, suffix)
} else {
volumeName = paths.FilterReservedNames(cacheType.Source)
}

vol := paths.FilterReservedNames(fmt.Sprintf("%s-%x", sanitizedRef(imageRef), sum[:6]))
return &VolumeCache{
volume: fmt.Sprintf("pack-cache-%s.%s", vol, suffix),
volume: volumeName,
docker: dockerClient,
}
}
Expand Down

0 comments on commit 7ff6918

Please sign in to comment.