Use etag in oci downloader to skip reloading bundle if tarball SHA di…

…d not change

Signed-off-by: carabasdaniel <dani@aserto.com>

download/oci_dowload.go

@@ -43,6 +43,7 @@ type OCIDownloader struct {
 	stopped        bool
 	persist        bool
 	store          *content.OCI
+	etag           string
 }
 
 // New returns a new Downloader that can be started.
@@ -98,8 +99,9 @@ func (d *OCIDownloader) WithBundlePersistence(persist bool) *OCIDownloader {
 func (d *OCIDownloader) ClearCache() {
 }
 
-// TODO: this will need implementation for the OCI downloader.
+// SetCache sets the etag value to the SHA of the loaded bundle
 func (d *OCIDownloader) SetCache(etag string) {
+	d.etag = etag
 }
 
 // Trigger can be used to control when the downloader attempts to download
@@ -213,6 +215,7 @@ func (d *OCIDownloader) oneShot(ctx context.Context) error {
 		}
 		return err
 	}
+	d.SetCache(resp.etag) // set the current etag sha to the cache
 
 	if d.f != nil {
 		d.f(ctx, Update{ETag: resp.etag, Bundle: resp.b, Error: nil, Metrics: m, Raw: resp.raw})
@@ -248,14 +251,26 @@ func (d *OCIDownloader) download(ctx context.Context, m metrics.Metrics) (*downl
 	if tarballDescriptor.MediaType == "" {
 		return nil, fmt.Errorf("no tarball descriptor found in the layers")
 	}
-
-	bundleFilePath := filepath.Join(d.localStorePath, "blobs", "sha256", string(tarballDescriptor.Digest.Hex()))
+	etag := string(tarballDescriptor.Digest.Hex())
+	bundleFilePath := filepath.Join(d.localStorePath, "blobs", "sha256", etag)
+	// if the downloader etag sha is the same with digest of the tarball it was already loaded
+	if d.etag == etag {
+		return &downloaderResponse{
+			b:        nil,
+			raw:      nil,
+			etag:     etag,
+			longPoll: false,
+		}, nil
+	}
 	fileReader, err := os.Open(bundleFilePath)
 	if err != nil {
 		return nil, err
 	}
 	loader := bundle.NewTarballLoaderWithBaseURL(fileReader, d.localStorePath)
-	reader := bundle.NewCustomReader(loader).WithBaseDir(d.localStorePath)
+	reader := bundle.NewCustomReader(loader).WithBaseDir(d.localStorePath).
+		WithMetrics(m).
+		WithBundleVerificationConfig(d.bvc).
+		WithBundleEtag(etag)
 	bundleInfo, err := reader.Read()
 	if err != nil {
 		return &downloaderResponse{}, fmt.Errorf("unexpected error %w", err)
@@ -266,7 +281,7 @@ func (d *OCIDownloader) download(ctx context.Context, m metrics.Metrics) (*downl
 	return &downloaderResponse{
 		b:        &bundleInfo,
 		raw:      fileReader,
-		etag:     "",
+		etag:     etag,
 		longPoll: false,
 	}, nil
 }

download/oci_download_test.go

@@ -84,3 +84,39 @@ func TestOCIFailureAuthn(t *testing.T) {
 		t.Fatal("expected 401 Unauthorized message")
 	}
 }
+
+func TestOCIEtag(t *testing.T) {
+	ctx := context.Background()
+	fixture := newTestFixture(t)
+	fixture.server.expAuth = "" // test on public registry
+	fixture.server.expEtag = "sha256:c5834dbce332cabe6ae68a364de171a50bf5b08024c27d7c08cc72878b4df7ff"
+	config := Config{}
+	if err := config.ValidateAndInjectDefaults(); err != nil {
+		t.Fatal(err)
+	}
+	firstResponse := Update{ETag: ""}
+	d := NewOCI(config, fixture.client, "ghcr.io/org/repo:latest", "/tmp/oci").WithCallback(func(_ context.Context, u Update) {
+		if firstResponse.ETag == "" {
+			firstResponse = u
+			return
+		}
+
+		if u.ETag != firstResponse.ETag || u.Bundle != nil {
+			t.Fatal("expected nil bundle and same etag but got:", u)
+		}
+	})
+
+	// fill firstResponse
+	err := d.oneShot(ctx)
+	if err != nil {
+		t.Fatal("unexpected error")
+	}
+	// Give time for some download events to occur
+	time.Sleep(1 * time.Second)
+
+	// second call to verify if nil bundle is returned and same etag
+	err = d.oneShot(ctx)
+	if err != nil {
+		t.Fatal("unexpected error")
+	}
+}

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/miekg/dns v1.1.43 // indirect
 	github.com/olekukonko/tablewriter v0.0.5
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799
 	github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d
 	github.com/pkg/errors v0.9.1