set mirror-by-digest-only for each mirror instead of registry

Close: #1407
Add the `mirror-by-digest-only` field to each mirror table, so it will
be configured for each mirror instead of for primary registry.
For the same primary registry, it can support both digest only pull
and tag allowed pull.

The `mirror-by-digest-only` for primary is still allowed configuring,
but it takes no effect.

Signed-off-by: Qi Wang <qiwan@redhat.com>

docs/containers-registries.conf.5.md

@@ -108,15 +108,15 @@ Each TOML table in the `mirror` array can contain the following fields, with the
 as if specified in the `[[registry]]` TOML table directly:
 - `location`
 - `insecure`
+- `mirror-by-digest-only`
 
-`mirror-by-digest-only`
-: `true` or `false`.
-If `true`, mirrors will only be used during pulling if the image reference includes a digest.
+`mirror-by-digest-only`: `true` or `false`.
+The mirror has `mirror-by-digest-only` set to `true` will only be used during pulling if the image reference includes a digest.
 Referencing an image by digest ensures that the same is always used
 (whereas referencing an image by a tag may cause different registries to return
 different images if the tag mapping is out of sync).
 
-Note that if this is `true`, images referenced by a tag will only use the primary
+Note that if all mirrors in the `mirror` array have `mirror-by-digest-only = true`, images referenced by a tag will only use the primary
 registry, failing if that registry is not accessible.
 
 *Note*: Redirection and mirrors are currently processed only when reading images, not when pushing

pkg/sysregistriesv2/system_registries_v2.go

@@ -55,6 +55,16 @@ type Endpoint struct {
 	Insecure bool `toml:"insecure,omitempty"`
 }
 
+// Mirror describes a mirror setting of a registry
+type Mirror struct {
+	// The registry's mirror location
+	Endpoint
+	// If true, the mirror will only be used for digest pulls. Pulling images by
+	// tag can potentially yield different images, depending on which endpoint
+	// we pull from.  Forcing digest-pulls for mirrors avoids that issue.
+	MirrorByDigestOnly bool `toml:"mirror-by-digest-only,omitempty"`
+}
+
 // userRegistriesFile is the path to the per user registry configuration file.
 var userRegistriesFile = filepath.FromSlash(".config/containers/registries.conf")
 
@@ -110,7 +120,7 @@ type Registry struct {
 	// A registry is an Endpoint too
 	Endpoint
 	// The registry's mirrors.
-	Mirrors []Endpoint `toml:"mirror,omitempty"`
+	Mirrors []Mirror `toml:"mirror,omitempty"`
 	// If true, pulling from the registry will be blocked.
 	Blocked bool `toml:"blocked,omitempty"`
 	// If true, mirrors will only be used for digest pulls. Pulling images by
@@ -130,17 +140,18 @@ type PullSource struct {
 // reference.
 func (r *Registry) PullSourcesFromReference(ref reference.Named) ([]PullSource, error) {
 	var endpoints []Endpoint
-
-	if r.MirrorByDigestOnly {
-		// Only use mirrors when the reference is a digest one.
-		if _, isDigested := ref.(reference.Canonical); isDigested {
-			endpoints = append(r.Mirrors, r.Endpoint)
+	_, isDigested := ref.(reference.Canonical)
+	for _, mirror := range r.Mirrors {
+		if mirror.MirrorByDigestOnly {
+			if isDigested {
+				// Only use the mirror when the reference is a digest one.
+				endpoints = append(endpoints, mirror.Endpoint)
+			}
 		} else {
-			endpoints = []Endpoint{r.Endpoint}
+			endpoints = append(endpoints, mirror.Endpoint)
 		}
-	} else {
-		endpoints = append(r.Mirrors, r.Endpoint)
 	}
+	endpoints = append(endpoints, r.Endpoint)
 
 	sources := []PullSource{}
 	for _, ep := range endpoints {
@@ -306,7 +317,7 @@ func (config *V1RegistriesConf) ConvertToV2() (*V2RegistriesConf, error) {
 		if !exists {
 			reg = &Registry{
 				Endpoint: Endpoint{Location: location},
-				Mirrors:  []Endpoint{},
+				Mirrors:  []Mirror{},
 				Prefix:   location,
 			}
 			regMap[location] = reg

pkg/sysregistriesv2/system_registries_v2_test.go

@@ -637,9 +637,9 @@ func TestPullSourcesFromReference(t *testing.T) {
 	}
 	registries, err := GetRegistries(sys)
 	assert.Nil(t, err)
-	assert.Equal(t, 2, len(registries))
+	assert.Equal(t, 4, len(registries))
 
-	// Registry A allowing any kind of pull from mirrors
+	// Registry A has mirrors allow any kind of pull
 	registryA, err := FindRegistry(sys, "registry-a.com/foo/image:latest")
 	assert.Nil(t, err)
 	assert.NotNil(t, registryA)
@@ -657,7 +657,7 @@ func TestPullSourcesFromReference(t *testing.T) {
 	assert.Equal(t, 3, len(pullSources))
 	assert.Equal(t, "registry-a.com/bar/image:aaa", pullSources[2].Reference.String())
 
-	// Registry B allowing digests pull only from mirrors
+	// Registry B has mirrors allow digests pull only
 	registryB, err := FindRegistry(sys, "registry-b.com/foo/image:latest")
 	assert.Nil(t, err)
 	assert.NotNil(t, registryB)
@@ -673,6 +673,40 @@ func TestPullSourcesFromReference(t *testing.T) {
 	pullSources, err = registryB.PullSourcesFromReference(referenceBTag)
 	assert.Nil(t, err)
 	assert.Equal(t, 1, len(pullSources))
+
+	// Registry C has a mirror allows digest pull only and a mirror allows any kind of pull
+	registryC, err := FindRegistry(sys, "registry-c.com/foo/image:latest")
+	assert.Nil(t, err)
+	assert.NotNil(t, registryB)
+	// Digest
+	referenceCDigest := toNamedRef(t, "registry-c.com/foo/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	pullSources, err = registryC.PullSourcesFromReference(referenceCDigest)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(pullSources))
+	assert.Equal(t, "registry-c.com/bar", pullSources[2].Endpoint.Location)
+	assert.Equal(t, "registry-c.com/bar/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", pullSources[2].Reference.String())
+	// Tag
+	referenceCTag := toNamedRef(t, "registry-c.com/foo/image:aaa")
+	pullSources, err = registryC.PullSourcesFromReference(referenceCTag)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(pullSources))
+	// Registry D set mirror-by-digest-only, it can be configured for registry, but takes no effect.
+	// Regsitry D has mirrors allow any kind of pull
+	registryD, err := FindRegistry(sys, "registry-d.com/foo/image:latest")
+	assert.Nil(t, err)
+	assert.NotNil(t, registryA)
+	// Digest
+	referenceDDigest := toNamedRef(t, "registry-d.com/foo/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	pullSources, err = registryD.PullSourcesFromReference(referenceDDigest)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(pullSources))
+	assert.Equal(t, "mirror-1.registry-d.com/image@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", pullSources[0].Reference.String())
+	// Tag
+	referenceDTag := toNamedRef(t, "registry-d.com/foo/image:aaa")
+	pullSources, err = registryD.PullSourcesFromReference(referenceDTag)
+	assert.Nil(t, err)
+	assert.Equal(t, 3, len(pullSources))
+	assert.Equal(t, "registry-d.com/bar/image:aaa", pullSources[2].Reference.String())
 }
 
 func TestTryUpdatingCache(t *testing.T) {

pkg/sysregistriesv2/testdata/pull-sources-from-reference.conf

@@ -12,10 +12,33 @@ insecure = true
 [[registry]]
 prefix = "registry-b.com/foo"
 location = "registry-b.com/bar"
-mirror-by-digest-only = true
 
 [[registry.mirror]]
+mirror-by-digest-only = true
 location = "mirror-1.registry-b.com"
 
 [[registry.mirror]]
+mirror-by-digest-only = true
 location = "mirror-2.registry-b.com"
+
+[[registry]]
+prefix = "registry-c.com/foo"
+location = "registry-c.com/bar"
+
+[[registry.mirror]]
+location = "mirror-1.registry-c.com"
+
+[[registry.mirror]]
+mirror-by-digest-only = true
+location = "mirror-2.registry-c.com"
+
+[[registry]]
+prefix = "registry-d.com/foo"
+location = "registry-d.com/bar"
+mirror-by-digest-only = true
+
+[[registry.mirror]]
+location = "mirror-1.registry-d.com"
+
+[[registry.mirror]]
+location = "mirror-2.registry-d.com"