Enable configuration of index dependency validation

Enable configuration options that can selectively disable validation
that dependencies exist within the registry before the image index
is uploaded.

This enables sparse indexes, where a registry holds a manifest index that
could be signed (so the digest must not change) but does not hold every
referenced image in the index. The use case for this is when a registry
mirror does not need to mirror all platforms, but does need to maintain
the digests of all manifests either because they are signed or because
they are pulled by digest.

The registry administrator can also select specific image architectures
that must exist in the registry, enabling a registry operator to select
only the platforms they care about and ensure all image indexes uploaded
to the registry are valid for those platforms.

Signed-off-by: James Hewitt <james.hewitt@uk.ibm.com>

configuration/configuration.go

@@ -175,25 +175,7 @@ type Configuration struct {
 	Proxy Proxy `yaml:"proxy,omitempty"`
 
 	// Validation configures validation options for the registry.
-	Validation struct {
-		// Enabled enables the other options in this section. This field is
-		// deprecated in favor of Disabled.
-		Enabled bool `yaml:"enabled,omitempty"`
-		// Disabled disables the other options in this section.
-		Disabled bool `yaml:"disabled,omitempty"`
-		// Manifests configures manifest validation.
-		Manifests struct {
-			// URLs configures validation for URLs in pushed manifests.
-			URLs struct {
-				// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
-				// that URLs in pushed manifests must match.
-				Allow []string `yaml:"allow,omitempty"`
-				// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
-				// that URLs in pushed manifests must not match.
-				Deny []string `yaml:"deny,omitempty"`
-			} `yaml:"urls,omitempty"`
-		} `yaml:"manifests,omitempty"`
-	} `yaml:"validation,omitempty"`
+	Validation Validation `yaml:"validation,omitempty"`
 
 	// Policy configures registry policy options.
 	Policy struct {
@@ -360,6 +342,13 @@ type Health struct {
 	} `yaml:"storagedriver,omitempty"`
 }
 
+type Platform struct {
+	// Architecture is the architecture for this platform
+	Architecture string `yaml:"architecture,omitempty"`
+	// OS is the operating system for this platform
+	OS string `yaml:"os,omitempty"`
+}
+
 // v0_1Configuration is a Version 0.1 Configuration struct
 // This is currently aliased to Configuration, as it is the current version
 type v0_1Configuration Configuration
@@ -630,6 +619,62 @@ type Proxy struct {
 	TTL *time.Duration `yaml:"ttl,omitempty"`
 }
 
+type Validation struct {
+	// Enabled enables the other options in this section. This field is
+	// deprecated in favor of Disabled.
+	Enabled bool `yaml:"enabled,omitempty"`
+	// Disabled disables the other options in this section.
+	Disabled bool `yaml:"disabled,omitempty"`
+	// Manifests configures manifest validation.
+	Manifests ValidationManifests `yaml:"manifests,omitempty"`
+	// ImageIndexes configures validation of image indexes
+	ImageIndexes ValidationImageIndexes `yaml:"imageindexes,omitempty"`
+}
+
+type ValidationManifests struct {
+	// URLs configures validation for URLs in pushed manifests.
+	URLs struct {
+		// Allow specifies regular expressions (https://godoc.org/regexp/syntax)
+		// that URLs in pushed manifests must match.
+		Allow []string `yaml:"allow,omitempty"`
+		// Deny specifies regular expressions (https://godoc.org/regexp/syntax)
+		// that URLs in pushed manifests must not match.
+		Deny []string `yaml:"deny,omitempty"`
+	} `yaml:"urls,omitempty"`
+}
+
+type ValidationImageIndexes struct {
+	// PlatformsExist configures the validation applies to the platform images included in an image index
+	PlatformsExist PlatformsExist `yaml:"platformsexist"`
+	// SelectedPlatforms filters the set of platforms to validate for image existence.
+	SelectedPlatforms []Platform `yaml:"selectedplatforms,omitempty"`
+}
+
+// PlatformsExist configures the validation applies to the platform images included in an image index
+// This can be all, none, or selected
+type PlatformsExist string
+
+// UnmarshalYAML implements the yaml.Umarshaler interface
+// Unmarshals a string into a PlatformsExist option, lowercasing the string and validating that it represents a
+// valid option
+func (platformsexist *PlatformsExist) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var platformsExistString string
+	err := unmarshal(&platformsExistString)
+	if err != nil {
+		return err
+	}
+
+	platformsExistString = strings.ToLower(platformsExistString)
+	switch platformsExistString {
+	case "all", "none", "selected":
+	default:
+		return fmt.Errorf("invalid platformsexist option %s Must be one of [all, none, selected]", platformsExistString)
+	}
+
+	*platformsexist = PlatformsExist(platformsExistString)
+	return nil
+}
+
 // Parse parses an input configuration yaml document into a Configuration struct
 // This should generally be capable of handling old configuration format versions
 //

configuration/configuration_test.go

@@ -144,6 +144,11 @@ var configStruct = Configuration{
 		ReadTimeout:  time.Millisecond * 10,
 		WriteTimeout: time.Millisecond * 10,
 	},
+	Validation: Validation{
+		ImageIndexes: ValidationImageIndexes{
+			PlatformsExist: "none",
+		},
+	},
 }
 
 // configYamlV0_1 is a Version 0.1 yaml document representing configStruct
@@ -197,6 +202,9 @@ redis:
   dialtimeout: 10ms
   readtimeout: 10ms
   writetimeout: 10ms
+validation:
+  imageindexes:
+    platformsexist: none
 `
 
 // inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
@@ -226,6 +234,9 @@ notifications:
 http:
   headers:
     X-Content-Type-Options: [nosniff]
+validation:
+  imageindexes:
+    platformsexist: none
 `
 
 type ConfigSuite struct {
@@ -284,6 +295,7 @@ func (suite *ConfigSuite) TestParseIncomplete(c *check.C) {
 	suite.expectedConfig.Notifications = Notifications{}
 	suite.expectedConfig.HTTP.Headers = nil
 	suite.expectedConfig.Redis = Redis{}
+	suite.expectedConfig.Validation.ImageIndexes.PlatformsExist = ""
 
 	// Note: this also tests that REGISTRY_STORAGE and
 	// REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY can be used together
@@ -551,5 +563,12 @@ func copyConfig(config Configuration) *Configuration {
 
 	configCopy.Redis = config.Redis
 
+	configCopy.Validation = Validation{
+		Enabled:      config.Validation.Enabled,
+		Disabled:     config.Validation.Disabled,
+		Manifests:    config.Validation.Manifests,
+		ImageIndexes: config.Validation.ImageIndexes,
+	}
+
 	return configCopy
 }

docs/configuration.md

@@ -283,6 +283,11 @@ validation:
         - ^https?://([^/]+\.)*example\.com/
       deny:
         - ^https?://www\.example\.com/
+    imageindex:
+      platformsexist: Selected
+      platforms:
+      - architecture: amd64
+        os: linux
 ```
 
 In some instances a configuration option is **optional** but it contains child
@@ -1110,16 +1115,16 @@ username (such as `batman`) and the password for that username.
 
 ## `validation`
 
-```none
+```yaml
 validation:
-  manifests:
-    urls:
-      allow:
-        - ^https?://([^/]+\.)*example\.com/
-      deny:
-        - ^https?://www\.example\.com/
+  disabled: false
 ```
 
+Use these settings to configure what validation the registry performs on content.
+
+Validation is performed when content is uploaded to the registry. Changing these
+settings will not validate content that has already been accepting into the registry.
+
 ### `disabled`
 
 The `disabled` flag disables the other options in the `validation`
@@ -1132,6 +1137,16 @@ Use the `manifests` subsection to configure validation of manifests. If
 
 #### `urls`
 
+```yaml
+validation:
+  manifests:
+    urls:
+      allow:
+        - ^https?://([^/]+\.)*example\.com/
+      deny:
+        - ^https?://www\.example\.com/
+```
+
 The `allow` and `deny` options are each a list of
 [regular expressions](https://pkg.go.dev/regexp/syntax) that restrict the URLs in
 pushed manifests.
@@ -1145,6 +1160,53 @@ one of the `allow` regular expressions **and** one of the following holds:
 2.  `deny` is set but no URLs within the manifest match any of the `deny` regular
     expressions.
 
+### `imageindexes`
+
+By default the registry will validate that all platform images exist when an image
+index is uploaded to the registry. Disabling this validatation is experimental
+because other tooling that uses the registry may expect the image index to be complete.
+
+validation:
+  imageindexes:
+    platformsexist: [All|None|Selected]
+    selectedplatforms:
+    - os: linux
+      architecture: amd64
+
+Use these settings to configure what validation the registry performs on image
+index manifests uploaded to the registry.
+
+#### `platformsexist`
+
+Set `platformexist` to `all` (the default) to validate all platform images exist.
+The registry will validate that the images referenced by the index exist in the
+registry before accepting the image index.
+
+Set `platformsexist` to `none` to disable all validation that images exist when an
+image index manifest is uploaded. This allows image lists to be uploaded to the
+registry without their associated images. This setting is experimental because
+other tooling that uses the registry may expect the image index to be complete.
+
+Set `platformsexist` to `selected` to selectively validate the existence of platforms
+within image index manifests. This setting is experimental because other tooling
+that uses the registry may expect the image index to be complete.
+
+#### `selectedplatforms`
+
+When `platformsexist` is set to `selected`, set `selectedplatforms` to an array of
+platforms to validate. If a platform is included in this the array and in the images
+contained within an index, the registry will validate that the platform specific image
+exists in the registry before accepting the index. The registry will not validate the
+existence of platform specific images in the index that do not appear in the
+`selectedplatforms` array.
+
+This parameter does not validate that the configured platforms are included in every
+index. If an image index does not include one of the platform specific images configured
+in the `selectedplatforms` array, it may still be accepted by the registry.
+
+Each platform is a map with two keys, `os` and `architecture`, as defined in the
+[OCI Image Index specification](https://github.com/opencontainers/image-spec/blob/main/image-index.md#image-index-property-descriptions).
+
 ## Example: Development configuration
 
 You can use this simple example for local development:

manifests.go

@@ -47,7 +47,7 @@ type ManifestBuilder interface {
 	AppendReference(dependency Describable) error
 }
 
-// ManifestService describes operations on image manifests.
+// ManifestService describes operations on manifests.
 type ManifestService interface {
 	// Exists returns true if the manifest exists.
 	Exists(ctx context.Context, dgst digest.Digest) (bool, error)

registry/handlers/api_test.go

@@ -2432,7 +2432,7 @@ func pushChunk(t *testing.T, ub *v2.URLBuilder, name reference.Named, uploadURLB
 
 func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) {
 	if resp.StatusCode != expectedStatus {
-		t.Logf("unexpected status %s: %v != %v", msg, resp.StatusCode, expectedStatus)
+		t.Logf("unexpected status %s: expected %v, got %v", msg, resp.StatusCode, expectedStatus)
 		maybeDumpResponse(t, resp)
 		t.FailNow()
 	}

registry/handlers/app.go

@@ -234,6 +234,21 @@ func NewApp(ctx context.Context, config *configuration.Configuration) *App {
 				options = append(options, storage.ManifestURLsDenyRegexp(re))
 			}
 		}
+
+		switch config.Validation.ImageIndexes.PlatformsExist {
+		case "selected":
+			options = append(options, storage.EnableValidateImageIndexImagesExist)
+			for _, platform := range config.Validation.ImageIndexes.SelectedPlatforms {
+				options = append(options, storage.AddValidateImageIndexImagesExistPlatform(platform.Architecture, platform.OS))
+			}
+			fallthrough
+		case "none":
+			dcontext.GetLogger(app).Warnf("Image index completeness validation has been disabled, which is an experimental option because other container tooling might expect all image indexes to be complete")
+		case "all":
+			fallthrough
+		default:
+			options = append(options, storage.EnableValidateImageIndexImagesExist)
+		}
 	}
 
 	// configure storage caches

registry/storage/manifestlisthandler.go

@@ -13,9 +13,11 @@ import (
 
 // manifestListHandler is a ManifestHandler that covers schema2 manifest lists.
 type manifestListHandler struct {
-	repository distribution.Repository
-	blobStore  distribution.BlobStore
-	ctx        context.Context
+	repository                   distribution.Repository
+	blobStore                    distribution.BlobStore
+	ctx                          context.Context
+	validateImagesExist          bool
+	validateImagesExistPlatforms []platform // [] = All platforms
 }
 
 var _ ManifestHandler = &manifestListHandler{}
@@ -74,7 +76,7 @@ func (ms *manifestListHandler) Put(ctx context.Context, manifestList distributio
 func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distribution.Manifest, skipDependencyVerification bool) error {
 	var errs distribution.ErrManifestVerification
 
-	if !skipDependencyVerification {
+	if ms.validateImagesExist && !skipDependencyVerification {
 		// This manifest service is different from the blob service
 		// returned by Blob. It uses a linked blob store to ensure that
 		// only manifests are accessible.
@@ -85,6 +87,10 @@ func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distrib
 		}
 
 		for _, manifestDescriptor := range mnfst.References() {
+			if !ms.platformMustExist(manifestDescriptor) {
+				continue
+			}
+
 			exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
 			if err != nil && err != distribution.ErrBlobUnknown {
 				errs = append(errs, err)
@@ -101,3 +107,20 @@ func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst distrib
 
 	return nil
 }
+
+func (ms *manifestListHandler) platformMustExist(descriptor distribution.Descriptor) bool {
+	if len(ms.validateImagesExistPlatforms) == 0 {
+		return true
+	}
+
+	imagePlatform := descriptor.Platform
+
+	for _, platform := range ms.validateImagesExistPlatforms {
+		if imagePlatform.Architecture == platform.architecture &&
+			imagePlatform.OS == platform.os {
+			return true
+		}
+	}
+
+	return false
+}