support external redis sentinel cluster distribution#2886

configuration/configuration.go

@@ -168,6 +168,9 @@ type Configuration struct {
 		// Addr specifies the the redis instance available to the application.
 		Addr string `yaml:"addr,omitempty"`
 
+		// SentinelMasterSet specifies the the redis sentinel master set name.
+		SentinelMasterSet string `yaml:"sentinelMasterSet,omitempty"`
+
 		// Password string to use when making a connection.
 		Password string `yaml:"password,omitempty"`
 

registry/handlers/app.go

@@ -3,6 +3,7 @@ package handlers
 import (
 	"context"
 	"crypto/rand"
+	"errors"
 	"expvar"
 	"fmt"
 	"math"
@@ -16,6 +17,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/FZambia/sentinel"
+
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/configuration"
 	dcontext "github.com/docker/distribution/context"
@@ -499,6 +502,44 @@ func (app *App) configureRedis(configuration *configuration.Configuration) {
 		return
 	}
 
+	var getRedisAddr func() (string, error)
+	var testOnBorrow func(c redis.Conn, t time.Time) error
+	if configuration.Redis.SentinelMasterSet != "" {
+		sntnl := &sentinel.Sentinel{
+			Addrs:      strings.Split(configuration.Redis.Addr, ","),
+			MasterName: configuration.Redis.SentinelMasterSet,
+			Dial: func(addr string) (redis.Conn, error) {
+				c, err := redis.DialTimeout("tcp", addr,
+					configuration.Redis.DialTimeout,
+					configuration.Redis.ReadTimeout,
+					configuration.Redis.WriteTimeout)
+				if err != nil {
+					return nil, err
+				}
+				return c, nil
+			},
+		}
+		getRedisAddr = func() (string, error) {
+			return sntnl.MasterAddr()
+		}
+		testOnBorrow = func(c redis.Conn, t time.Time) error {
+			if !sentinel.TestRole(c, "master") {
+				return errors.New("role check failed")
+			}
+			return nil
+		}
+
+	} else {
+		getRedisAddr = func() (string, error) {
+			return configuration.Redis.Addr, nil
+		}
+		testOnBorrow = func(c redis.Conn, t time.Time) error {
+			// TODO(stevvooe): We can probably do something more interesting
+			// here with the health package.
+			_, err := c.Do("PING")
+			return err
+		}
+	}
 	pool := &redis.Pool{
 		Dial: func() (redis.Conn, error) {
 			// TODO(stevvooe): Yet another use case for contextual timing.
@@ -514,8 +555,11 @@ func (app *App) configureRedis(configuration *configuration.Configuration) {
 				}
 			}
 
-			conn, err := redis.DialTimeout("tcp",
-				configuration.Redis.Addr,
+			redisAddr, err := getRedisAddr()
+			if err != nil {
+				return nil, err
+			}
+			conn, err := redis.DialTimeout("tcp", redisAddr,
 				configuration.Redis.DialTimeout,
 				configuration.Redis.ReadTimeout,
 				configuration.Redis.WriteTimeout)
@@ -547,16 +591,11 @@ func (app *App) configureRedis(configuration *configuration.Configuration) {
 			done(nil)
 			return conn, nil
 		},
-		MaxIdle:     configuration.Redis.Pool.MaxIdle,
-		MaxActive:   configuration.Redis.Pool.MaxActive,
-		IdleTimeout: configuration.Redis.Pool.IdleTimeout,
-		TestOnBorrow: func(c redis.Conn, t time.Time) error {
-			// TODO(stevvooe): We can probably do something more interesting
-			// here with the health package.
-			_, err := c.Do("PING")
-			return err
-		},
-		Wait: false, // if a connection is not available, proceed without cache.
+		MaxIdle:      configuration.Redis.Pool.MaxIdle,
+		MaxActive:    configuration.Redis.Pool.MaxActive,
+		IdleTimeout:  configuration.Redis.Pool.IdleTimeout,
+		TestOnBorrow: testOnBorrow,
+		Wait:         false, // if a connection is not available, proceed without cache.
 	}
 
 	app.redis = pool

registry/handlers/app_test.go

@@ -140,7 +140,29 @@ func TestAppDispatcher(t *testing.T) {
 // TestNewApp covers the creation of an application via NewApp with a
 // configuration.
 func TestNewApp(t *testing.T) {
-	ctx := context.Background()
+
+	config := configuration.Configuration{
+		Storage: configuration.Storage{
+			"testdriver": nil,
+			"maintenance": configuration.Parameters{"uploadpurging": map[interface{}]interface{}{
+				"enabled": false,
+			}},
+		},
+		Auth: configuration.Auth{
+			// For now, we simply test that new auth results in a viable
+			// application.
+			"silly": {
+				"realm":   "realm-test",
+				"service": "service-test",
+			},
+		},
+	}
+	runAppWithConfig(t, config)
+}
+
+// TestNewApp covers the creation of an application via NewApp with a
+// configuration(with redis).
+func TestNewAppWithRedis(t *testing.T) {
 	config := configuration.Configuration{
 		Storage: configuration.Storage{
 			"testdriver": nil,
@@ -157,7 +179,38 @@ func TestNewApp(t *testing.T) {
 			},
 		},
 	}
+	config.Redis.Addr = "127.0.0.1:6379"
+	config.Redis.DB = 0
+	runAppWithConfig(t, config)
+}
 
+// TestNewApp covers the creation of an application via NewApp with a
+// configuration(with redis sentinel cluster).
+func TestNewAppWithRedisSentinelCluster(t *testing.T) {
+	config := configuration.Configuration{
+		Storage: configuration.Storage{
+			"testdriver": nil,
+			"maintenance": configuration.Parameters{"uploadpurging": map[interface{}]interface{}{
+				"enabled": false,
+			}},
+		},
+		Auth: configuration.Auth{
+			// For now, we simply test that new auth results in a viable
+			// application.
+			"silly": {
+				"realm":   "realm-test",
+				"service": "service-test",
+			},
+		},
+	}
+	config.Redis.Addr = "192.168.0.11:26379,192.168.0.12:26379"
+	config.Redis.DB = 0
+	config.Redis.SentinelMasterSet = "mymaster"
+	runAppWithConfig(t, config)
+}
+
+func runAppWithConfig(t *testing.T, config configuration.Configuration) {
+	ctx := context.Background()
 	// Mostly, with this test, given a sane configuration, we are simply
 	// ensuring that NewApp doesn't panic. We might want to tweak this
 	// behavior.

vendor.conf

@@ -49,3 +49,4 @@ gopkg.in/yaml.v2 v2.2.1
 rsc.io/letsencrypt e770c10b0f1a64775ae91d240407ce00d1a5bdeb https://github.com/dmcgowan/letsencrypt.git
 github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb
 github.com/opencontainers/image-spec 67d2d5658fe0476ab9bf414cec164077ebff3920 # v1.0.2
+github.com/FZambia/sentinel 5585739eb4b6478aa30161866ccf9ce0ef5847c7 https://github.com/jeremyxu2010/sentinel.git