server,test: refresh cache on each NewAuthStore

- permissions were incorrectly loaded on restarts.
- #14355
- Backport of #14358

Signed-off-by: vivekpatani <9080894+vivekpatani@users.noreply.github.com>

server/auth/store.go

@@ -1073,6 +1073,8 @@ func NewAuthStore(lg *zap.Logger, be backend.Backend, tp TokenProvider, bcryptCo
 
 	as.setupMetricsReporter()
 
+	as.refreshRangePermCache(tx)
+
 	tx.Unlock()
 	be.ForceCommit()
 

tests/e2e/ctl_v3_auth_test.go

@@ -70,6 +70,7 @@ func TestCtlV3AuthSnapshot(t *testing.T)            { testCtl(t, authTestSnapsho
 func TestCtlV3AuthSnapshotJWT(t *testing.T)         { testCtl(t, authTestSnapshot, withCfg(*newConfigJWT())) }
 func TestCtlV3AuthJWTExpire(t *testing.T)           { testCtl(t, authTestJWTExpire, withCfg(*newConfigJWT())) }
 func TestCtlV3AuthRevisionConsistency(t *testing.T) { testCtl(t, authTestRevisionConsistency) }
+func TestCtlV3AuthTestCacheReload(t *testing.T)     { testCtl(t, authTestCacheReload) }
 
 func authEnableTest(cx ctlCtx) {
 	if err := authEnable(cx); err != nil {
@@ -1206,3 +1207,85 @@ func authTestRevisionConsistency(cx ctlCtx) {
 		cx.t.Fatalf("auth revison shouldn't change when restarting etcd, expected: %d, got: %d", oldAuthRevision, newAuthRevision)
 	}
 }
+
+// authTestCacheReload tests the permissions when a member restarts
+func authTestCacheReload(cx ctlCtx) {
+
+	authData := []struct {
+		user string
+		role string
+		pass string
+	}{
+		{
+			user: "root",
+			role: "root",
+			pass: "123",
+		},
+		{
+			user: "user0",
+			role: "role0",
+			pass: "123",
+		},
+	}
+
+	node0 := cx.epc.procs[0]
+	endpoint := node0.EndpointsV3()[0]
+
+	// create a client
+	c, err := clientv3.New(clientv3.Config{Endpoints: []string{endpoint}, DialTimeout: 3 * time.Second})
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+	defer c.Close()
+
+	for _, authObj := range authData {
+		// add role
+		if _, err = c.RoleAdd(context.TODO(), authObj.role); err != nil {
+			cx.t.Fatal(err)
+		}
+
+		// add user
+		if _, err = c.UserAdd(context.TODO(), authObj.user, authObj.pass); err != nil {
+			cx.t.Fatal(err)
+		}
+
+		// grant role to user
+		if _, err = c.UserGrantRole(context.TODO(), authObj.user, authObj.role); err != nil {
+			cx.t.Fatal(err)
+		}
+	}
+
+	// role grant permission to role0
+	if _, err = c.RoleGrantPermission(context.TODO(), authData[1].role, "foo", "", clientv3.PermissionType(clientv3.PermReadWrite)); err != nil {
+		cx.t.Fatal(err)
+	}
+
+	// enable auth
+	if _, err = c.AuthEnable(context.TODO()); err != nil {
+		cx.t.Fatal(err)
+	}
+
+	// create another client with ID:Password
+	c2, err := clientv3.New(clientv3.Config{Endpoints: []string{endpoint}, Username: authData[1].user, Password: authData[1].pass, DialTimeout: 3 * time.Second})
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+	defer c2.Close()
+
+	// create foo since that is within the permission set
+	// expectation is to succeed
+	if _, err = c2.Put(context.TODO(), "foo", "bar"); err != nil {
+		cx.t.Fatal(err)
+	}
+
+	// restart the node
+	node0.WithStopSignal(syscall.SIGINT)
+	if err := node0.Restart(); err != nil {
+		cx.t.Fatal(err)
+	}
+
+	// nothing has changed, but it fails without refreshing cache after restart
+	if _, err = c2.Put(context.TODO(), "foo", "bar2"); err != nil {
+		cx.t.Fatal(err)
+	}
+}

tests/integration/v3_auth_test.go

@@ -421,3 +421,79 @@ func TestV3AuthOldRevConcurrent(t *testing.T) {
 	}
 	wg.Wait()
 }
+
+func TestV3AuthRestartMember(t *testing.T) {
+	BeforeTest(t)
+
+	// create a cluster with 1 member
+	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	// create a client
+	c, cerr := NewClient(t, clientv3.Config{
+		Endpoints:   clus.Client(0).Endpoints(),
+		DialTimeout: 5 * time.Second,
+	})
+	testutil.AssertNil(t, cerr)
+	defer c.Close()
+
+	authData := []struct {
+		user string
+		role string
+		pass string
+	}{
+		{
+			user: "root",
+			role: "root",
+			pass: "123",
+		},
+		{
+			user: "user0",
+			role: "role0",
+			pass: "123",
+		},
+	}
+
+	for _, authObj := range authData {
+		// add a role
+		_, err := c.RoleAdd(context.TODO(), authObj.role)
+		testutil.AssertNil(t, err)
+		// add a user
+		_, err = c.UserAdd(context.TODO(), authObj.user, authObj.pass)
+		testutil.AssertNil(t, err)
+		// grant role to user
+		_, err = c.UserGrantRole(context.TODO(), authObj.user, authObj.role)
+		testutil.AssertNil(t, err)
+	}
+
+	// role grant permission to role0
+	_, err := c.RoleGrantPermission(context.TODO(), authData[1].role, "foo", "", clientv3.PermissionType(clientv3.PermReadWrite))
+	testutil.AssertNil(t, err)
+
+	// enable auth
+	_, err = c.AuthEnable(context.TODO())
+	testutil.AssertNil(t, err)
+
+	// create another client with ID:Password
+	c2, cerr := NewClient(t, clientv3.Config{
+		Endpoints:   clus.Client(0).Endpoints(),
+		DialTimeout: 5 * time.Second,
+		Username:    authData[1].user,
+		Password:    authData[1].pass,
+	})
+	testutil.AssertNil(t, cerr)
+	defer c2.Close()
+
+	// create foo since that is within the permission set
+	// expectation is to succeed
+	_, err = c2.Put(context.TODO(), "foo", "bar")
+	testutil.AssertNil(t, err)
+
+	clus.Members[0].Stop(t)
+	err = clus.Members[0].Restart(t)
+	testutil.AssertNil(t, err)
+
+	// nothing has changed, but it fails without refreshing cache after restart
+	_, err = c2.Put(context.TODO(), "foo", "bar2")
+	testutil.AssertNil(t, err)
+}