From 9ac02a086c79d3fc31bb5bdb0af182441746fbab Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Tue, 28 Dec 2021 12:30:10 -0500 Subject: [PATCH] handle leader changes --- e2e/newenemy/newenemy_test.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/e2e/newenemy/newenemy_test.go b/e2e/newenemy/newenemy_test.go index be2e120f8c..8bf92238b9 100644 --- a/e2e/newenemy/newenemy_test.go +++ b/e2e/newenemy/newenemy_test.go @@ -149,7 +149,7 @@ func TestNoNewEnemy(t *testing.T) { for i := 0; i < sampleSize; i++ { t.Log(i, "check vulnerability with mitigations disabled") checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Minute) - protected, attempts := checkNoNewEnemy(checkCtx, t, slowPrefix, crdb, vulnerableSpiceDb, -1) + protected, attempts := checkNoNewEnemy(checkCtx, t, slowPrefix, slowNodeId, crdb, vulnerableSpiceDb, -1) require.NotNil(protected, "unable to determine if spicedb displays newenemy when mitigations are disabled within the time limit") require.False(*protected) checkCancel() @@ -183,8 +183,10 @@ func TestNoNewEnemy(t *testing.T) { } // checkNoNewEnemy returns true if the service is protected, false if it is vulnerable, and nil if we couldn't determine -func checkNoNewEnemy(ctx context.Context, t testing.TB, prefix string, crdb cockroach.Cluster, spicedb spice.Cluster, candidateCount int) (*bool, int) { +func checkNoNewEnemy(ctx context.Context, t testing.TB, schemaData []SchemaData, slowNodeId int, crdb cockroach.Cluster, spicedb spice.Cluster, candidateCount int) (*bool, int) { var attempts int + prefix := prefixForNode(ctx, crdb[1].Conn(), schemaData, slowNodeId) + for { attempts++ directs, excludes := generateTuples(prefix, 1, generator.NewUniqueGenerator(objIDRegex)) @@ -229,7 +231,13 @@ func checkNoNewEnemy(ctx context.Context, t testing.TB, prefix string, crdb cock t.Log("service is subject to the new enemy problem") } - analyzeCalls(os.Stdout, crdb[1].Conn(), excludes[0].Tuple, directs[0].Tuple, r1.GetRevision(), r2.GetRevision()) + if !analyzeCalls(os.Stdout, crdb[1].Conn(), excludes[0].Tuple, directs[0].Tuple, r1.GetRevision(), r2.GetRevision(), slowNodeId) { + t.Log("namespace leader changed, pick new prefix and fill") + prefix = prefixForNode(ctx, crdb[1].Conn(), schemaData, slowNodeId) + // need to fill new prefix + t.Log("filling with data to span multiple ranges") + fill(require.New(t), spicedb[0].Client().V0().ACL(), prefix, 4000, 1000) + } if canHas.IsMember { t.Log("service is subject to the new enemy problem") @@ -399,15 +407,14 @@ func generateTuples(prefix string, n int, objIdGenerator *generator.UniqueGenera } // after we've checked, analyze the previous calls -func analyzeCalls(out io.Writer, conn *pgx.Conn, t1, t2 *v0.RelationTuple, r1, r2 *v0.Zookie) { +// returns false if the namespace leader is no longer on the slow node +func analyzeCalls(out io.Writer, conn *pgx.Conn, t1, t2 *v0.RelationTuple, r1, r2 *v0.Zookie, slowNodeId int) bool { l1, l2 := getLeaderNode(testCtx, conn, t1), getLeaderNode(testCtx, conn, t2) if l1 != l2 { fmt.Fprintln(out, "different leaders for writes: ", l1, l2) } - nl1, nl2 := getLeaderNodeForNamespace(testCtx, conn, t1.ObjectAndRelation.Namespace), getLeaderNodeForNamespace(testCtx, conn, t1.User.GetUserset().Namespace) - nl3, nl4 := getLeaderNodeForNamespace(testCtx, conn, t2.ObjectAndRelation.Namespace), getLeaderNodeForNamespace(testCtx, conn, t2.User.GetUserset().Namespace) - + namespaceLeaderNode := getLeaderNodeForNamespace(testCtx, conn, t1.ObjectAndRelation.Namespace) z1, _ := zookie.DecodeRevision(r1) z2, _ := zookie.DecodeRevision(r2) @@ -417,7 +424,9 @@ func analyzeCalls(out io.Writer, conn *pgx.Conn, t1, t2 *v0.RelationTuple, r1, r fmt.Fprintln(out, "candidate found") } - fmt.Fprintln(out, z1, z2, z1.Sub(z2).String(), l1, l2, nl1, nl2, nl3, nl4) + fmt.Fprintln(out, z1, z2, z1.Sub(z2).String(), l1, l2, namespaceLeaderNode) + + return namespaceLeaderNode == slowNodeId } // getLeaderNode returns the node with the lease leader for the range containing the tuple