handle leader changes

e2e/newenemy/newenemy_test.go

@@ -149,7 +149,7 @@ func TestNoNewEnemy(t *testing.T) {
 	for i := 0; i < sampleSize; i++ {
 		t.Log(i, "check vulnerability with mitigations disabled")
 		checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Minute)
-		protected, attempts := checkNoNewEnemy(checkCtx, t, slowPrefix, crdb, vulnerableSpiceDb, -1)
+		protected, attempts := checkNoNewEnemy(checkCtx, t, schemaData, slowNodeId, crdb, vulnerableSpiceDb, -1)
 		require.NotNil(protected, "unable to determine if spicedb displays newenemy when mitigations are disabled within the time limit")
 		require.False(*protected)
 		checkCancel()
@@ -177,14 +177,16 @@ func TestNoNewEnemy(t *testing.T) {
 	}
 
 	t.Logf("check spicedb is protected after %d attempts", iterations)
-	protected, _ := checkNoNewEnemy(ctx, t, slowPrefix, crdb, protectedSpiceDb, iterations)
+	protected, _ := checkNoNewEnemy(ctx, t, schemaData, slowNodeId, crdb, protectedSpiceDb, iterations)
 	require.NotNil(protected, "unable to determine if spicedb is protected within the time limit")
 	require.True(*protected, "protection is enabled, but newenemy detected")
 }
 
 // checkNoNewEnemy returns true if the service is protected, false if it is vulnerable, and nil if we couldn't determine
-func checkNoNewEnemy(ctx context.Context, t testing.TB, prefix string, crdb cockroach.Cluster, spicedb spice.Cluster, candidateCount int) (*bool, int) {
+func checkNoNewEnemy(ctx context.Context, t testing.TB, schemaData []SchemaData, slowNodeId int, crdb cockroach.Cluster, spicedb spice.Cluster, candidateCount int) (*bool, int) {
 	var attempts int
+	prefix := prefixForNode(ctx, crdb[1].Conn(), schemaData, slowNodeId)
+
 	for {
 		attempts++
 		directs, excludes := generateTuples(prefix, 1, generator.NewUniqueGenerator(objIDRegex))
@@ -229,7 +231,13 @@ func checkNoNewEnemy(ctx context.Context, t testing.TB, prefix string, crdb cock
 			t.Log("service is subject to the new enemy problem")
 		}
 
-		analyzeCalls(os.Stdout, crdb[1].Conn(), excludes[0].Tuple, directs[0].Tuple, r1.GetRevision(), r2.GetRevision())
+		if !analyzeCalls(os.Stdout, crdb[1].Conn(), excludes[0].Tuple, directs[0].Tuple, r1.GetRevision(), r2.GetRevision(), slowNodeId) {
+			t.Log("namespace leader changed, pick new prefix and fill")
+			prefix = prefixForNode(ctx, crdb[1].Conn(), schemaData, slowNodeId)
+			// need to fill new prefix
+			t.Log("filling with data to span multiple ranges")
+			fill(require.New(t), spicedb[0].Client().V0().ACL(), prefix, 4000, 1000)
+		}
 
 		if canHas.IsMember {
 			t.Log("service is subject to the new enemy problem")
@@ -290,7 +298,7 @@ func BenchmarkConflictingTupleWrites(b *testing.B) {
 
 	b.ResetTimer()
 
-	checkNoNewEnemy(ctx, b, "", crdb, spicedb, b.N)
+	checkNoNewEnemy(ctx, b, generateSchemaData(1, 1), 1, crdb, spicedb, b.N)
 }
 
 func fill(require *require.Assertions, client v0.ACLServiceClient, prefix string, fillerCount, batchSize int) {
@@ -399,15 +407,14 @@ func generateTuples(prefix string, n int, objIdGenerator *generator.UniqueGenera
 }
 
 // after we've checked, analyze the previous calls
-func analyzeCalls(out io.Writer, conn *pgx.Conn, t1, t2 *v0.RelationTuple, r1, r2 *v0.Zookie) {
+// returns false if the namespace leader is no longer on the slow node
+func analyzeCalls(out io.Writer, conn *pgx.Conn, t1, t2 *v0.RelationTuple, r1, r2 *v0.Zookie, slowNodeId int) bool {
 	l1, l2 := getLeaderNode(testCtx, conn, t1), getLeaderNode(testCtx, conn, t2)
 	if l1 != l2 {
 		fmt.Fprintln(out, "different leaders for writes: ", l1, l2)
 	}
 
-	nl1, nl2 := getLeaderNodeForNamespace(testCtx, conn, t1.ObjectAndRelation.Namespace), getLeaderNodeForNamespace(testCtx, conn, t1.User.GetUserset().Namespace)
-	nl3, nl4 := getLeaderNodeForNamespace(testCtx, conn, t2.ObjectAndRelation.Namespace), getLeaderNodeForNamespace(testCtx, conn, t2.User.GetUserset().Namespace)
-
+	namespaceLeaderNode := getLeaderNodeForNamespace(testCtx, conn, t1.ObjectAndRelation.Namespace)
 	z1, _ := zookie.DecodeRevision(r1)
 	z2, _ := zookie.DecodeRevision(r2)
 
@@ -417,7 +424,9 @@ func analyzeCalls(out io.Writer, conn *pgx.Conn, t1, t2 *v0.RelationTuple, r1, r
 		fmt.Fprintln(out, "candidate found")
 	}
 
-	fmt.Fprintln(out, z1, z2, z1.Sub(z2).String(), l1, l2, nl1, nl2, nl3, nl4)
+	fmt.Fprintln(out, z1, z2, z1.Sub(z2).String(), l1, l2, namespaceLeaderNode)
+
+	return namespaceLeaderNode == slowNodeId
 }
 
 // getLeaderNode returns the node with the lease leader for the range containing the tuple