From 04efef4fc8bdccb3608648818f304721967ab599 Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Wed, 22 Dec 2021 09:59:00 -0500 Subject: [PATCH] e2e: fill schema with many namespaces to span ranges --- .github/workflows/build.yaml | 2 - e2e/cockroach/cockroach.go | 24 ++++ e2e/generator/names.go | 25 ++++ e2e/go.mod | 2 +- e2e/go.sum | 9 ++ e2e/newenemy/newenemy_test.go | 241 ++++++++++++++++++++++++++-------- e2e/spice/spicedb.go | 1 + internal/namespace/caching.go | 1 + internal/namespace/manager.go | 2 +- 9 files changed, 246 insertions(+), 61 deletions(-) create mode 100644 e2e/generator/names.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 47121007a0..4939dac5b9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -64,8 +64,6 @@ jobs: run: "go test -tags ci ./..." e2e: - # Explicitly disabling e2e testing - if: "false" name: "E2E" runs-on: "ubuntu-latest" steps: diff --git a/e2e/cockroach/cockroach.go b/e2e/cockroach/cockroach.go index 08d790852e..b705f1bf34 100644 --- a/e2e/cockroach/cockroach.go +++ b/e2e/cockroach/cockroach.go @@ -73,6 +73,30 @@ func (c *Node) Conn() *pgx.Conn { return c.conn } +// NodeID returns the cockroach-internal node id for this connection. This is +// the value that is referenced by other crdb metadata to identify range leader, +// follower nodes, etc. +func (c *Node) NodeID(ctx context.Context) (int, error) { + rows, err := c.conn.Query(ctx, "SHOW node_id") + defer rows.Close() + if err != nil { + return -1, err + } + // despite being an int, crdb returns node id as a string + var nodeID string + for rows.Next() { + if err := rows.Scan(&nodeID); err != nil { + return -1, err + } + break + } + i, err := strconv.Atoi(nodeID) + if err != nil { + return -1, err + } + return i, nil +} + // Cluster represents a set of Node nodes configured to talk to // each other. type Cluster []*Node diff --git a/e2e/generator/names.go b/e2e/generator/names.go new file mode 100644 index 0000000000..a81230f4d1 --- /dev/null +++ b/e2e/generator/names.go @@ -0,0 +1,25 @@ +package generator + +import "github.com/brianvoe/gofakeit/v6" + +type UniqueGenerator struct { + seen map[string]struct{} + regex string +} + +func NewUniqueGenerator(regex string) *UniqueGenerator { + return &UniqueGenerator{ + seen: make(map[string]struct{}, 0), + regex: regex, + } +} + +func (g *UniqueGenerator) Next() string { + for { + val := gofakeit.Regex(g.regex) + if _, ok := g.seen[val]; !ok { + g.seen[val] = struct{}{} + return val + } + } +} diff --git a/e2e/go.mod b/e2e/go.mod index 4428ddf021..da921ac256 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -6,6 +6,7 @@ require ( github.com/authzed/authzed-go v0.3.1-0.20211220220442-a36f72252b43 github.com/authzed/grpcutil v0.0.0-20211020204402-aba1876830e6 github.com/authzed/spicedb v0.0.0 + github.com/brianvoe/gofakeit/v6 v6.10.0 github.com/jackc/pgtype v1.9.1 github.com/jackc/pgx/v4 v4.14.1 github.com/stretchr/testify v1.7.0 @@ -25,7 +26,6 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.2.0 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/kr/text v0.2.0 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect diff --git a/e2e/go.sum b/e2e/go.sum index 80d8907429..a265e0bf0a 100644 --- a/e2e/go.sum +++ b/e2e/go.sum @@ -81,6 +81,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= +github.com/brianvoe/gofakeit/v6 v6.10.0 h1:0lZpqKzY2xVfjmCQBn9g9+SHIGg58SX+vu/ejuSVGMc= +github.com/brianvoe/gofakeit/v6 v6.10.0/go.mod h1:palrJUk4Fyw38zIFB/uBZqsgzW5VsNllhHKKwAebzew= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= @@ -158,6 +160,7 @@ github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0C github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -389,6 +392,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -464,6 +468,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -498,6 +503,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= @@ -842,6 +849,7 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1110,6 +1118,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +mvdan.cc/gofumpt v0.2.1/go.mod h1:a/rvZPhsNaedOJBzqRD9omnwVwHZsBdJirXHa9Gh9Ig= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/e2e/newenemy/newenemy_test.go b/e2e/newenemy/newenemy_test.go index a1841e1cfd..73397d1bc3 100644 --- a/e2e/newenemy/newenemy_test.go +++ b/e2e/newenemy/newenemy_test.go @@ -2,34 +2,58 @@ package newenemy import ( "context" + "database/sql" + "flag" "fmt" "io" + "log" "math" "math/rand" "os" + "strings" "testing" + "text/template" "time" v0 "github.com/authzed/authzed-go/proto/authzed/api/v0" "github.com/authzed/authzed-go/proto/authzed/api/v1alpha1" "github.com/authzed/spicedb/pkg/zookie" + "github.com/jackc/pgtype" + "github.com/jackc/pgx/v4" "github.com/stretchr/testify/require" "github.com/authzed/spicedb/e2e" "github.com/authzed/spicedb/e2e/cockroach" + "github.com/authzed/spicedb/e2e/generator" "github.com/authzed/spicedb/e2e/spice" ) -const schema = ` -definition user {} -definition resource { - relation direct: user - relation excluded: user +type SchemaData struct { + Prefixes []string +} + +const schemaText = ` +{{ range .Prefixes }} +definition {{.}}/user {} +definition {{.}}/resource { + relation direct: {{.}}/user + relation excluded: {{.}}/user permission allowed = direct - excluded } +{{ end }} ` -var testCtx context.Context +const ( + objIDRegex = "[a-zA-Z0-9_][a-zA-Z0-9/_-]{0,127}" + namespacePrefixRegex = "[a-z][a-z0-9_]{2,62}[a-z0-9]" +) + +var ( + maxIterations = flag.Int("max-iterations", 1000, "iteration cap for statistic-based tests (0 for no limit)") + + schemaTpl = template.Must(template.New("schema").Parse(schemaText)) + testCtx context.Context +) func TestMain(m *testing.M) { var cancel context.CancelFunc @@ -45,7 +69,7 @@ const ( dbName = "spicedbnetest" ) -func startCluster(ctx context.Context, t testing.TB) cockroach.Cluster { +func initializeTestCRDBCluster(ctx context.Context, t testing.TB) cockroach.Cluster { require := require.New(t) fmt.Println("starting cockroach...") @@ -65,15 +89,18 @@ func startCluster(ctx context.Context, t testing.TB) cockroach.Cluster { fmt.Println("attempting to connect...") require.NoError(crdbCluster[2].Connect(ctx, os.Stdout, dbName)) + require.NoError(crdbCluster[1].Connect(ctx, os.Stdout, dbName)) + require.NoError(crdbCluster[0].Connect(ctx, os.Stdout, dbName)) return crdbCluster } func TestNoNewEnemy(t *testing.T) { require := require.New(t) + rand.Seed(time.Now().UnixNano()) ctx, cancel := context.WithCancel(testCtx) defer cancel() - crdb := startCluster(ctx, t) + crdb := initializeTestCRDBCluster(ctx, t) t.Log("starting vulnerable spicedb...") vulnerableSpiceDb := spice.NewClusterFromCockroachCluster(crdb, spice.WithDbName(dbName)) @@ -97,28 +124,32 @@ func TestNoNewEnemy(t *testing.T) { fmt.Sprintf(setSmallRanges, dbName), )) - t.Log("modifying time") - timeDelay := 100 * time.Millisecond - require.NoError(crdb.TimeDelay(ctx, e2e.MustFile(ctx, t, "timeattack.log"), 1, -1*timeDelay)) + t.Log("fill with schemas to span multiple ranges") + // 4000 is larger than we need to span all three nodes, but a higher number + // seems to make the test converge faster + schemaData := generateSchemaData(4000, 500) + require.NoError(fillSchema(schemaData, vulnerableSpiceDb[1].Client().V1Alpha1().Schema())) - t.Log("modifying network") - networkDelay := timeDelay / 2 - require.NoError(crdb.NetworkDelay(ctx, e2e.MustFile(ctx, t, "netattack.log"), 1, networkDelay)) + t.Log("determining a prefix with a leader on the slow node") + slowNodeId, err := crdb[1].NodeID(testCtx) + require.NoError(err) + slowPrefix := prefixForNode(ctx, crdb[1].Conn(), schemaData, slowNodeId) - t.Log("create initial test schema") - require.NoError(initSchema(vulnerableSpiceDb[0].Client().V1Alpha1().Schema())) + t.Logf("using prefix %s for slow node %d", slowPrefix, slowNodeId) t.Log("filling with data to span multiple ranges") - rand.Seed(time.Now().UnixNano()) - fill(require, vulnerableSpiceDb[0].Client().V0().ACL(), 4000, 100) + fill(require, vulnerableSpiceDb[0].Client().V0().ACL(), slowPrefix, 4000, 1000) + + t.Log("modifying time") + require.NoError(crdb.TimeDelay(ctx, e2e.MustFile(ctx, t, "timeattack.log"), 1, -200*time.Millisecond)) const sampleSize = 5 samples := make([]int, sampleSize) for i := 0; i < sampleSize; i++ { t.Log(i, "check vulnerability with mitigations disabled") - checkCtx, checkCancel := context.WithTimeout(ctx, 5*time.Minute) - protected, attempts := checkNoNewEnemy(checkCtx, t, vulnerableSpiceDb, -1) + checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Minute) + protected, attempts := checkNoNewEnemy(checkCtx, t, slowPrefix, crdb, vulnerableSpiceDb, -1) require.NotNil(protected, "unable to determine if spicedb displays newenemy when mitigations are disabled within the time limit") require.False(*protected) checkCancel() @@ -139,22 +170,26 @@ func TestNoNewEnemy(t *testing.T) { samplestddev := stddev / math.Sqrt(float64(sampleSize)) // how many iterations do we need to get > 3sigma from the mean? - // cap max_iterations to control test runtime - const max_iterations = 100 - iterations := int(math.Min(max_iterations, math.Ceil(3*stddev*samplestddev+mean))) + // cap maxIterations to control test runtime. + iterations := int(math.Ceil(3*stddev*samplestddev + mean)) + if *maxIterations != 0 { + if *maxIterations < iterations { + iterations = *maxIterations + } + } t.Logf("check spicedb is protected after %d attempts", iterations) - protected, _ := checkNoNewEnemy(ctx, t, protectedSpiceDb, iterations) + protected, _ := checkNoNewEnemy(ctx, t, slowPrefix, crdb, protectedSpiceDb, iterations) require.NotNil(protected, "unable to determine if spicedb is protected within the time limit") require.True(*protected, "protection is enabled, but newenemy detected") } // checkNoNewEnemy returns true if the service is protected, false if it is vulnerable, and nil if we couldn't determine -func checkNoNewEnemy(ctx context.Context, t testing.TB, spicedb spice.Cluster, candidateCount int) (*bool, int) { +func checkNoNewEnemy(ctx context.Context, t testing.TB, prefix string, crdb cockroach.Cluster, spicedb spice.Cluster, candidateCount int) (*bool, int) { var attempts int for { attempts++ - directs, excludes := generateTuples(1) + directs, excludes := generateTuples(prefix, 1, generator.NewUniqueGenerator(objIDRegex)) // write to node 1 r1, err := spicedb[0].Client().V0().ACL().Write(testCtx, &v0.WriteRequest{ @@ -165,6 +200,11 @@ func checkNoNewEnemy(ctx context.Context, t testing.TB, spicedb spice.Cluster, c continue } + // the first write has to read the namespaces from the second node, + // which will resync the timestamps. sleeping allows the clocks to get + // back out of sync + time.Sleep(100 * time.Millisecond) + // write to node 2 (clock is behind) r2, err := spicedb[1].Client().V0().ACL().Write(testCtx, &v0.WriteRequest{ Updates: []*v0.RelationTupleUpdate{directs[0]}, @@ -174,10 +214,10 @@ func checkNoNewEnemy(ctx context.Context, t testing.TB, spicedb spice.Cluster, c continue } - canHas, err := spicedb[2].Client().V0().ACL().Check(context.Background(), &v0.CheckRequest{ + canHas, err := spicedb[1].Client().V0().ACL().Check(context.Background(), &v0.CheckRequest{ TestUserset: &v0.ObjectAndRelation{ - Namespace: "resource", - ObjectId: "thegoods", + Namespace: directs[0].Tuple.ObjectAndRelation.Namespace, + ObjectId: directs[0].Tuple.ObjectAndRelation.ObjectId, Relation: "allowed", }, User: directs[0].Tuple.GetUser(), @@ -191,7 +231,7 @@ func checkNoNewEnemy(ctx context.Context, t testing.TB, spicedb spice.Cluster, c t.Log("service is subject to the new enemy problem") } - analyzeCalls(os.Stdout, r1.GetRevision(), r2.GetRevision()) + analyzeCalls(os.Stdout, crdb[1].Conn(), excludes[0].Tuple, directs[0].Tuple, r1.GetRevision(), r2.GetRevision()) if canHas.IsMember { t.Log("service is subject to the new enemy problem") @@ -218,15 +258,15 @@ func checkNoNewEnemy(ctx context.Context, t testing.TB, spicedb spice.Cluster, c func BenchmarkBatchWrites(b *testing.B) { ctx, cancel := context.WithCancel(testCtx) defer cancel() - crdb := startCluster(ctx, b) + crdb := initializeTestCRDBCluster(ctx, b) spicedb := spice.NewClusterFromCockroachCluster(crdb, spice.WithDbName(dbName)) require.NoError(b, spicedb.Start(ctx, os.Stdout, "")) require.NoError(b, spicedb.Connect(ctx, os.Stdout)) - exludes, directs := generateTuples(b.N * 20) + directs, excludes := generateTuples("", b.N*20, generator.NewUniqueGenerator(objIDRegex)) b.ResetTimer() _, err := spicedb[0].Client().V0().ACL().Write(testCtx, &v0.WriteRequest{ - Updates: exludes, + Updates: excludes, }) if err != nil { fmt.Println(err) @@ -242,23 +282,24 @@ func BenchmarkBatchWrites(b *testing.B) { func BenchmarkConflictingTupleWrites(b *testing.B) { ctx, cancel := context.WithCancel(testCtx) defer cancel() - crdb := startCluster(ctx, b) + crdb := initializeTestCRDBCluster(ctx, b) spicedb := spice.NewClusterFromCockroachCluster(crdb, spice.WithDbName(dbName)) require.NoError(b, spicedb.Start(ctx, os.Stdout, "")) require.NoError(b, spicedb.Connect(ctx, os.Stdout)) // fill with tuples to ensure we span multiple ranges - fill(require.New(b), spicedb[0].Client().V0().ACL(), 2000, 100) + fill(require.New(b), spicedb[0].Client().V0().ACL(), "", 2000, 100) b.ResetTimer() - checkNoNewEnemy(ctx, b, spicedb, b.N) + checkNoNewEnemy(ctx, b, "", crdb, spicedb, b.N) } -func fill(require *require.Assertions, client v0.ACLServiceClient, fillerCount, batchSize int) { - directs, excludes := generateTuples(fillerCount) +func fill(require *require.Assertions, client v0.ACLServiceClient, prefix string, fillerCount, batchSize int) { + fmt.Println("filling prefix", prefix) + directs, excludes := generateTuples(prefix, fillerCount, generator.NewUniqueGenerator(objIDRegex)) for i := 0; i < fillerCount/batchSize; i++ { - fmt.Println("filling ", i*batchSize, "to", (i+1)*batchSize) + fmt.Println("filling", i*batchSize, "to", (i+1)*batchSize) _, err := client.Write(testCtx, &v0.WriteRequest{ Updates: excludes[i*batchSize : (i+1)*batchSize], }) @@ -270,29 +311,70 @@ func fill(require *require.Assertions, client v0.ACLServiceClient, fillerCount, } } -func initSchema(schemaClient v1alpha1.SchemaServiceClient) error { - _, err := schemaClient.WriteSchema(context.Background(), &v1alpha1.WriteSchemaRequest{ - Schema: schema, - }) - return err +func fillSchema(data []SchemaData, schemaClient v1alpha1.SchemaServiceClient) error { + var b strings.Builder + batchSize := len(data[0].Prefixes) + for i, d := range data { + fmt.Println("filling ", i*batchSize, "to", (i+1)*batchSize) + b.Reset() + err := schemaTpl.Execute(&b, d) + if err != nil { + return err + } + _, err = schemaClient.WriteSchema(context.Background(), &v1alpha1.WriteSchemaRequest{ + Schema: b.String(), + }) + if err != nil { + return err + } + } + return nil +} + +// prefixForNode finds a prefix with a leader the node id +func prefixForNode(ctx context.Context, conn *pgx.Conn, data []SchemaData, node int) (prefix string) { + for prefix == "" { + // get a random prefix + d := data[rand.Intn(len(data))] + candidate := d.Prefixes[rand.Intn(len(d.Prefixes))] + fmt.Println("checking prefix", candidate) + leader := getLeaderNodeForNamespace(ctx, conn, candidate) + if leader == node { + prefix = candidate + } + } + return +} + +func generateSchemaData(n int, batchSize int) (data []SchemaData) { + data = make([]SchemaData, 0, n/batchSize) + prefixGenerator := generator.NewUniqueGenerator(namespacePrefixRegex) + for i := 0; i < n/batchSize; i++ { + schema := SchemaData{Prefixes: make([]string, 0, batchSize)} + for j := i * batchSize; j < (i+1)*batchSize; j++ { + schema.Prefixes = append(schema.Prefixes, prefixGenerator.Next()) + } + data = append(data, schema) + } + return } -func generateTuples(n int) (directs []*v0.RelationTupleUpdate, excludes []*v0.RelationTupleUpdate) { +func generateTuples(prefix string, n int, objIdGenerator *generator.UniqueGenerator) (directs []*v0.RelationTupleUpdate, excludes []*v0.RelationTupleUpdate) { directs = make([]*v0.RelationTupleUpdate, 0, n) excludes = make([]*v0.RelationTupleUpdate, 0, n) for i := 0; i < n; i++ { user := &v0.User{ UserOneof: &v0.User_Userset{ Userset: &v0.ObjectAndRelation{ - Namespace: "user", - ObjectId: randSeq(16), + Namespace: prefix + "/user", + ObjectId: objIdGenerator.Next(), Relation: "...", }, }, } tupleExclude := &v0.RelationTuple{ ObjectAndRelation: &v0.ObjectAndRelation{ - Namespace: "resource", + Namespace: prefix + "/resource", ObjectId: "thegoods", Relation: "excluded", }, @@ -300,7 +382,7 @@ func generateTuples(n int) (directs []*v0.RelationTupleUpdate, excludes []*v0.Re } tupleDirect := &v0.RelationTuple{ ObjectAndRelation: &v0.ObjectAndRelation{ - Namespace: "resource", + Namespace: prefix + "/resource", ObjectId: "thegoods", Relation: "direct", }, @@ -319,7 +401,15 @@ func generateTuples(n int) (directs []*v0.RelationTupleUpdate, excludes []*v0.Re } // after we've checked, analyze the previous calls -func analyzeCalls(out io.Writer, r1, r2 *v0.Zookie) { +func analyzeCalls(out io.Writer, conn *pgx.Conn, t1, t2 *v0.RelationTuple, r1, r2 *v0.Zookie) { + l1, l2 := getLeaderNode(testCtx, conn, t1), getLeaderNode(testCtx, conn, t2) + if l1 != l2 { + fmt.Fprintln(out, "different leaders for writes: ", l1, l2) + } + + nl1, nl2 := getLeaderNodeForNamespace(testCtx, conn, t1.ObjectAndRelation.Namespace), getLeaderNodeForNamespace(testCtx, conn, t1.User.GetUserset().Namespace) + nl3, nl4 := getLeaderNodeForNamespace(testCtx, conn, t2.ObjectAndRelation.Namespace), getLeaderNodeForNamespace(testCtx, conn, t2.User.GetUserset().Namespace) + z1, _ := zookie.DecodeRevision(r1) z2, _ := zookie.DecodeRevision(r2) @@ -329,15 +419,52 @@ func analyzeCalls(out io.Writer, r1, r2 *v0.Zookie) { fmt.Fprintln(out, "candidate found") } - fmt.Fprintln(out, z1, z2, z1.Sub(z2).String()) + fmt.Fprintln(out, z1, z2, z1.Sub(z2).String(), l1, l2, nl1, nl2, nl3, nl4) } -var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") +// getLeaderNode returns the node with the lease leader for the range containing the tuple +func getLeaderNode(ctx context.Context, conn *pgx.Conn, tuple *v0.RelationTuple) int { + t := tuple + rows, err := conn.Query(ctx, "SHOW RANGE FROM TABLE relation_tuple FOR ROW ($1::text,$2::text,$3::text,$4::text,$5::text,$6::text)", + t.ObjectAndRelation.Namespace, + t.ObjectAndRelation.ObjectId, + t.ObjectAndRelation.Relation, + t.User.GetUserset().Namespace, + t.User.GetUserset().ObjectId, + t.User.GetUserset().Relation, + ) + defer rows.Close() + if err != nil { + log.Fatalf("failed to exec: %v", err) + } + return leaderFromRangeRow(rows) +} -func randSeq(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letters[rand.Intn(len(letters))] +// getLeaderNodeForNamespace returns the node with the lease leader for the range containing the namespace +func getLeaderNodeForNamespace(ctx context.Context, conn *pgx.Conn, namespace string) int { + rows, err := conn.Query(ctx, "SHOW RANGE FROM TABLE namespace_config FOR ROW ($1::text)", + namespace, + ) + defer rows.Close() + if err != nil { + log.Fatalf("failed to exec: %v", err) + } + return leaderFromRangeRow(rows) +} + +func leaderFromRangeRow(rows pgx.Rows) int { + var startKey sql.NullString + var endKey sql.NullString + var rangeID int + var leaseHolder int + var leaseHoldeLocality sql.NullString + var replicas pgtype.Int8Array + var replicaLocalities pgtype.TextArray + for rows.Next() { + if err := rows.Scan(&startKey, &endKey, &rangeID, &leaseHolder, &leaseHoldeLocality, &replicas, &replicaLocalities); err != nil { + panic(err) + } + break } - return string(b) + return leaseHolder } diff --git a/e2e/spice/spicedb.go b/e2e/spice/spicedb.go index 5fbf11966c..2f7eea46e7 100644 --- a/e2e/spice/spicedb.go +++ b/e2e/spice/spicedb.go @@ -76,6 +76,7 @@ func (s *Node) Start(ctx context.Context, logprefix string, args ...string) erro "./spicedb", "serve", "--log-level=debug", + "--datastore-request-hedging=false", "--grpc-preshared-key=" + s.PresharedKey, "--datastore-engine=" + s.Datastore, "--datastore-conn-uri=" + s.URI, diff --git a/internal/namespace/caching.go b/internal/namespace/caching.go index 90e35b5742..60908ad7b9 100644 --- a/internal/namespace/caching.go +++ b/internal/namespace/caching.go @@ -75,6 +75,7 @@ func (nsc *cachingManager) ReadNamespace(ctx context.Context, nsName string, rev nsRevisionKey := cacheKey(nsName, revision) value, found := nsc.c.Get(nsRevisionKey) if found { + // Check the cache. return value.(*v0.NamespaceDefinition), nil } diff --git a/internal/namespace/manager.go b/internal/namespace/manager.go index b9e9d12227..235144abda 100644 --- a/internal/namespace/manager.go +++ b/internal/namespace/manager.go @@ -29,7 +29,7 @@ type Manager interface { // ReadNamespaceAndTypes reads a namespace definition, version, and type system and returns it if found. ReadNamespaceAndTypes(ctx context.Context, nsName string, revision decimal.Decimal) (*v0.NamespaceDefinition, *NamespaceTypeSystem, error) - // Closes the namespace manager, disposing of any resources. + // Close closes the namespace manager, disposing of any resources. // // NOTE: Should *not* call Close on the datastore. Close() error