Merge pull request #85 from grafana/fix-memberlist-corrupted-packets

Fix memberlist corrupted packets

CHANGELOG.md

@@ -20,3 +20,4 @@
 * [ENHANCEMENT] Add runutil.CloseWithLogOnErr function. #58
 * [ENHANCEMENT] Optimise memberlist receive path when used as a backing store for rings with a large number of members. #76 #77
 * [BUGFIX] spanlogger: Support multiple tenant IDs. #59
+* [BUGFIX] Memberlist: fixed corrupted packets when sending compound messages with more than 255 messages or messages bigger than 64KB. #85

go.mod

@@ -38,4 +38,6 @@ require (
 
 replace k8s.io/client-go v12.0.0+incompatible => k8s.io/client-go v0.21.4
 
-replace github.com/hashicorp/memberlist v0.2.3 => github.com/grafana/memberlist v0.2.3
+// Replace memberlist with our fork which includes some fixes that haven't been
+// merged upstream yet.
+replace github.com/hashicorp/memberlist v0.2.3 => github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b

go.sum

@@ -157,8 +157,8 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/grafana/memberlist v0.2.3 h1:jFS903O5XIPOKNJdVSFM+LY3pfqk1LX53KkPW81V+7A=
-github.com/grafana/memberlist v0.2.3/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
+github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b h1:UlCBLaqvS4wVYNrMKSfqTBVsed/EOY9dnenhYZMUnrA=
+github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg=
 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE=

kv/memberlist/memberlist_client.go

@@ -902,18 +902,6 @@ func (m *KV) broadcastNewValue(key string, change Mergeable, version uint, codec
 		return
 	}
 
-	if len(pairData) > 65535 {
-		// Unfortunately, memberlist will happily let us send bigger messages via gossip,
-		// but then it will fail to parse them properly, because its own size field is 2-bytes only.
-		// (github.com/hashicorp/memberlist@v0.1.4/util.go:167, makeCompoundMessage function)
-		//
-		// Typically messages are smaller (when dealing with couple of updates only), but can get bigger
-		// when broadcasting result of push/pull update.
-		level.Debug(m.logger).Log("msg", "broadcast message too big, not broadcasting", "key", key, "version", version, "len", len(pairData))
-		m.numberOfBroadcastMessagesDropped.Inc()
-		return
-	}
-
 	m.addSentMessage(message{
 		Time:    time.Now(),
 		Size:    len(pairData),