From a862d91790813f186821ae5d6f80202e934c3609 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 27 May 2019 14:35:05 -0700 Subject: [PATCH 1/2] feat: improve protocol interning This used to specify a max size/count but that was only because this logic was shared between all peerstores (including the on-disk one). Now that: 1. This only applies to the in-memory one. 2. We never actually GC these. We can just intern indefinitely. --- p2p/host/peerstore/pstoremem/protobook.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/p2p/host/peerstore/pstoremem/protobook.go b/p2p/host/peerstore/pstoremem/protobook.go index 04cd14591c..3a5550d2b6 100644 --- a/p2p/host/peerstore/pstoremem/protobook.go +++ b/p2p/host/peerstore/pstoremem/protobook.go @@ -8,11 +8,6 @@ import ( pstore "github.com/libp2p/go-libp2p-peerstore" ) -const ( - maxInternedProtocols = 512 - maxInternedProtocolSize = 256 -) - type protoSegment struct { sync.RWMutex protocols map[peer.ID]map[string]struct{} @@ -35,7 +30,7 @@ var _ pstore.ProtoBook = (*memoryProtoBook)(nil) func NewProtoBook() pstore.ProtoBook { return &memoryProtoBook{ - interned: make(map[string]string, maxInternedProtocols), + interned: make(map[string]string, 256), segments: func() (ret protoSegments) { for i := range ret { ret[i] = &protoSegment{ @@ -48,10 +43,6 @@ func NewProtoBook() pstore.ProtoBook { } func (pb *memoryProtoBook) internProtocol(proto string) string { - if len(proto) > maxInternedProtocolSize { - return proto - } - // check if it is interned with the read lock pb.lk.RLock() interned, ok := pb.interned[proto] @@ -71,11 +62,6 @@ func (pb *memoryProtoBook) internProtocol(proto string) string { return interned } - // if we've filled the table, throw it away and start over - if len(pb.interned) >= maxInternedProtocols { - pb.interned = make(map[string]string, maxInternedProtocols) - } - pb.interned[proto] = proto return proto } From 5197d8f344f14ead84c85b4ffc3a6c4bb7d520af Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 27 May 2019 14:41:29 -0700 Subject: [PATCH 2/2] feat: intern agent versions This also starts taking up a bunch of memory after a while. --- p2p/host/peerstore/pstoremem/metadata.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/p2p/host/peerstore/pstoremem/metadata.go b/p2p/host/peerstore/pstoremem/metadata.go index 162bee5087..9adebd8ae9 100644 --- a/p2p/host/peerstore/pstoremem/metadata.go +++ b/p2p/host/peerstore/pstoremem/metadata.go @@ -7,6 +7,11 @@ import ( pstore "github.com/libp2p/go-libp2p-peerstore" ) +var internKeys = map[string]bool{ + "AgentVersion": true, + "ProtocolVersion": true, +} + type metakey struct { id peer.ID key string @@ -15,8 +20,9 @@ type metakey struct { type memoryPeerMetadata struct { // store other data, like versions //ds ds.ThreadSafeDatastore - ds map[metakey]interface{} - dslock sync.RWMutex + ds map[metakey]interface{} + dslock sync.RWMutex + interned map[string]interface{} } var _ pstore.PeerMetadata = (*memoryPeerMetadata)(nil) @@ -30,6 +36,13 @@ func NewPeerMetadata() pstore.PeerMetadata { func (ps *memoryPeerMetadata) Put(p peer.ID, key string, val interface{}) error { ps.dslock.Lock() defer ps.dslock.Unlock() + if vals, ok := val.(string); ok && internKeys[key] { + if interned, ok := ps.interned[vals]; ok { + val = interned + } else { + ps.interned[vals] = val + } + } ps.ds[metakey{p, key}] = val return nil }