feat: daemon: automatically set GOMEMLIMIT if it is unset

I have a rather big collection of profiles where someone claims that Kubo is ooming on XGiB.
Then you open the profile and it is using half of that, this is due to the default GOGC=200%.
That means, go will only run the GC once it's twice as being as the previous alive set.

This situation happen more than it should / almost always because many parts of Kubo are memory garbage factories.

Adding a GOMEMLIMIT helps by trading off more and more CPU running GC more often when memory is about to run out,
it's not healthy to run at the edge of the limit because the GC will continously run killing performance.
So this doesn't double the effective memory usable by Kubo, but we should expect to be able to use ~1.5x~1.75x before performance drastically falling off.

Closes: ipfs#8798

cmd/ipfs/daemon.go

@@ -4,17 +4,23 @@ import (
 	"errors"
 	_ "expvar"
 	"fmt"
+	"math"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"runtime"
+	"runtime/debug"
 	"sort"
 	"sync"
 	"time"
 
 	multierror "github.com/hashicorp/go-multierror"
 
+	"github.com/dustin/go-humanize"
+	options "github.com/ipfs/boxo/coreiface/options"
+	cmds "github.com/ipfs/go-ipfs-cmds"
+	mprome "github.com/ipfs/go-metrics-prometheus"
 	version "github.com/ipfs/kubo"
 	utilmain "github.com/ipfs/kubo/cmd/ipfs/util"
 	oldcmds "github.com/ipfs/kubo/commands"
@@ -30,16 +36,13 @@ import (
 	fsrepo "github.com/ipfs/kubo/repo/fsrepo"
 	"github.com/ipfs/kubo/repo/fsrepo/migrations"
 	"github.com/ipfs/kubo/repo/fsrepo/migrations/ipfsfetcher"
+	goprocess "github.com/jbenet/goprocess"
 	p2pcrypto "github.com/libp2p/go-libp2p/core/crypto"
 	pnet "github.com/libp2p/go-libp2p/core/pnet"
 	sockets "github.com/libp2p/go-socket-activation"
-
-	options "github.com/ipfs/boxo/coreiface/options"
-	cmds "github.com/ipfs/go-ipfs-cmds"
-	mprome "github.com/ipfs/go-metrics-prometheus"
-	goprocess "github.com/jbenet/goprocess"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
+	"github.com/pbnjay/memory"
 	prometheus "github.com/prometheus/client_golang/prometheus"
 	promauto "github.com/prometheus/client_golang/prometheus/promauto"
 )
@@ -197,6 +200,42 @@ func defaultMux(path string) corehttp.ServeOption {
 	}
 }
 
+// setMemoryLimit a soft memory limit to enforce running the GC more often when
+// we are about to run out.
+// This allows to recoop memory when it's about to run out and cancel the
+// doubled memory footprint most go programs experience, at the cost of more CPU
+// usage in memory tight conditions. This does not increase CPU usage when memory
+// is plenty available, it will use more CPU and continue to run in cases where Kubo
+// would OOM.
+func setMemoryLimit() {
+	// From the STD documentation:
+	// A negative input does not adjust the limit, and allows for retrieval of the currently set memory limit.
+	if currentMemoryLimit := debug.SetMemoryLimit(-1); currentMemoryLimit != math.MaxInt64 {
+		fmt.Printf("GOMEMLIMIT already set to %s, leaving as-is.\n", humanize.IBytes(uint64(currentMemoryLimit)))
+		// only update the memory limit if it wasn't set with GOMEMLIMIT already
+		return
+	}
+
+	// this is a proportional negative-rate increase curve fitted to thoses points:
+	// 0GiB   -> 0GiB
+	// 4GiB   -> 0.5GiB
+	// 6GiB   -> 0.75GiB
+	// 12GiB  -> 1GiB
+	// 256GiB -> 2GiB
+	totalMemory := memory.TotalMemory()
+	memoryMargin := int64(213865e4 - 209281e4*math.Pow(math.E, -588918e-16*float64(totalMemory)))
+	// if memory is extremely small this approximation / is useless
+	if memoryMargin <= 0 {
+		// then don't bother setting a limit and rely on GOGC
+		fmt.Println("TotalMemory is too tight, continuing without GOMEMLIMIT.")
+		return
+	}
+
+	remainingMemory := totalMemory - uint64(memoryMargin)
+	debug.SetMemoryLimit(int64(remainingMemory))
+	fmt.Printf("Set GOMEMLIMIT to %s.\n", humanize.IBytes(remainingMemory))
+}
+
 func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment) (_err error) {
 	// Inject metrics before we do anything
 	err := mprome.Inject()
@@ -219,6 +258,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 	// print the ipfs version
 	printVersion()
 
+	setMemoryLimit()
+
 	managefd, _ := req.Options[adjustFDLimitKwd].(bool)
 	if managefd {
 		if _, _, err := utilmain.ManageFdLimit(); err != nil {

test/sharness/t0060-daemon.sh

@@ -86,7 +86,9 @@ test_expect_success "ipfs daemon output looks good" '
   echo "WebUI: http://'$API_ADDR'/webui" >>expected_daemon &&
   echo "Gateway server listening on '$GWAY_MADDR'" >>expected_daemon &&
   echo "Daemon is ready" >>expected_daemon &&
-  test_cmp expected_daemon actual_daemon
+  grep -q "^Set GOMEMLIMIT to" actual_daemon &&
+  grep -v "^Set GOMEMLIMIT to" actual_daemon > actual_daemon_filtered &&
+  test_cmp expected_daemon actual_daemon_filtered
 '
 
 test_expect_success ".ipfs/ has been created" '