Integrate the latest index-provider with config to disable announcements

Integrate the latest `index-provider` and reflect the changes to engine
configuration. Note that this commit disables announcements of indices
on the network by default as requested for initial merge to master.

Introduce dedicated index provider configuration parameters with
documentation and defaults that match the defaults in index-provider.

Re-generate code as needed.

documentation/en/default-lotus-miner-config.toml

@@ -266,24 +266,46 @@
 
 
 [IndexProvider]
-  # env var: LOTUS_INDEXPROVIDER_LINKCACHESIZE
-  #LinkCacheSize = 1024
-
-  # env var: LOTUS_INDEXPROVIDER_LINKEDCHUNKSIZE
-  #LinkedChunkSize = 16384
+  # Enable set whether to enable indexing announcement to the network and expose endpoints that
+  # allow indexer nodes to process announcements. Disabled by default.
+  #
+  # type: bool
+  # env var: LOTUS_INDEXPROVIDER_ENABLE
+  #Enable = false
 
-  # env var: LOTUS_INDEXPROVIDER_PUBSUBTOPIC
-  #PubSubTopic = "/indexer/ingest/mainnet"
+  # EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+  # entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The
+  # maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize and
+  # the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+  # with the default EntriesCacheCapacity, and EntriesChunkSize means the cache size can grow to
+  # 256MiB when full.
+  #
+  # type: int
+  # env var: LOTUS_INDEXPROVIDER_ENTRIESCACHECAPACITY
+  #EntriesCacheCapacity = 1024
 
-  # env var: LOTUS_INDEXPROVIDER_PURGELINKCACHE
-  #PurgeLinkCache = false
+  # EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk.
+  # Defaults to 16384 if not specified. Note that chunks are chained together for indexing
+  # advertisements that include more multihashes than the configured EntriesChunkSize.
+  #
+  # type: int
+  # env var: LOTUS_INDEXPROVIDER_ENTRIESCHUNKSIZE
+  #EntriesChunkSize = 16384
 
-  # env var: LOTUS_INDEXPROVIDER_PUBLISHERKIND
-  #PublisherKind = "dtsync"
+  # TopicName sets the topic name on which the changes to the advertised content are announced.
+  # Defaults to '/indexer/ingest/mainnet' if not specified.
+  #
+  # type: string
+  # env var: LOTUS_INDEXPROVIDER_TOPICNAME
+  #TopicName = "/indexer/ingest/mainnet"
 
-  [IndexProvider.HttpPublisher]
-    # env var: LOTUS_INDEXPROVIDER_HTTPPUBLISHER_LISTENMULTIADDR
-    #ListenMultiaddr = "/ip4/0.0.0.0/tcp/3104/http"
+  # PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine
+  # starts. By default, the cache is rehydrated from previously cached entries stored in
+  # datastore if any is present.
+  #
+  # type: bool
+  # env var: LOTUS_INDEXPROVIDER_PURGECACHEONSTART
+  #PurgeCacheOnStart = false
 
 
 [Sealing]

go.mod

@@ -40,14 +40,14 @@ require (
 	github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
 	github.com/filecoin-project/go-fil-markets v1.20.0
 	github.com/filecoin-project/go-jsonrpc v0.1.5
-	github.com/filecoin-project/go-legs v0.3.4
+	github.com/filecoin-project/go-legs v0.3.7
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.4
 	github.com/filecoin-project/go-state-types v0.1.3
 	github.com/filecoin-project/go-statemachine v1.0.1
 	github.com/filecoin-project/go-statestore v0.2.0
 	github.com/filecoin-project/go-storedcounter v0.1.0
-	github.com/filecoin-project/index-provider v0.3.1
+	github.com/filecoin-project/index-provider v0.4.0
 	github.com/filecoin-project/specs-actors v0.9.14
 	github.com/filecoin-project/specs-actors/v2 v2.3.6
 	github.com/filecoin-project/specs-actors/v3 v3.1.1
@@ -150,7 +150,7 @@ require (
 	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
 	github.com/urfave/cli/v2 v2.3.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
-	github.com/whyrusleeping/cbor-gen v0.0.0-20211110122933-f57984553008
+	github.com/whyrusleeping/cbor-gen v0.0.0-20220224212727-7a699437a831
 	github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	github.com/whyrusleeping/pubsub v0.0.0-20190708150250-92bcb0691325

go.sum

@@ -348,8 +348,9 @@ github.com/filecoin-project/go-indexer-core v0.2.8/go.mod h1:IagNfTdFuX4057kla43
 github.com/filecoin-project/go-jsonrpc v0.1.5 h1:ckxqZ09ivBAVf5CSmxxrqqNHC7PJm3GYGtYKiNQ+vGk=
 github.com/filecoin-project/go-jsonrpc v0.1.5/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
 github.com/filecoin-project/go-legs v0.3.0/go.mod h1:x6nwM+DuN7NzlPndOoJuiHYCX+pze6+efPRx17nIA7M=
-github.com/filecoin-project/go-legs v0.3.4 h1:yid2IivTJ8JeF1ROA8jxNKNKJshU/HO1sRo5lEMNOUc=
 github.com/filecoin-project/go-legs v0.3.4/go.mod h1:P7ZPHqFG96OFaT11rnXyTdcxRdGkaH1dRyPxJSOiSuM=
+github.com/filecoin-project/go-legs v0.3.7 h1:yfm7fx+iy1nPtgPEQ6kQjvhoJOVbXide50STYdy+yos=
+github.com/filecoin-project/go-legs v0.3.7/go.mod h1:pgekGm8/gKY5zCtQ/qGAoSjGP92wTLFqpO3GPHeu8YU=
 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak=
 github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs=
 github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ=
@@ -371,8 +372,9 @@ github.com/filecoin-project/go-statestore v0.2.0 h1:cRRO0aPLrxKQCZ2UOQbzFGn4WDNd
 github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo=
 github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5zQkWJM/CpAZa/uOuus=
 github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8=
-github.com/filecoin-project/index-provider v0.3.1 h1:ySGk2FnDbUEhq6QwakeU/p8tKnIoUm6JIpXPQ8RG1iM=
 github.com/filecoin-project/index-provider v0.3.1/go.mod h1:o8ineTLWnwtfBv7SnELpncza6SBU295esxOaXsTCl2I=
+github.com/filecoin-project/index-provider v0.4.0 h1:laCdWms1OvgHwxkU0DSfkzqPQBRAMPO2TKes5s0TdHo=
+github.com/filecoin-project/index-provider v0.4.0/go.mod h1:SDfBxhD9/wet7e5wv1J4IZjWfVkln/k4oTzMcNccLZA=
 github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
 github.com/filecoin-project/specs-actors v0.9.14 h1:68PVstg2UB3ZsMLF+DKFTAs/YKsqhKWynkr0IqmVRQY=
 github.com/filecoin-project/specs-actors v0.9.14/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao=
@@ -396,8 +398,9 @@ github.com/filecoin-project/specs-actors/v7 v7.0.0 h1:FQN7tjt3o68hfb3qLFSJBoLMuO
 github.com/filecoin-project/specs-actors/v7 v7.0.0/go.mod h1:TA5FwCna+Yi36POaT7SLKXsgEDvJwc0V/L6ZsO19B9M=
 github.com/filecoin-project/specs-storage v0.2.0 h1:Y4UDv0apRQ3zI2GiPPubi8JblpUZZphEdaJUxCutfyg=
 github.com/filecoin-project/specs-storage v0.2.0/go.mod h1:Tb88Zq+IBJbvAn3mS89GYj3jdRThBTE/771HCVZdRJU=
-github.com/filecoin-project/storetheindex v0.3.2 h1:KsTJer5Di/iapSrOWtbFTPMt5kS7Y3n3C9kOYH93gEA=
 github.com/filecoin-project/storetheindex v0.3.2/go.mod h1:d7y0FHAwPPSIdg+Oh7xV5B9TLebL7rvoKu591N/l2ZU=
+github.com/filecoin-project/storetheindex v0.3.5 h1:KoS9TvjPm6zIZfUH8atAHJbVHOO7GTP1MdTG+v0eE+Q=
+github.com/filecoin-project/storetheindex v0.3.5/go.mod h1:0r3d0kSpK63O6AvLr1CjAINLi+nWD49clzcnKV+GLpI=
 github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg=
 github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
@@ -1056,6 +1059,7 @@ github.com/libp2p/go-libp2p v0.14.4/go.mod h1:EIRU0Of4J5S8rkockZM7eJp2S0UrCyi55m
 github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76fyMX364/O4=
 github.com/libp2p/go-libp2p v0.17.0/go.mod h1:Fkin50rsGdv5mm5BshBUtPRZknt9esfmYXBOYcwOTgw=
 github.com/libp2p/go-libp2p v0.18.0-rc1/go.mod h1:RgYlH7IIWHXREimC92bw5Lg1V2R5XmSzuLHb5fTnr+8=
+github.com/libp2p/go-libp2p v0.18.0-rc3/go.mod h1:WYL+Xw1iuwi6rdfzw5VIEpD+HqzYucHZ6fcUuumbI3M=
 github.com/libp2p/go-libp2p v0.18.0-rc5 h1:88wWDHb9nNo0vBNCupLde3OTnFAkugOCNkrDfl3ivK4=
 github.com/libp2p/go-libp2p v0.18.0-rc5/go.mod h1:aZPS5l84bDvCvP4jkyEUT/J6YOpUq33Fgqrs3K59mpI=
 github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo=
@@ -1253,6 +1257,7 @@ github.com/libp2p/go-libp2p-swarm v0.5.3/go.mod h1:NBn7eNW2lu568L7Ns9wdFrOhgRlkR
 github.com/libp2p/go-libp2p-swarm v0.8.0/go.mod h1:sOMp6dPuqco0r0GHTzfVheVBh6UEL0L1lXUZ5ot2Fvc=
 github.com/libp2p/go-libp2p-swarm v0.9.0/go.mod h1:2f8d8uxTJmpeqHF/1ujjdXZp+98nNIbujVOMEZxCbZ8=
 github.com/libp2p/go-libp2p-swarm v0.10.0/go.mod h1:71ceMcV6Rg/0rIQ97rsZWMzto1l9LnNquef+efcRbmA=
+github.com/libp2p/go-libp2p-swarm v0.10.1/go.mod h1:Pdkq0QU5a+qu+oyqIV3bknMsnzk9lnNyKvB9acJ5aZs=
 github.com/libp2p/go-libp2p-swarm v0.10.2 h1:UaXf+CTq6Ns1N2V1EgqJ9Q3xaRsiN7ImVlDMpirMAWw=
 github.com/libp2p/go-libp2p-swarm v0.10.2/go.mod h1:Pdkq0QU5a+qu+oyqIV3bknMsnzk9lnNyKvB9acJ5aZs=
 github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
@@ -1928,8 +1933,9 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:f
 github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
-github.com/whyrusleeping/cbor-gen v0.0.0-20211110122933-f57984553008 h1:7WtW9D9VGpmRLuQmrPy2JobUNdka95z3MKEVpELtOjo=
 github.com/whyrusleeping/cbor-gen v0.0.0-20211110122933-f57984553008/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
+github.com/whyrusleeping/cbor-gen v0.0.0-20220224212727-7a699437a831 h1:9blPRrm7ebDqDm6nHXXzCSru+sp20gcA3CLu+G/kHUc=
+github.com/whyrusleeping/cbor-gen v0.0.0-20220224212727-7a699437a831/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-ctrlnet v0.0.0-20180313164037-f564fbbdaa95/go.mod h1:SJqKCCPXRfBFCwXjfNT/skfsceF7+MBFLI2OrvuRA7g=
@@ -2040,8 +2046,9 @@ go.uber.org/fx v1.15.0/go.mod h1:jI3RazQUhGv5KkpZIRv+kuP4CcgX3fnc0qX8bLnzbx8=
 go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=

node/config/def.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
-	ipconfig "github.com/filecoin-project/index-provider/config"
 	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
 
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
@@ -185,7 +184,11 @@ func DefaultStorageMiner() *StorageMiner {
 		},
 
 		IndexProvider: IndexProviderConfig{
-			Ingest: ipconfig.NewIngest(),
+			Enable:               false,
+			EntriesCacheCapacity: 1024,
+			EntriesChunkSize:     16384,
+			TopicName:            "/indexer/ingest/mainnet",
+			PurgeCacheOnStart:    false,
 		},
 
 		Subsystems: MinerSubsystemConfig{

node/config/types.go

@@ -3,8 +3,6 @@ package config
 import (
 	"github.com/ipfs/go-cid"
 
-	"github.com/filecoin-project/index-provider/config"
-
 	"github.com/filecoin-project/lotus/chain/types"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 )
@@ -166,7 +164,32 @@ type DealmakingConfig struct {
 }
 
 type IndexProviderConfig struct {
-	config.Ingest
+
+	// Enable set whether to enable indexing announcement to the network and expose endpoints that
+	// allow indexer nodes to process announcements. Disabled by default.
+	Enable bool
+
+	// EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+	// entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The
+	// maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize and
+	// the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+	// with the default EntriesCacheCapacity, and EntriesChunkSize means the cache size can grow to
+	// 256MiB when full.
+	EntriesCacheCapacity int
+
+	// EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk.
+	// Defaults to 16384 if not specified. Note that chunks are chained together for indexing
+	// advertisements that include more multihashes than the configured EntriesChunkSize.
+	EntriesChunkSize int
+
+	// TopicName sets the topic name on which the changes to the advertised content are announced.
+	// Defaults to '/indexer/ingest/mainnet' if not specified.
+	TopicName string
+
+	// PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine
+	// starts. By default, the cache is rehydrated from previously cached entries stored in
+	// datastore if any is present.
+	PurgeCacheOnStart bool
 }
 
 type RetrievalPricing struct {

node/modules/storageminer_idxprov.go

@@ -2,10 +2,7 @@ package modules
 
 import (
 	"context"
-
 	"github.com/ipfs/go-datastore"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/libp2p/go-libp2p-core/peerstore"
 	"go.uber.org/fx"
 
 	"github.com/filecoin-project/go-address"
@@ -24,33 +21,38 @@ type IdxProv struct {
 
 	fx.Lifecycle
 	Datastore dtypes.MetadataDS
-	PeerID    peer.ID
-	peerstore.Peerstore
 }
 
 func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHost host.Host, dt dtypes.ProviderDataTransfer, maddr dtypes.MinerAddress) (provider.Interface, error) {
 	return func(args IdxProv, marketHost host.Host, dt dtypes.ProviderDataTransfer, maddr dtypes.MinerAddress) (provider.Interface, error) {
 		ipds := namespace.Wrap(args.Datastore, datastore.NewKey("/index-provider"))
-
-		pkey := args.Peerstore.PrivKey(args.PeerID)
-		if pkey == nil {
-			return nil, xerrors.Errorf("missing private key for node ID: %s", args.PeerID)
+		var opts = []engine.Option{
+			engine.WithDatastore(ipds),
+			engine.WithHost(marketHost),
+			engine.WithRetrievalAddrs(marketHost.Addrs()...),
 		}
 
-		var retAdds []string
-		for _, a := range marketHost.Addrs() {
-			retAdds = append(retAdds, a.String())
+		llog := log.With("idxProvEnabled", cfg.Enable, "pid", marketHost.ID(), "retAddrs", marketHost.Addrs())
+		// If announcements to the network are enabled, then set options for datatransfer publisher.
+		if cfg.Enable {
+			// Get the miner ID and set as extra gossip data.
+			// The extra data is required by the lotus-specific index-provider gossip message validators.
+			ma := address.Address(maddr)
+			opts = append(opts,
+				engine.WithPublisherKind(engine.DataTransferPublisher),
+				engine.WithDataTransfer(dt),
+				engine.WithExtraGossipData(ma.Bytes()))
+			llog = llog.With("extraGossipData", ma)
+		} else {
+			opts = append(opts, engine.WithPublisherKind(engine.NoPublisher))
 		}
 
-		// Get the miner ID and set as extra gossip data.
-		// The extra data is required by the lotus-specific index-provider gossip message validators.
-		ma := address.Address(maddr)
-		log.Infof("Using extra gossip data in index provider engine: %s", ma)
-
-		e, err := engine.New(cfg.Ingest, pkey, dt, marketHost, ipds, retAdds, engine.WithExtraGossipData(ma.Bytes()))
+		// Instantiate the index provider engine.
+		e, err := engine.New(opts...)
 		if err != nil {
 			return nil, xerrors.Errorf("creating indexer provider engine: %w", err)
 		}
+		llog.Info("Instantiated index provider engine")
 
 		args.Lifecycle.Append(fx.Hook{
 			OnStart: func(ctx context.Context) error {
@@ -60,6 +62,7 @@ func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHo
 				if err := e.Start(ctx); err != nil {
 					return xerrors.Errorf("starting indexer provider engine: %w", err)
 				}
+				log.Infof("Started index provider engine")
 				return nil
 			},
 			OnStop: func(_ context.Context) error {