Merge pull request #6345 from ellemouton/bitcoind-rpc-polling

multi: bitcoind rpc polling option

.github/workflows/main.yml

@@ -259,6 +259,8 @@ jobs:
             args: backend=bitcoind
           - name: bitcoind-notxindex
             args: backend="bitcoind notxindex"
+          - name: bitcoind-rpcpolling
+            args: backend="bitcoind rpcpolling"
           - name: bitcoind-etcd
             args: backend=bitcoind dbbackend=etcd
           - name: bitcoind-postgres

chainntnfs/bitcoindnotify/bitcoind.go

@@ -19,9 +19,14 @@ import (
 )
 
 const (
-	// notifierType uniquely identifies this concrete implementation of the
-	// ChainNotifier interface.
-	notifierType = "bitcoind"
+	// notifierType uniquely identifies a concrete implementation of the
+	// ChainNotifier interface that makes use of the bitcoind ZMQ interface.
+	notifierTypeZMQ = "bitcoind"
+
+	// notifierTypeRPCPolling uniquely identifies a concrete implementation
+	// of the ChainNotifier interface that makes use of the bitcoind RPC
+	// interface.
+	notifierTypeRPCPolling = "bitcoind-rpc-polling"
 )
 
 // TODO(roasbeef): generalize struct below:

chainntnfs/bitcoindnotify/bitcoind_test.go

@@ -108,13 +108,18 @@ func syncNotifierWithMiner(t *testing.T, notifier *BitcoindNotifier,
 // TestHistoricalConfDetailsTxIndex ensures that we correctly retrieve
 // historical confirmation details using the backend node's txindex.
 func TestHistoricalConfDetailsTxIndex(t *testing.T) {
+	testHistoricalConfDetailsTxIndex(t, true)
+	testHistoricalConfDetailsTxIndex(t, false)
+}
+
+func testHistoricalConfDetailsTxIndex(t *testing.T, rpcPolling bool) {
 	miner, tearDown := chainntnfs.NewMiner(
 		t, []string{"--txindex"}, true, 25,
 	)
 	defer tearDown()
 
 	bitcoindConn, cleanUp := chainntnfs.NewBitcoindBackend(
-		t, miner.P2PAddress(), true,
+		t, miner.P2PAddress(), true, rpcPolling,
 	)
 	defer cleanUp()
 
@@ -206,11 +211,16 @@ func TestHistoricalConfDetailsTxIndex(t *testing.T) {
 // historical confirmation details using the set of fallback methods when the
 // backend node's txindex is disabled.
 func TestHistoricalConfDetailsNoTxIndex(t *testing.T) {
+	testHistoricalConfDetailsNoTxIndex(t, true)
+	testHistoricalConfDetailsNoTxIndex(t, false)
+}
+
+func testHistoricalConfDetailsNoTxIndex(t *testing.T, rpcpolling bool) {
 	miner, tearDown := chainntnfs.NewMiner(t, nil, true, 25)
 	defer tearDown()
 
 	bitcoindConn, cleanUp := chainntnfs.NewBitcoindBackend(
-		t, miner.P2PAddress(), false,
+		t, miner.P2PAddress(), false, rpcpolling,
 	)
 	defer cleanUp()
 

chainntnfs/bitcoindnotify/driver.go

@@ -56,13 +56,21 @@ func createNewNotifier(args ...interface{}) (chainntnfs.ChainNotifier, error) {
 // chainntnfs.ChainNotifier interface.
 func init() {
 	// Register the driver.
-	notifier := &chainntnfs.NotifierDriver{
-		NotifierType: notifierType,
+	notifierZMQ := &chainntnfs.NotifierDriver{
+		NotifierType: notifierTypeZMQ,
 		New:          createNewNotifier,
 	}
+	if err := chainntnfs.RegisterNotifier(notifierZMQ); err != nil {
+		panic(fmt.Sprintf("failed to register notifier driver '%s': %v",
+			notifierTypeZMQ, err))
+	}
 
-	if err := chainntnfs.RegisterNotifier(notifier); err != nil {
+	notifierRPC := &chainntnfs.NotifierDriver{
+		NotifierType: notifierTypeRPCPolling,
+		New:          createNewNotifier,
+	}
+	if err := chainntnfs.RegisterNotifier(notifierRPC); err != nil {
 		panic(fmt.Sprintf("failed to register notifier driver '%s': %v",
-			notifierType, err))
+			notifierTypeRPCPolling, err))
 	}
 }

chainntnfs/test/bitcoind/bitcoind_test.go

@@ -13,4 +13,5 @@ import (
 // powered chain notifier.
 func TestInterfaces(t *testing.T) {
 	chainntnfstest.TestInterfaces(t, "bitcoind")
+	chainntnfstest.TestInterfaces(t, "bitcoind-rpc-polling")
 }

chainntnfs/test/test_interface.go

@@ -1966,7 +1966,19 @@ func TestInterfaces(t *testing.T, targetBackEnd string) {
 		case "bitcoind":
 			var bitcoindConn *chain.BitcoindConn
 			bitcoindConn, cleanUp = chainntnfs.NewBitcoindBackend(
-				t, p2pAddr, true,
+				t, p2pAddr, true, false,
+			)
+			newNotifier = func() (chainntnfs.TestChainNotifier, error) {
+				return bitcoindnotify.New(
+					bitcoindConn, chainntnfs.NetParams,
+					hintCache, hintCache, blockCache,
+				), nil
+			}
+
+		case "bitcoind-rpc-polling":
+			var bitcoindConn *chain.BitcoindConn
+			bitcoindConn, cleanUp = chainntnfs.NewBitcoindBackend(
+				t, p2pAddr, true, true,
 			)
 			newNotifier = func() (chainntnfs.TestChainNotifier, error) {
 				return bitcoindnotify.New(

chainntnfs/test_utils.go

@@ -26,6 +26,7 @@ import (
 	"github.com/btcsuite/btcwallet/walletdb"
 	"github.com/lightninglabs/neutrino"
 	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/lntest/wait"
 )
 
 var (
@@ -193,10 +194,12 @@ func NewMiner(t *testing.T, extraArgs []string, createChain bool,
 
 // NewBitcoindBackend spawns a new bitcoind node that connects to a miner at the
 // specified address. The txindex boolean can be set to determine whether the
-// backend node should maintain a transaction index. A connection to the newly
-// spawned bitcoind node is returned.
-func NewBitcoindBackend(t *testing.T, minerAddr string,
-	txindex bool) (*chain.BitcoindConn, func()) {
+// backend node should maintain a transaction index. The rpcpolling boolean
+// can be set to determine whether bitcoind's RPC polling interface should be
+// used for block and tx notifications or if its ZMQ interface should be used.
+// A connection to the newly spawned bitcoind node is returned.
+func NewBitcoindBackend(t *testing.T, minerAddr string, txindex,
+	rpcpolling bool) (*chain.BitcoindConn, func()) {
 
 	t.Helper()
 
@@ -231,29 +234,41 @@ func NewBitcoindBackend(t *testing.T, minerAddr string,
 	}
 
 	// Wait for the bitcoind instance to start up.
-	time.Sleep(time.Second)
-
 	host := fmt.Sprintf("127.0.0.1:%d", rpcPort)
-	conn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{
-		ChainParams:     NetParams,
-		Host:            host,
-		User:            "weks",
-		Pass:            "weks",
-		ZMQBlockHost:    zmqBlockHost,
-		ZMQTxHost:       zmqTxHost,
-		ZMQReadDeadline: 5 * time.Second,
+	cfg := &chain.BitcoindConfig{
+		ChainParams: NetParams,
+		Host:        host,
+		User:        "weks",
+		Pass:        "weks",
 		// Fields only required for pruned nodes, not needed for these
 		// tests.
 		Dialer:             nil,
 		PrunedModeMaxPeers: 0,
-	})
-	if err != nil {
-		bitcoind.Process.Kill()
-		bitcoind.Wait()
-		os.RemoveAll(tempBitcoindDir)
-		t.Fatalf("unable to establish connection to bitcoind: %v", err)
 	}
-	if err := conn.Start(); err != nil {
+
+	if rpcpolling {
+		cfg.ZMQConfig = &chain.ZMQConfig{
+			ZMQBlockHost:    zmqBlockHost,
+			ZMQTxHost:       zmqTxHost,
+			ZMQReadDeadline: 5 * time.Second,
+		}
+	} else {
+		cfg.PollingConfig = &chain.PollingConfig{
+			BlockPollingInterval: time.Millisecond * 20,
+			TxPollingInterval:    time.Millisecond * 20,
+		}
+	}
+
+	var conn *chain.BitcoindConn
+	err = wait.NoError(func() error {
+		conn, err = chain.NewBitcoindConn(cfg)
+		if err != nil {
+			return err
+		}
+
+		return conn.Start()
+	}, 10*time.Second)
+	if err != nil {
 		bitcoind.Process.Kill()
 		bitcoind.Wait()
 		os.RemoveAll(tempBitcoindDir)

chainreg/chainregistry.go

@@ -399,19 +399,31 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) {
 			}
 		}
 
-		// Establish the connection to bitcoind and create the clients
-		// required for our relevant subsystems.
-		bitcoindConn, err := chain.NewBitcoindConn(&chain.BitcoindConfig{
+		bitcoindCfg := &chain.BitcoindConfig{
 			ChainParams:        cfg.ActiveNetParams.Params,
 			Host:               bitcoindHost,
 			User:               bitcoindMode.RPCUser,
 			Pass:               bitcoindMode.RPCPass,
-			ZMQBlockHost:       bitcoindMode.ZMQPubRawBlock,
-			ZMQTxHost:          bitcoindMode.ZMQPubRawTx,
-			ZMQReadDeadline:    5 * time.Second,
 			Dialer:             cfg.Dialer,
 			PrunedModeMaxPeers: bitcoindMode.PrunedNodeMaxPeers,
-		})
+		}
+
+		if bitcoindMode.RPCPolling {
+			bitcoindCfg.PollingConfig = &chain.PollingConfig{
+				BlockPollingInterval: bitcoindMode.BlockPollingInterval,
+				TxPollingInterval:    bitcoindMode.TxPollingInterval,
+			}
+		} else {
+			bitcoindCfg.ZMQConfig = &chain.ZMQConfig{
+				ZMQBlockHost:    bitcoindMode.ZMQPubRawBlock,
+				ZMQTxHost:       bitcoindMode.ZMQPubRawTx,
+				ZMQReadDeadline: bitcoindMode.ZMQReadDeadline,
+			}
+		}
+
+		// Establish the connection to bitcoind and create the clients
+		// required for our relevant subsystems.
+		bitcoindConn, err := chain.NewBitcoindConn(bitcoindCfg)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -495,7 +507,7 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) {
 		// version 0.17.0) we make sure lnd subscribes to the correct
 		// zmq events. We do this to avoid a situation in which we are
 		// not notified of new transactions or blocks.
-		if ver >= 170000 {
+		if ver >= 170000 && !bitcoindMode.RPCPolling {
 			zmqPubRawBlockURL, err := url.Parse(bitcoindMode.ZMQPubRawBlock)
 			if err != nil {
 				return nil, nil, err

config.go

@@ -83,6 +83,10 @@ const (
 	defaultTorV2PrivateKeyFilename = "v2_onion_private_key"
 	defaultTorV3PrivateKeyFilename = "v3_onion_private_key"
 
+	// defaultZMQReadDeadline is the default read deadline to be used for
+	// both the block and tx ZMQ subscriptions.
+	defaultZMQReadDeadline = 5 * time.Second
+
 	// DefaultAutogenValidity is the default validity of a self-signed
 	// certificate. The value corresponds to 14 months
 	// (14 months * 30 days * 24 hours).
@@ -483,6 +487,7 @@ func DefaultConfig() Config {
 			RPCHost:            defaultRPCHost,
 			EstimateMode:       defaultBitcoindEstimateMode,
 			PrunedNodeMaxPeers: defaultPrunedNodeMaxPeers,
+			ZMQReadDeadline:    defaultZMQReadDeadline,
 		},
 		Litecoin: &lncfg.Chain{
 			MinHTLCIn:     chainreg.DefaultLitecoinMinHTLCInMSat,
@@ -1772,11 +1777,19 @@ func parseRPCParams(cConfig *lncfg.Chain, nodeConfig interface{},
 			}
 		}
 
-		// If all of RPCUser, RPCPass, ZMQBlockHost, and ZMQTxHost are
-		// set, we assume those parameters are good to use.
-		if conf.RPCUser != "" && conf.RPCPass != "" &&
-			conf.ZMQPubRawBlock != "" && conf.ZMQPubRawTx != "" {
-			return nil
+		if conf.RPCUser != "" && conf.RPCPass != "" {
+			// If all of RPCUser, RPCPass, ZMQBlockHost, and
+			// ZMQTxHost are set, we assume those parameters are
+			// good to use.
+			if conf.ZMQPubRawBlock != "" && conf.ZMQPubRawTx != "" {
+				return nil
+			}
+
+			// If RPCUser and RPCPass are set and RPCPolling is
+			// enabled, we assume the parameters are good to use.
+			if conf.RPCPolling {
+				return nil
+			}
 		}
 
 		// Get the daemon name for displaying proper errors.

docs/release-notes/release-notes-0.15.0.md

@@ -64,6 +64,12 @@ to Bitcoin nodes that advertise a Tor v3 onion service address.
 capable of status checks, adding, disconnecting and listing peers, fetching
 compact filters and block/block headers.
 
+## Btcwallet
+
+* [Add option to configure the block and transaction subscription 
+  notifications from bitcoind to be obtained through polling of the RPC 
+  interface instead of using ZMQ](https://github.com/lightningnetwork/lnd/pull/6345)
+
 ## Bug Fixes
 
 * [Pipelining an UpdateFulfillHTLC message now only happens when the related UpdateAddHTLC is locked-in.](https://github.com/lightningnetwork/lnd/pull/6246)

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/btcsuite/btcd/btcutil/psbt v1.1.3
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
-	github.com/btcsuite/btcwallet v0.14.1-0.20220412233800-3a6d5d0702b7
+	github.com/btcsuite/btcwallet v0.15.0
 	github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3
 	github.com/btcsuite/btcwallet/wallet/txrules v1.2.0
 	github.com/btcsuite/btcwallet/walletdb v1.4.0

go.sum

@@ -95,8 +95,8 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtyd
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/btcwallet v0.14.1-0.20220412233800-3a6d5d0702b7 h1:K7omr0FIjwwwQXTMgp2d+DTRH71RlkjRKj/Ir6Zkqb0=
-github.com/btcsuite/btcwallet v0.14.1-0.20220412233800-3a6d5d0702b7/go.mod h1:EE9BactCCWhCFoVfxCJrSFINrYqLx/Tq6quxRlPTpzM=
+github.com/btcsuite/btcwallet v0.15.0 h1:FdgC7JySVQJIcU+3W+kswDPv8rtzArGNQLOn2g3TiLg=
+github.com/btcsuite/btcwallet v0.15.0/go.mod h1:EE9BactCCWhCFoVfxCJrSFINrYqLx/Tq6quxRlPTpzM=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.2.1/go.mod h1:/74bubxX5Js48d76nf/TsNabpYp/gndUuJw4chzCmhU=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3 h1:M2yr5UlULvpqtxUqpMxTME/pA92Z9cpqeyvAFk9lAg0=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.2.3/go.mod h1:T2xSiKGpUkSLCh68aF+FMXmKK9mFqNdHl9VaqOr+JjU=

lncfg/bitcoind.go

@@ -1,16 +1,22 @@
 package lncfg
 
+import "time"
+
 // Bitcoind holds the configuration options for the daemon's connection to
 // bitcoind.
 type Bitcoind struct {
-	Dir                string `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."`
-	ConfigPath         string `long:"config" description:"Configuration filepath. If not set, will default to the default filename under 'dir'."`
-	RPCCookie          string `long:"rpccookie" description:"Authentication cookie file for RPC connections. If not set, will default to .cookie under 'dir'."`
-	RPCHost            string `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."`
-	RPCUser            string `long:"rpcuser" description:"Username for RPC connections"`
-	RPCPass            string `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`
-	ZMQPubRawBlock     string `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"`
-	ZMQPubRawTx        string `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"`
-	EstimateMode       string `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."`
-	PrunedNodeMaxPeers int    `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."`
+	Dir                  string        `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."`
+	ConfigPath           string        `long:"config" description:"Configuration filepath. If not set, will default to the default filename under 'dir'."`
+	RPCCookie            string        `long:"rpccookie" description:"Authentication cookie file for RPC connections. If not set, will default to .cookie under 'dir'."`
+	RPCHost              string        `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."`
+	RPCUser              string        `long:"rpcuser" description:"Username for RPC connections"`
+	RPCPass              string        `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`
+	ZMQPubRawBlock       string        `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"`
+	ZMQPubRawTx          string        `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"`
+	ZMQReadDeadline      time.Duration `long:"zmqreaddeadline" description:"The read deadline for reading ZMQ messages from both the block and tx subscriptions"`
+	EstimateMode         string        `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."`
+	PrunedNodeMaxPeers   int           `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."`
+	RPCPolling           bool          `long:"rpcpolling" description:"Poll the bitcoind RPC interface for block and transaction notifications instead of using the ZMQ interface"`
+	BlockPollingInterval time.Duration `long:"blockpollinginterval" description:"The interval that will be used to poll bitcoind for new blocks. Only used if rpcpolling is true."`
+	TxPollingInterval    time.Duration `long:"txpollinginterval" description:"The interval that will be used to poll bitcoind for new tx. Only used if rpcpolling is true."`
 }

lntest/bitcoind.go

@@ -1,5 +1,5 @@
-//go:build bitcoind && !notxindex
-// +build bitcoind,!notxindex
+//go:build bitcoind && !notxindex && !rpcpolling
+// +build bitcoind,!notxindex,!rpcpolling
 
 package lntest
 
@@ -19,5 +19,5 @@ func NewBackend(miner string, netParams *chaincfg.Params) (
 		"-disablewallet",
 	}
 
-	return newBackend(miner, netParams, extraArgs)
+	return newBackend(miner, netParams, extraArgs, false)
 }