Add option to poll for blocks/txs via RPC instead of using ZMQ

chainntnfs/bitcoindnotify/bitcoind_test.go

@@ -114,7 +114,7 @@ func TestHistoricalConfDetailsTxIndex(t *testing.T) {
 	defer tearDown()
 
 	bitcoindConn, cleanUp := chainntnfs.NewBitcoindBackend(
-		t, miner.P2PAddress(), true,
+		t, miner.P2PAddress(), true, false,
 	)
 	defer cleanUp()
 
@@ -206,11 +206,17 @@ func TestHistoricalConfDetailsTxIndex(t *testing.T) {
 // historical confirmation details using the set of fallback methods when the
 // backend node's txindex is disabled.
 func TestHistoricalConfDetailsNoTxIndex(t *testing.T) {
+	// Execute this test twice, once with rpcpolling off, and once on.
+	testHistoricalConfDetailsNoTxIndex(t, false)
+	testHistoricalConfDetailsNoTxIndex(t, true)
+}
+
+func testHistoricalConfDetailsNoTxIndex(t *testing.T, rpcPolling bool) {
 	miner, tearDown := chainntnfs.NewMiner(t, nil, true, 25)
 	defer tearDown()
 
 	bitcoindConn, cleanUp := chainntnfs.NewBitcoindBackend(
-		t, miner.P2PAddress(), false,
+		t, miner.P2PAddress(), false, rpcPolling,
 	)
 	defer cleanUp()
 

chainntnfs/bitcoindnotify/driver.go

@@ -56,13 +56,21 @@ func createNewNotifier(args ...interface{}) (chainntnfs.ChainNotifier, error) {
 // chainntnfs.ChainNotifier interface.
 func init() {
 	// Register the driver.
-	notifier := &chainntnfs.NotifierDriver{
-		NotifierType: notifierType,
-		New:          createNewNotifier,
+	notifiers := []*chainntnfs.NotifierDriver{
+		{
+			NotifierType: notifierType,
+			New:          createNewNotifier,
+		},
+		{
+			NotifierType: "bitcoind rpcpolling",
+			New:          createNewNotifier,
+		},
 	}
 
-	if err := chainntnfs.RegisterNotifier(notifier); err != nil {
-		panic(fmt.Sprintf("failed to register notifier driver '%s': %v",
-			notifierType, err))
+	for _, notifier := range notifiers {
+		if err := chainntnfs.RegisterNotifier(notifier); err != nil {
+			panic(fmt.Sprintf("failed to register notifier driver '%s': %v",
+				notifierType, err))
+		}
 	}
 }

chainntnfs/test/bitcoind/bitcoind_test.go

@@ -13,4 +13,5 @@ import (
 // powered chain notifier.
 func TestInterfaces(t *testing.T) {
 	chainntnfstest.TestInterfaces(t, "bitcoind")
+	chainntnfstest.TestInterfaces(t, "bitcoind rpcpolling")
 }

chainntnfs/test/test_interface.go

@@ -17,7 +17,6 @@ import (
 	"github.com/btcsuite/btcd/rpcclient"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
-	"github.com/btcsuite/btcwallet/chain"
 	_ "github.com/btcsuite/btcwallet/walletdb/bdb" // Required to auto-register the boltdb walletdb implementation.
 	"github.com/lightninglabs/neutrino"
 	"github.com/lightningnetwork/lnd/blockcache"
@@ -1962,16 +1961,14 @@ func TestInterfaces(t *testing.T, targetBackEnd string) {
 
 		switch notifierType {
 		case "bitcoind":
-			var bitcoindConn *chain.BitcoindConn
-			bitcoindConn, cleanUp = chainntnfs.NewBitcoindBackend(
-				t, p2pAddr, true,
+			newNotifier, cleanUp = startBitcoind(
+				t, p2pAddr, hintCache, blockCache, false,
+			)
+
+		case "bitcoind rpcpolling":
+			newNotifier, cleanUp = startBitcoind(
+				t, p2pAddr, hintCache, blockCache, true,
 			)
-			newNotifier = func() (chainntnfs.TestChainNotifier, error) {
-				return bitcoindnotify.New(
-					bitcoindConn, chainntnfs.NetParams,
-					hintCache, hintCache, blockCache,
-				), nil
-			}
 
 		case "btcd":
 			newNotifier = func() (chainntnfs.TestChainNotifier, error) {
@@ -2064,3 +2061,24 @@ func TestInterfaces(t *testing.T, targetBackEnd string) {
 		}
 	}
 }
+
+// startBitcoind sets up a new notifier backend and the notifier that builds
+// on top of it.
+func startBitcoind(t *testing.T, p2pAddr string,
+	hintCache *chainntnfs.HeightHintCache,
+	blockCache *blockcache.BlockCache, rpcPolling bool) (
+	func() (chainntnfs.TestChainNotifier, error), func()) {
+
+	bitcoindConn, cleanUp := chainntnfs.NewBitcoindBackend(
+		t, p2pAddr, true, rpcPolling,
+	)
+
+	newNotifier := func() (chainntnfs.TestChainNotifier, error) {
+		return bitcoindnotify.New(
+			bitcoindConn, chainntnfs.NetParams,
+			hintCache, hintCache, blockCache,
+		), nil
+	}
+
+	return newNotifier, cleanUp
+}

chainntnfs/test_utils.go

@@ -196,7 +196,7 @@ func NewMiner(t *testing.T, extraArgs []string, createChain bool,
 // backend node should maintain a transaction index. A connection to the newly
 // spawned bitcoind node is returned.
 func NewBitcoindBackend(t *testing.T, minerAddr string,
-	txindex bool) (*chain.BitcoindConn, func()) {
+	txindex bool, rpcPolling bool) (*chain.BitcoindConn, func()) {
 
 	t.Helper()
 
@@ -242,6 +242,9 @@ func NewBitcoindBackend(t *testing.T, minerAddr string,
 		ZMQBlockHost:    zmqBlockHost,
 		ZMQTxHost:       zmqTxHost,
 		ZMQReadDeadline: 5 * time.Second,
+		RPCPolling:      rpcPolling,
+		PollBlockTimer:  time.Millisecond * 500,
+		PollTxTimer:     time.Millisecond * 500,
 		// Fields only required for pruned nodes, not needed for these
 		// tests.
 		Dialer:             nil,

chainreg/chainregistry.go

@@ -408,6 +408,9 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) {
 			ZMQBlockHost:       bitcoindMode.ZMQPubRawBlock,
 			ZMQTxHost:          bitcoindMode.ZMQPubRawTx,
 			ZMQReadDeadline:    5 * time.Second,
+			RPCPolling:         bitcoindMode.RPCPolling,
+			PollBlockTimer:     bitcoindMode.PollBlockTimer,
+			PollTxTimer:        bitcoindMode.PollTxTimer,
 			Dialer:             cfg.Dialer,
 			PrunedModeMaxPeers: bitcoindMode.PrunedNodeMaxPeers,
 		})

config.go

@@ -1694,13 +1694,6 @@ func parseRPCParams(cConfig *lncfg.Chain, nodeConfig interface{},
 			}
 		}
 
-		// If all of RPCUser, RPCPass, ZMQBlockHost, and ZMQTxHost are
-		// set, we assume those parameters are good to use.
-		if conf.RPCUser != "" && conf.RPCPass != "" &&
-			conf.ZMQPubRawBlock != "" && conf.ZMQPubRawTx != "" {
-			return nil
-		}
-
 		// Get the daemon name for displaying proper errors.
 		switch net {
 		case chainreg.BitcoinChain:
@@ -1713,16 +1706,45 @@ func parseRPCParams(cConfig *lncfg.Chain, nodeConfig interface{},
 			confFile = "litecoin"
 		}
 
-		// If not all of the parameters are set, we'll assume the user
-		// did this unintentionally.
-		if conf.RPCUser != "" || conf.RPCPass != "" ||
-			conf.ZMQPubRawBlock != "" || conf.ZMQPubRawTx != "" {
+		// Both ZMQ and RPC polling cannot be set at the same time.
+		// Make sure only one is set.
+		if (conf.ZMQPubRawBlock != "" || conf.ZMQPubRawTx != "") &&
+			conf.RPCPolling {
 
-			return fmt.Errorf("please set all or none of "+
-				"%[1]v.rpcuser, %[1]v.rpcpass, "+
-				"%[1]v.zmqpubrawblock, %[1]v.zmqpubrawtx",
+			return fmt.Errorf("please set either both "+
+				"%[1]v.zmqpubrawblock and %[1]v.zmqpubrawtx, "+
+				"or just %[1]v.rpcpolling, but not both",
 				daemonName)
 		}
+
+		// Check that the needed parameters for polling blocks and
+		// transactions via RPC are all set.
+		if conf.RPCPolling {
+			if conf.RPCUser == "" || conf.RPCPass == "" {
+
+				// If not all of the RPC parameters are set, we'll
+				// assume the user did this unintentionally.
+				return fmt.Errorf("please set all or none of "+
+					"%[1]v.rpcuser, %[1]v.rpcpass, "+
+					"%[1]v.rpcpolling", daemonName)
+			}
+			return nil
+		}
+
+		// Check that all the needed parameters for ZMQ are set.
+		if conf.ZMQPubRawBlock != "" || conf.ZMQPubRawTx != "" {
+			// If not all of the parameters are set, we'll assume the user
+			// did this unintentionally.
+			if conf.RPCUser == "" || conf.RPCPass == "" ||
+				conf.ZMQPubRawBlock == "" || conf.ZMQPubRawTx == "" {
+
+				return fmt.Errorf("please set all or none of "+
+					"%[1]v.rpcuser, %[1]v.rpcpass, "+
+					"%[1]v.zmqpubrawblock, %[1]v.zmqpubrawtx",
+					daemonName)
+			}
+			return nil
+		}
 	}
 
 	// If we're in simnet mode, then the running btcd instance won't read

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
 	github.com/btcsuite/btcutil v1.0.3-0.20210527170813-e2ba6805a890
 	github.com/btcsuite/btcutil/psbt v1.0.3-0.20210527170813-e2ba6805a890
-	github.com/btcsuite/btcwallet v0.13.0
+	github.com/btcsuite/btcwallet v0.13.1-0.20211201210108-79de92f527dc
 	github.com/btcsuite/btcwallet/wallet/txauthor v1.1.0
 	github.com/btcsuite/btcwallet/wallet/txrules v1.1.0
 	github.com/btcsuite/btcwallet/walletdb v1.3.6-0.20210803004036-eebed51155ec
@@ -75,6 +75,8 @@ require (
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 )
 
+replace github.com/btcsuite/btcwallet => github.com/orbitalturtle/btcwallet v0.13.1-0.20220131052201-c2ded756781a
+
 replace github.com/lightningnetwork/lnd/ticker => ./ticker
 
 replace github.com/lightningnetwork/lnd/queue => ./queue

go.sum

@@ -97,8 +97,8 @@ github.com/btcsuite/btcutil v1.0.3-0.20210527170813-e2ba6805a890/go.mod h1:0DVlH
 github.com/btcsuite/btcutil/psbt v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:LVveMu4VaNSkIRTZu2+ut0HDBRuYjqGocxDMNS1KuGQ=
 github.com/btcsuite/btcutil/psbt v1.0.3-0.20210527170813-e2ba6805a890 h1:0xUNvvwJ7RjzBs4nCF+YrK28S5P/b4uHkpPxY1ovGY4=
 github.com/btcsuite/btcutil/psbt v1.0.3-0.20210527170813-e2ba6805a890/go.mod h1:LVveMu4VaNSkIRTZu2+ut0HDBRuYjqGocxDMNS1KuGQ=
-github.com/btcsuite/btcwallet v0.13.0 h1:gtLWwueRm27KQiHJpycybv3uMdK1eo87JexfTfzvEhk=
-github.com/btcsuite/btcwallet v0.13.0/go.mod h1:iLN1lG1MW0eREm+SikmPO8AZPz5NglBTEK/ErqkjGpo=
+github.com/btcsuite/btcwallet v0.13.1-0.20211201210108-79de92f527dc h1:lAbAEAp4eWvsSwJfcpdHXpKz78X2sVF9aDK4nJveXmY=
+github.com/btcsuite/btcwallet v0.13.1-0.20211201210108-79de92f527dc/go.mod h1:iLN1lG1MW0eREm+SikmPO8AZPz5NglBTEK/ErqkjGpo=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0/go.mod h1:VufDts7bd/zs3GV13f/lXc/0lXrPnvxD/NvmpG/FEKU=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.1.0 h1:8pO0pvPX1rFRfRiol4oV6kX7dY5y4chPwhfVwUfvwtk=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.1.0/go.mod h1:ktYuJyumYtwG+QQ832Q+kqvxWJRAei3Nqs5qhSn4nww=
@@ -643,6 +643,8 @@ github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/orbitalturtle/btcwallet v0.13.1-0.20220131052201-c2ded756781a h1:1rhrSi/7FmRknqOzxwLhCePVNWMxblTVwp4BgDe7O7A=
+github.com/orbitalturtle/btcwallet v0.13.1-0.20220131052201-c2ded756781a/go.mod h1:28pFfBjI+nrkqL8OJU8riooZidKV17f+SkF5y9dbk5g=
 github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0=
 github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw=
 github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE=

lncfg/bitcoind.go

@@ -1,14 +1,19 @@
 package lncfg
 
+import "time"
+
 // Bitcoind holds the configuration options for the daemon's connection to
 // bitcoind.
 type Bitcoind struct {
-	Dir                string `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."`
-	RPCHost            string `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."`
-	RPCUser            string `long:"rpcuser" description:"Username for RPC connections"`
-	RPCPass            string `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`
-	ZMQPubRawBlock     string `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"`
-	ZMQPubRawTx        string `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"`
-	EstimateMode       string `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."`
-	PrunedNodeMaxPeers int    `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."`
+	Dir                string        `long:"dir" description:"The base directory that contains the node's data, logs, configuration file, etc."`
+	RPCHost            string        `long:"rpchost" description:"The daemon's rpc listening address. If a port is omitted, then the default port for the selected chain parameters will be used."`
+	RPCUser            string        `long:"rpcuser" description:"Username for RPC connections"`
+	RPCPass            string        `long:"rpcpass" default-mask:"-" description:"Password for RPC connections"`
+	ZMQPubRawBlock     string        `long:"zmqpubrawblock" description:"The address listening for ZMQ connections to deliver raw block notifications"`
+	ZMQPubRawTx        string        `long:"zmqpubrawtx" description:"The address listening for ZMQ connections to deliver raw transaction notifications"`
+	RPCPolling         bool          `long:"rpcpolling" description:"Opt to use RPC Polling to retrieve block and tx updates, instead of ZMQ"`
+	PollBlockTimer     time.Duration `long:"pollblocktimer" description:"PollBlockTimer sets how often we'll poll for fresh blocks, if rpcpolling is true."`
+	PollTxTimer        time.Duration `long:"polltxtimer" description:"PollTxTimer sets how often we'll poll for fresh blocks, if rpcpolling is true."`
+	EstimateMode       string        `long:"estimatemode" description:"The fee estimate mode. Must be either ECONOMICAL or CONSERVATIVE."`
+	PrunedNodeMaxPeers int           `long:"pruned-node-max-peers" description:"The maximum number of peers lnd will choose from the backend node to retrieve pruned blocks from. This only applies to pruned nodes."`
 }

lntest/bitcoind.go

@@ -1,5 +1,5 @@
-//go:build bitcoind && !notxindex
-// +build bitcoind,!notxindex
+//go:build bitcoind && !notxindex && !rpcpolling
+// +build bitcoind,!notxindex,!rpcpolling
 
 package lntest
 
@@ -19,5 +19,5 @@ func NewBackend(miner string, netParams *chaincfg.Params) (
 		"-disablewallet",
 	}
 
-	return newBackend(miner, netParams, extraArgs)
+	return newBackend(miner, netParams, extraArgs, false)
 }

lntest/bitcoind_common.go

@@ -28,6 +28,7 @@ type BitcoindBackendConfig struct {
 	zmqBlockPath string
 	zmqTxPath    string
 	p2pPort      int
+	RpcPolling   bool
 	rpcClient    *rpcclient.Client
 
 	// minerAddr is the p2p address of the miner to connect to.
@@ -46,10 +47,17 @@ func (b BitcoindBackendConfig) GenArgs() []string {
 	args = append(args, fmt.Sprintf("--bitcoind.rpchost=%v", b.rpcHost))
 	args = append(args, fmt.Sprintf("--bitcoind.rpcuser=%v", b.rpcUser))
 	args = append(args, fmt.Sprintf("--bitcoind.rpcpass=%v", b.rpcPass))
-	args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawblock=%v",
-		b.zmqBlockPath))
-	args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawtx=%v",
-		b.zmqTxPath))
+	if b.RpcPolling {
+		args = append(args, "--bitcoind.rpcpolling",
+			"--bitcoind.pollblocktimer=500ms",
+			"--bitcoind.polltxtimer=500ms",
+		)
+	} else {
+		args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawblock=%v",
+			b.zmqBlockPath))
+		args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawtx=%v",
+			b.zmqTxPath))
+	}
 
 	return args
 }
@@ -71,8 +79,8 @@ func (b BitcoindBackendConfig) Name() string {
 
 // newBackend starts a bitcoind node with the given extra parameters and returns
 // a BitcoindBackendConfig for that node.
-func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
-	*BitcoindBackendConfig, func() error, error) {
+func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string,
+	rpcPolling bool) (*BitcoindBackendConfig, func() error, error) {
 
 	baseLogDir := fmt.Sprintf(logDirPattern, GetLogDir())
 	if netParams != &chaincfg.RegressionNetParams {
@@ -111,6 +119,7 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
 		"-zmqpubrawtx=" + zmqTxAddr,
 		"-debuglogfile=" + logFile,
 	}
+
 	cmdArgs = append(cmdArgs, extraArgs...)
 	bitcoind := exec.Command("bitcoind", cmdArgs...)
 
@@ -185,6 +194,7 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
 		zmqBlockPath: zmqBlockAddr,
 		zmqTxPath:    zmqTxAddr,
 		p2pPort:      p2pPort,
+		RpcPolling:   rpcPolling,
 		rpcClient:    client,
 		minerAddr:    miner,
 	}

lntest/bitcoind_notxindex.go

@@ -18,5 +18,5 @@ func NewBackend(miner string, netParams *chaincfg.Params) (
 		"-disablewallet",
 	}
 
-	return newBackend(miner, netParams, extraArgs)
+	return newBackend(miner, netParams, extraArgs, false)
 }

lntest/bitcoind_rpcpolling.go

@@ -0,0 +1,22 @@
+//go:build bitcoind && rpcpolling
+// +build bitcoind,rpcpolling
+
+package lntest
+
+import (
+	"github.com/btcsuite/btcd/chaincfg"
+)
+
+// NewBackend starts a bitcoind node with the txindex enabled and returns a
+// BitcoindBackendConfig for that node.
+func NewBackend(miner string, netParams *chaincfg.Params) (
+	*BitcoindBackendConfig, func() error, error) {
+
+	extraArgs := []string{
+		"-debug",
+		"-regtest",
+		"-disablewallet",
+	}
+
+	return newBackend(miner, netParams, extraArgs, true)
+}

lnwallet/test/bitcoind/bitcoind_rpcpolling_test.go

@@ -0,0 +1,13 @@
+package bitcoind_test
+
+import (
+	"testing"
+
+	lnwallettest "github.com/lightningnetwork/lnd/lnwallet/test"
+)
+
+// TestLightningWallet tests LightningWallet powered by bitcoind against our
+// suite of interface tests.
+func TestLightningWalletRPCPolling(t *testing.T) {
+	lnwallettest.TestLightningWallet(t, "bitcoind rpcpolling")
+}