allow functional tests to be used externally with a dynamic signer factory

.gitignore

@@ -12,5 +12,6 @@ lightning/net_graph-*.bin
 lightning-rapid-gossip-sync/res/full_graph.lngossip
 lightning-custom-message/target
 lightning-transaction-sync/target
+ext-functional-test-demo/target
 no-std-check/target
 msrv-no-dev-deps-check/target

Cargo.toml

@@ -12,9 +12,11 @@ members = [
     "lightning-custom-message",
     "lightning-transaction-sync",
     "possiblyrandom",
+    "ext-test-macro",
 ]
 
 exclude = [
+    "ext-functional-test-demo",
     "no-std-check",
     "msrv-no-dev-deps-check",
     "bench",

ci/check-cfg-flags.py

@@ -158,7 +158,11 @@ def check_cfg_args(cfg):
             line = file.readline()
             if not line:
                 break
-            if "#[cfg(" in line:
+            if "#[cfg(" in line and not "quote!" in line:
                 if not line.strip().startswith("//"):
-                    cfg_part = cfg_regex.match(line.strip()).group(1)
+                    match = cfg_regex.match(line.strip())
+                    if match is None:
+                        print("Bad cfg line: " + line.strip())
+                        continue
+                    cfg_part = match.group(1)
                     check_cfg_args(cfg_part)

ci/ci-tests.sh

@@ -157,6 +157,12 @@ cargo check --verbose --color always --features lightning-transaction-sync
 [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean
 popd
 
+echo -e "\n\Running functional tests from outside the workspace"
+pushd ext-functional-test-demo
+cargo test --color always
+[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean
+popd
+
 # Test that we can build downstream code with only the "release pins".
 pushd msrv-no-dev-deps-check
 PIN_RELEASE_DEPS

ext-functional-test-demo/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "ext-functional-tester"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+lightning = { path = "../lightning", features = ["_test_utils"] }

ext-functional-test-demo/src/main.rs

@@ -0,0 +1,31 @@
+fn main() {
+	println!("Hello, world!");
+}
+
+#[cfg(test)]
+mod tests {
+	use lightning::util::dyn_signer::{DynKeysInterfaceTrait, DynSigner};
+	use lightning::util::test_utils::{TestSignerFactory, SIGNER_FACTORY};
+	use std::panic::catch_unwind;
+	use std::sync::Arc;
+	use std::time::Duration;
+
+	struct BrokenSignerFactory();
+
+	impl TestSignerFactory for BrokenSignerFactory {
+		fn make_signer(
+			&self, _seed: &[u8; 32], _now: Duration,
+		) -> Box<dyn DynKeysInterfaceTrait<EcdsaSigner = DynSigner>> {
+			panic!()
+		}
+	}
+
+	#[test]
+	fn test_functional() {
+		lightning::ln::functional_tests::test_insane_channel_opens();
+		lightning::ln::functional_tests::fake_network_test();
+
+		SIGNER_FACTORY.set(Arc::new(BrokenSignerFactory()));
+		catch_unwind(|| lightning::ln::functional_tests::fake_network_test()).unwrap_err();
+	}
+}

ext-test-macro/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "ext-test-macro"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+syn = { version = "1.0", features = ["full"] }
+quote = "1.0"
+proc-macro2 = "1.0"

ext-test-macro/src/lib.rs

@@ -0,0 +1,73 @@
+use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
+use quote::quote;
+use syn::{parse2, ItemFn, ItemMod};
+
+/// An exposed test.  This is a test that will run locally and also be
+/// made available to other crates that want to run it in their own context.
+///
+/// For example:
+/// ```rust
+/// use ext_test_macro::xtest;
+///
+/// fn f1() {}
+///
+/// #[xtest(feature = "_test_utils")]
+/// pub fn test_f1() {
+///     f1();
+/// }
+/// ```
+///
+/// May also be applied to modules, like so:
+///
+/// ```rust
+/// use ext_test_macro::xtest;
+///
+/// #[xtest(feature = "_test_utils")]
+/// pub mod tests {
+/// 	use super::*;
+///
+///     fn f1() {}
+///
+/// 	#[xtest]
+/// 	pub fn test_f1() {
+/// 	    f1();
+/// 	}
+/// }
+/// ```
+///
+/// Which will include the module if we are testing or the `externalize-the-tests` feature
+/// is on.
+#[proc_macro_attribute]
+pub fn xtest(attrs: TokenStream, item: TokenStream) -> TokenStream {
+	let input = syn::parse_macro_input!(item as TokenStream2);
+	let attrs = syn::parse_macro_input!(attrs as TokenStream2);
+
+	let expanded = if is_module_definition(input.clone()) {
+		let cfg = if attrs.is_empty() {
+			quote! { #[cfg(test)] }
+		} else {
+			quote! { #[cfg(any(test, #attrs))] }
+		};
+		quote! {
+			#cfg
+			#input
+		}
+	} else if is_function_definition(input.clone()) {
+		quote! {
+			#[cfg_attr(test, ::core::prelude::v1::test)]
+			#input
+		}
+	} else {
+		panic!("xtest can only be applied to functions or modules");
+	};
+	expanded.into()
+}
+
+fn is_module_definition(input: TokenStream2) -> bool {
+	parse2::<ItemMod>(input).is_ok()
+}
+
+fn is_function_definition(input: TokenStream2) -> bool {
+	parse2::<ItemFn>(input).is_ok()
+}

fuzz/src/chanmon_consistency.rs

@@ -71,6 +71,7 @@ use std::sync::{Arc,Mutex};
 use std::sync::atomic;
 use std::io::Cursor;
 use bitcoin::bech32::u5;
+use lightning::util::dyn_signer::DynSigner;
 
 const MAX_FEE: u32 = 10_000;
 struct FuzzEstimator {
@@ -276,6 +277,7 @@ impl SignerProvider for KeyProvider {
 			channel_keys_id,
 		);
 		let revoked_commitment = self.make_enforcement_state_cell(keys.commitment_seed);
+		let keys = DynSigner::new(keys);
 		TestChannelSigner::new_with_revoked(keys, revoked_commitment, false)
 	}
 
@@ -284,6 +286,7 @@ impl SignerProvider for KeyProvider {
 
 		let inner: InMemorySigner = ReadableArgs::read(&mut reader, self)?;
 		let state = self.make_enforcement_state_cell(inner.commitment_seed);
+		let inner = DynSigner::new(inner);
 
 		Ok(TestChannelSigner {
 			inner,

fuzz/src/full_stack.rs

@@ -69,6 +69,7 @@ use std::cmp;
 use std::sync::{Arc, Mutex};
 use std::sync::atomic::{AtomicU64,AtomicUsize,Ordering};
 use bitcoin::bech32::u5;
+use lightning::util::dyn_signer::DynSigner;
 
 #[inline]
 pub fn slice_to_be16(v: &[u8]) -> u16 {
@@ -370,7 +371,7 @@ impl SignerProvider for KeyProvider {
 		let secp_ctx = Secp256k1::signing_only();
 		let ctr = channel_keys_id[0];
 		let (inbound, state) = self.signer_state.borrow().get(&ctr).unwrap().clone();
-		TestChannelSigner::new_with_revoked(if inbound {
+		let inner = if inbound {
 			InMemorySigner::new(
 				&secp_ctx,
 				SecretKey::from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, ctr]).unwrap(),
@@ -396,12 +397,15 @@ impl SignerProvider for KeyProvider {
 				channel_keys_id,
 				channel_keys_id,
 			)
-		}, state, false)
+		};
+		let inner = DynSigner::new(inner);
+		TestChannelSigner::new_with_revoked(inner, state, false)
 	}
 
 	fn read_chan_signer(&self, mut data: &[u8]) -> Result<TestChannelSigner, DecodeError> {
 		let inner: InMemorySigner = ReadableArgs::read(&mut data, self)?;
 		let state = Arc::new(Mutex::new(EnforcementState::new()));
+		let inner = DynSigner::new(inner);
 
 		Ok(TestChannelSigner::new_with_revoked(
 			inner,

lightning-background-processor/src/lib.rs

@@ -1918,6 +1918,8 @@ mod tests {
 				failure: PathFailure::OnPath { network_update: None },
 				path: path.clone(),
 				short_channel_id: Some(scored_scid),
+				error_code: None,
+				error_data: None,
 			});
 			let event = $receive.expect("PaymentPathFailed not handled within deadline");
 			match event {
@@ -1935,6 +1937,8 @@ mod tests {
 				failure: PathFailure::OnPath { network_update: None },
 				path: path.clone(),
 				short_channel_id: None,
+				error_code: None,
+				error_data: None,
 			});
 			let event = $receive.expect("PaymentPathFailed not handled within deadline");
 			match event {

lightning-custom-message/src/lib.rs

@@ -167,6 +167,16 @@
 //! #     }
 //! }
 //!
+//! #[macro_export]
+//! macro_rules! foo_bar_type_ids {
+//!     () => { foo_type_id!() | bar_type_id!() }
+//! }
+//!
+//! #[macro_export]
+//! macro_rules! foo_bar_baz_type_ids {
+//!     () => { foo_bar_type_ids!() | baz_type_id!() }
+//! }
+//!
 //! # fn main() {
 //! // The first crate may define a handler composing `FooHandler` and `BarHandler` and export the
 //! // corresponding message type ids as a macro to use in further composition.
@@ -183,11 +193,6 @@
 //!     }
 //! );
 //!
-//! #[macro_export]
-//! macro_rules! foo_bar_type_ids {
-//!     () => { foo_type_id!() | bar_type_id!() }
-//! }
-//!
 //! // Another crate can then define a handler further composing `FooBarHandler` with `BazHandler`
 //! // and similarly export the composition of message type ids as a macro.
 //!
@@ -203,10 +208,6 @@
 //!     }
 //! );
 //!
-//! #[macro_export]
-//! macro_rules! foo_bar_baz_type_ids {
-//!     () => { foo_bar_type_ids!() | baz_type_id!() }
-//! }
 //! # }
 //!```
 //!

lightning-invoice/src/payment.rs

@@ -173,6 +173,9 @@ mod tests {
 		use lightning::ln::channelmanager::{Retry, PaymentId};
 		use lightning::ln::msgs::ChannelMessageHandler;
 		use lightning::ln::functional_test_utils::*;
+		use lightning::sign::{NodeSigner, Recipient};
+		use bech32::ToBase32;
+
 		// Test that a payment metadata read from an invoice passed to `pay_invoice` makes it all
 		// the way out through the `PaymentClaimable` event.
 		let chanmon_cfgs = create_chanmon_cfgs(2);
@@ -194,11 +197,10 @@ mod tests {
 			.min_final_cltv_expiry_delta(144)
 			.amount_milli_satoshis(50_000)
 			.payment_metadata(payment_metadata.clone())
-			.build_signed(|hash| {
-				Secp256k1::new().sign_ecdsa_recoverable(hash,
-					&nodes[1].keys_manager.backing.get_node_secret_key())
-			})
-			.unwrap();
+			.build_raw().unwrap();
+		let sig = nodes[1].keys_manager.backing.sign_invoice(invoice.hrp.to_string().as_bytes(), &invoice.data.to_base32(), Recipient::Node).unwrap();
+		let invoice = invoice.sign::<_, ()>(|_| Ok(sig)).unwrap();
+		let invoice = Bolt11Invoice::from_signed(invoice).unwrap();
 
 		let (hash, onion, params) = payment_parameters_from_invoice(&invoice).unwrap();
 		nodes[0].node.send_payment(hash, onion, PaymentId(hash.0), params, Retry::Attempts(0)).unwrap();