benchmark_group.rs

use crate::analysis;
use crate::benchmark::PartialBenchmarkConfig;
use crate::connection::OutgoingMessage;
use crate::measurement::Measurement;
use crate::report::BenchmarkId as InternalBenchmarkId;
use crate::report::Report;
use crate::report::ReportContext;
use crate::routine::{Function, Routine};
use crate::{Bencher, Criterion, DurationExt, Mode, PlotConfiguration, SamplingMode, Throughput};
use std::time::Duration;

/// Structure used to group together a set of related benchmarks, along with custom configuration
/// settings for groups of benchmarks. All benchmarks performed using a benchmark group will be
/// grouped together in the final report.
///
/// # Examples:
///
/// ```no_run
/// #[macro_use] extern crate criterion;
/// use self::criterion::*;
/// use std::time::Duration;
///
/// fn bench_simple(c: &mut Criterion) {
///     let mut group = c.benchmark_group("My Group");
///
///     // Now we can perform benchmarks with this group
///     group.bench_function("Bench 1", |b| b.iter(|| 1 ));
///     group.bench_function("Bench 2", |b| b.iter(|| 2 ));
///    
///     // It's recommended to call group.finish() explicitly at the end, but if you don't it will
///     // be called automatically when the group is dropped.
///     group.finish();
/// }
///
/// fn bench_nested(c: &mut Criterion) {
///     let mut group = c.benchmark_group("My Second Group");
///     // We can override the configuration on a per-group level
///     group.measurement_time(Duration::from_secs(1));
///
///     // We can also use loops to define multiple benchmarks, even over multiple dimensions.
///     for x in 0..3 {
///         for y in 0..3 {
///             let point = (x, y);
///             let parameter_string = format!("{} * {}", x, y);
///             group.bench_with_input(BenchmarkId::new("Multiply", parameter_string), &point,
///                 |b, (p_x, p_y)| b.iter(|| p_x * p_y));
///         }
///     }
///    
///     group.finish();
/// }
///
/// fn bench_throughput(c: &mut Criterion) {
///     let mut group = c.benchmark_group("Summation");
///     
///     for size in [1024, 2048, 4096].iter() {
///         // Generate input of an appropriate size...
///         let input = vec![1u64, *size];
///
///         // We can use the throughput function to tell Criterion.rs how large the input is
///         // so it can calculate the overall throughput of the function. If we wanted, we could
///         // even change the benchmark configuration for different inputs (eg. to reduce the
///         // number of samples for extremely large and slow inputs) or even different functions.
///         group.throughput(Throughput::Elements(*size as u64));
///
///         group.bench_with_input(BenchmarkId::new("sum", *size), &input,
///             |b, i| b.iter(|| i.iter().sum::<u64>()));
///         group.bench_with_input(BenchmarkId::new("fold", *size), &input,
///             |b, i| b.iter(|| i.iter().fold(0u64, |a, b| a + b)));
///     }
///
///     group.finish();
/// }
///
/// criterion_group!(benches, bench_simple, bench_nested, bench_throughput);
/// criterion_main!(benches);
/// ```
pub struct BenchmarkGroup<'a, M: Measurement> {
    criterion: &'a mut Criterion<M>,
    group_name: String,
    all_ids: Vec<InternalBenchmarkId>,
    any_matched: bool,
    partial_config: PartialBenchmarkConfig,
    throughput: Option<Throughput>,
}
impl<'a, M: Measurement> BenchmarkGroup<'a, M> {
    /// Changes the size of the sample for this benchmark
    ///
    /// A bigger sample should yield more accurate results if paired with a sufficiently large
    /// measurement time.
    ///
    /// Sample size must be at least 10.
    ///
    /// # Panics
    ///
    /// Panics if n < 10.
    pub fn sample_size(&mut self, n: usize) -> &mut Self {
        assert!(n >= 10);

        self.partial_config.sample_size = Some(n);
        self
    }

    /// Changes the warm up time for this benchmark
    ///
    /// # Panics
    ///
    /// Panics if the input duration is zero
    pub fn warm_up_time(&mut self, dur: Duration) -> &mut Self {
        assert!(dur.to_nanos() > 0);

        self.partial_config.warm_up_time = Some(dur);
        self
    }

    /// Changes the target measurement time for this benchmark group.
    ///
    /// Criterion will attempt to spent approximately this amount of time measuring each
    /// benchmark on a best-effort basis. If it is not possible to perform the measurement in
    /// the requested time (eg. because each iteration of the benchmark is long) then Criterion
    /// will spend as long as is needed to collect the desired number of samples. With a longer
    /// time, the measurement will become more resilient to interference from other programs.
    ///
    /// # Panics
    ///
    /// Panics if the input duration is zero
    pub fn measurement_time(&mut self, dur: Duration) -> &mut Self {
        assert!(dur.to_nanos() > 0);

        self.partial_config.measurement_time = Some(dur);
        self
    }

    /// Changes the number of resamples for this benchmark group
    ///
    /// Number of resamples to use for the
    /// [bootstrap](http://en.wikipedia.org/wiki/Bootstrapping_(statistics)#Case_resampling)
    ///
    /// A larger number of resamples reduces the random sampling errors which are inherent to the
    /// bootstrap method, but also increases the analysis time.
    ///
    /// # Panics
    ///
    /// Panics if the number of resamples is set to zero
    pub fn nresamples(&mut self, n: usize) -> &mut Self {
        assert!(n > 0);
        if n <= 1000 {
            println!("\nWarning: It is not recommended to reduce nresamples below 1000.");
        }

        self.partial_config.nresamples = Some(n);
        self
    }

    /// Changes the noise threshold for benchmarks in this group. The noise threshold
    /// is used to filter out small changes in performance from one run to the next, even if they
    /// are statistically significant. Sometimes benchmarking the same code twice will result in
    /// small but statistically significant differences solely because of noise. This provides a way
    /// to filter out some of these false positives at the cost of making it harder to detect small
    /// changes to the true performance of the benchmark.
    ///
    /// The default is 0.01, meaning that changes smaller than 1% will be ignored.
    ///
    /// # Panics
    ///
    /// Panics if the threshold is set to a negative value
    pub fn noise_threshold(&mut self, threshold: f64) -> &mut Self {
        assert!(threshold >= 0.0);

        self.partial_config.noise_threshold = Some(threshold);
        self
    }

    /// Changes the confidence level for benchmarks in this group. The confidence
    /// level is the desired probability that the true runtime lies within the estimated
    /// [confidence interval](https://en.wikipedia.org/wiki/Confidence_interval). The default is
    /// 0.95, meaning that the confidence interval should capture the true value 95% of the time.
    ///
    /// # Panics
    ///
    /// Panics if the confidence level is set to a value outside the `(0, 1)` range
    pub fn confidence_level(&mut self, cl: f64) -> &mut Self {
        assert!(cl > 0.0 && cl < 1.0);
        if cl < 0.5 {
            println!("\nWarning: It is not recommended to reduce confidence level below 0.5.");
        }

        self.partial_config.confidence_level = Some(cl);
        self
    }

    /// Changes the [significance level](https://en.wikipedia.org/wiki/Statistical_significance)
    /// for benchmarks in this group. This is used to perform a
    /// [hypothesis test](https://en.wikipedia.org/wiki/Statistical_hypothesis_testing) to see if
    /// the measurements from this run are different from the measured performance of the last run.
    /// The significance level is the desired probability that two measurements of identical code
    /// will be considered 'different' due to noise in the measurements. The default value is 0.05,
    /// meaning that approximately 5% of identical benchmarks will register as different due to
    /// noise.
    ///
    /// This presents a trade-off. By setting the significance level closer to 0.0, you can increase
    /// the statistical robustness against noise, but it also weakens Criterion.rs' ability to
    /// detect small but real changes in the performance. By setting the significance level
    /// closer to 1.0, Criterion.rs will be more able to detect small true changes, but will also
    /// report more spurious differences.
    ///
    /// See also the noise threshold setting.
    ///
    /// # Panics
    ///
    /// Panics if the significance level is set to a value outside the `(0, 1)` range
    pub fn significance_level(&mut self, sl: f64) -> &mut Self {
        assert!(sl > 0.0 && sl < 1.0);

        self.partial_config.significance_level = Some(sl);
        self
    }

    /// Changes the plot configuration for this benchmark group.
    pub fn plot_config(&mut self, new_config: PlotConfiguration) -> &mut Self {
        self.partial_config.plot_config = new_config;
        self
    }

    /// Set the input size for this benchmark group. Used for reporting the
    /// throughput.
    pub fn throughput(&mut self, throughput: Throughput) -> &mut Self {
        self.throughput = Some(throughput);
        self
    }

    /// Set the sampling mode for this benchmark group.
    pub fn sampling_mode(&mut self, new_mode: SamplingMode) -> &mut Self {
        self.partial_config.sampling_mode = Some(new_mode);
        self
    }

    pub(crate) fn new(criterion: &mut Criterion<M>, group_name: String) -> BenchmarkGroup<'_, M> {
        BenchmarkGroup {
            criterion,
            group_name,
            all_ids: vec![],
            any_matched: false,
            partial_config: PartialBenchmarkConfig::default(),
            throughput: None,
        }
    }

    /// Benchmark the given parameterless function inside this benchmark group.
    pub fn bench_function<ID: IntoBenchmarkId, F>(&mut self, id: ID, mut f: F) -> &mut Self
    where
        F: FnMut(&mut Bencher<'_, M>),
    {
        self.run_bench(id.into_benchmark_id(), &(), |b, _| f(b));
        self
    }

    /// Benchmark the given parameterized function inside this benchmark group.
    pub fn bench_with_input<ID: IntoBenchmarkId, F, I>(
        &mut self,
        id: ID,
        input: &I,
        f: F,
    ) -> &mut Self
    where
        F: FnMut(&mut Bencher<'_, M>, &I),
        I: ?Sized,
    {
        self.run_bench(id.into_benchmark_id(), input, f);
        self
    }

    fn run_bench<F, I>(&mut self, id: BenchmarkId, input: &I, f: F)
    where
        F: FnMut(&mut Bencher<'_, M>, &I),
        I: ?Sized,
    {
        let config = self.partial_config.to_complete(&self.criterion.config);
        let report_context = ReportContext {
            output_directory: self.criterion.output_directory.clone(),
            plot_config: self.partial_config.plot_config.clone(),
        };

        let mut id = InternalBenchmarkId::new(
            self.group_name.clone(),
            id.function_name,
            id.parameter,
            self.throughput.clone(),
        );

        assert!(
            !self.all_ids.contains(&id),
            "Benchmark IDs must be unique within a group."
        );

        id.ensure_directory_name_unique(&self.criterion.all_directories);
        self.criterion
            .all_directories
            .insert(id.as_directory_name().to_owned());
        id.ensure_title_unique(&self.criterion.all_titles);
        self.criterion.all_titles.insert(id.as_title().to_owned());

        let do_run = self.criterion.filter_matches(id.id());
        self.any_matched |= do_run;
        let mut func = Function::new(f);

        match self.criterion.mode {
            Mode::Benchmark => {
                if let Some(conn) = &self.criterion.connection {
                    if do_run {
                        conn.send(&OutgoingMessage::BeginningBenchmark { id: (&id).into() })
                            .unwrap();
                    } else {
                        conn.send(&OutgoingMessage::SkippingBenchmark { id: (&id).into() })
                            .unwrap();
                    }
                }
                if do_run {
                    analysis::common(
                        &id,
                        &mut func,
                        &config,
                        self.criterion,
                        &report_context,
                        input,
                        self.throughput.clone(),
                    );
                }
            }
            Mode::List => {
                if do_run {
                    println!("{}: bench", id);
                }
            }
            Mode::Test => {
                if do_run {
                    // In test mode, run the benchmark exactly once, then exit.
                    self.criterion.report.test_start(&id, &report_context);
                    func.test(&self.criterion.measurement, input);
                    self.criterion.report.test_pass(&id, &report_context);
                }
            }
            Mode::Profile(duration) => {
                if do_run {
                    func.profile(
                        &self.criterion.measurement,
                        &id,
                        self.criterion,
                        &report_context,
                        duration,
                        input,
                    );
                }
            }
        }

        self.all_ids.push(id);
    }

    /// Consume the benchmark group and generate the summary reports for the group.
    ///
    /// It is recommended to call this explicitly, but if you forget it will be called when the
    /// group is dropped.
    pub fn finish(self) {
        ::std::mem::drop(self);
    }
}
impl<'a, M: Measurement> Drop for BenchmarkGroup<'a, M> {
    fn drop(&mut self) {
        // I don't really like having a bunch of non-trivial code in drop, but this is the only way
        // to really write linear types like this in Rust...
        if let Some(conn) = &mut self.criterion.connection {
            conn.send(&OutgoingMessage::FinishedBenchmarkGroup {
                group: &self.group_name,
            })
            .unwrap();

            conn.serve_value_formatter(self.criterion.measurement.formatter())
                .unwrap();
        }

        if self.all_ids.len() > 1 && self.any_matched && self.criterion.mode.is_benchmark() {
            let report_context = ReportContext {
                output_directory: self.criterion.output_directory.clone(),
                plot_config: self.partial_config.plot_config.clone(),
            };

            self.criterion.report.summarize(
                &report_context,
                &self.all_ids,
                self.criterion.measurement.formatter(),
            );
        }
        if self.any_matched {
            self.criterion.report.group_separator();
        }
    }
}

/// Simple structure representing an ID for a benchmark. The ID must be unique within a benchmark
/// group.
#[derive(Clone, Eq, PartialEq, Hash)]
pub struct BenchmarkId {
    pub(crate) function_name: Option<String>,
    pub(crate) parameter: Option<String>,
}
impl BenchmarkId {
    /// Construct a new benchmark ID from a string function name and a parameter value.
    ///
    /// Note that the parameter value need not be the same as the parameter passed to your
    /// actual benchmark. For instance, you might have a benchmark that takes a 1MB string as
    /// input. It would be impractical to embed the whole string in the benchmark ID, so instead
    /// your parameter value might be a descriptive string like "1MB Alphanumeric".
    ///
    /// # Examples
    /// ```
    /// # use criterion::{BenchmarkId, Criterion};
    /// // A basic benchmark ID is typically constructed from a constant string and a simple
    /// // parameter
    /// let basic_id = BenchmarkId::new("my_id", 5);
    ///
    /// // The function name can be a string
    /// let function_name = "test_string".to_string();
    /// let string_id = BenchmarkId::new(function_name, 12);
    ///
    /// // Benchmark IDs are passed to benchmark groups:
    /// let mut criterion = Criterion::default();
    /// let mut group = criterion.benchmark_group("My Group");
    /// // Generate a very large input
    /// let input : String = ::std::iter::repeat("X").take(1024 * 1024).collect();
    ///
    /// // Note that we don't have to use the input as the parameter in the ID
    /// group.bench_with_input(BenchmarkId::new("Test long string", "1MB X's"), &input, |b, i| {
    ///     b.iter(|| i.len())
    /// });
    /// ```
    pub fn new<S: Into<String>, P: ::std::fmt::Display>(
        function_name: S,
        parameter: P,
    ) -> BenchmarkId {
        BenchmarkId {
            function_name: Some(function_name.into()),
            parameter: Some(format!("{}", parameter)),
        }
    }

    /// Construct a new benchmark ID from just a parameter value. Use this when benchmarking a
    /// single function with a variety of different inputs.
    pub fn from_parameter<P: ::std::fmt::Display>(parameter: P) -> BenchmarkId {
        BenchmarkId {
            function_name: None,
            parameter: Some(format!("{}", parameter)),
        }
    }

    pub(crate) fn no_function() -> BenchmarkId {
        BenchmarkId {
            function_name: None,
            parameter: None,
        }
    }

    pub(crate) fn no_function_with_input<P: ::std::fmt::Display>(parameter: P) -> BenchmarkId {
        BenchmarkId {
            function_name: None,
            parameter: Some(format!("{}", parameter)),
        }
    }
}

mod private {
    pub trait Sealed {}
    impl Sealed for super::BenchmarkId {}
    impl<S: Into<String>> Sealed for S {}
}

/// Sealed trait which allows users to automatically convert strings to benchmark IDs.
pub trait IntoBenchmarkId: private::Sealed {
    fn into_benchmark_id(self) -> BenchmarkId;
}
impl IntoBenchmarkId for BenchmarkId {
    fn into_benchmark_id(self) -> BenchmarkId {
        self
    }
}
impl<S: Into<String>> IntoBenchmarkId for S {
    fn into_benchmark_id(self) -> BenchmarkId {
        let function_name = self.into();
        if function_name.is_empty() {
            panic!("Function name must not be empty.");
        }

        BenchmarkId {
            function_name: Some(function_name),
            parameter: None,
        }
    }
}