From 8f2dfbf8e1fa33cc703627bfa5d83b3736a4f060 Mon Sep 17 00:00:00 2001 From: Onsi Fakhouri Date: Wed, 26 May 2021 19:11:34 -0600 Subject: [PATCH] gmeasure provides BETA support for benchmarking (#447) gmeasure is a new gomega subpackage intended to provide measurement and benchmarking support for durations and values. gmeasure replaces Ginkgo V1s deprecated Measure nodes and provides a migration path for users migrating to Ginkgo V2. gmeasure is organized around an Experiment metaphor. Experiments can record several different Measurements, with each Measurement comprised of multiple data points. Measurements can hold time.Durations and float64 values and gmeasure includes support measuring the duraiton of callback functions and for sampling functions repeatedly to build an ensemble of data points. In addition, gmeasure introduces a Stopwatch abtraction for easily measuring and recording durations of code segments. Once measured, users can readily generate Stats for Measurements to capture their key statistics and these stats can be ranked using a Ranking and associated RankingCriteria. Experiments can be Cached to disk to speed up subsequent runs. Experiments are cached by name and version number which makes it easy to manage and bust the cache. Finally, gmeasure integrates with Ginkgo V2 via the new ReportEntry abstraction. Experiments, Measurements, and Rankings can all be registered via AddReportEntry. Doing so generates colorful reports as part of Ginkgo's test output. gmeasure is currently in beta and will go GA around when Ginkgo V2 goes GA. --- gmeasure/cache.go | 201 +++++++++++++ gmeasure/cache_test.go | 109 +++++++ gmeasure/enum_support.go | 43 +++ gmeasure/experiment.go | 518 ++++++++++++++++++++++++++++++++ gmeasure/experiment_test.go | 365 ++++++++++++++++++++++ gmeasure/gmeasure_suite_test.go | 13 + gmeasure/measurement.go | 235 +++++++++++++++ gmeasure/measurement_test.go | 256 ++++++++++++++++ gmeasure/rank.go | 141 +++++++++ gmeasure/rank_test.go | 178 +++++++++++ gmeasure/stats.go | 153 ++++++++++ gmeasure/stats_test.go | 104 +++++++ gmeasure/stopwatch.go | 117 ++++++++ gmeasure/stopwatch_test.go | 66 ++++ gmeasure/table/table.go | 370 +++++++++++++++++++++++ 15 files changed, 2869 insertions(+) create mode 100644 gmeasure/cache.go create mode 100644 gmeasure/cache_test.go create mode 100644 gmeasure/enum_support.go create mode 100644 gmeasure/experiment.go create mode 100644 gmeasure/experiment_test.go create mode 100644 gmeasure/gmeasure_suite_test.go create mode 100644 gmeasure/measurement.go create mode 100644 gmeasure/measurement_test.go create mode 100644 gmeasure/rank.go create mode 100644 gmeasure/rank_test.go create mode 100644 gmeasure/stats.go create mode 100644 gmeasure/stats_test.go create mode 100644 gmeasure/stopwatch.go create mode 100644 gmeasure/stopwatch_test.go create mode 100644 gmeasure/table/table.go diff --git a/gmeasure/cache.go b/gmeasure/cache.go new file mode 100644 index 000000000..bf52b4e18 --- /dev/null +++ b/gmeasure/cache.go @@ -0,0 +1,201 @@ +package gmeasure + +import ( + "crypto/md5" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +const CACHE_EXT = ".gmeasure-cache" + +/* +ExperimentCache provides a director-and-file based cache of experiments +*/ +type ExperimentCache struct { + Path string +} + +/* +NewExperimentCache creates and initializes a new cache. Path must point to a directory (if path does not exist, NewExperimentCache will create a directory at path). + +Cached Experiments are stored as separate files in the cache directory - the filename is a hash of the Experiment name. Each file contains two JSON-encoded objects - a CachedExperimentHeader that includes the experiment's name and cache version number, and then the Experiment itself. +*/ +func NewExperimentCache(path string) (ExperimentCache, error) { + stat, err := os.Stat(path) + if os.IsNotExist(err) { + err := os.MkdirAll(path, 0777) + if err != nil { + return ExperimentCache{}, err + } + } else if !stat.IsDir() { + return ExperimentCache{}, fmt.Errorf("%s is not a directory", path) + } + + return ExperimentCache{ + Path: path, + }, nil +} + +/* +CachedExperimentHeader captures the name of the Cached Experiment and its Version +*/ +type CachedExperimentHeader struct { + Name string + Version int +} + +func (cache ExperimentCache) hashOf(name string) string { + return fmt.Sprintf("%x", md5.Sum([]byte(name))) +} + +func (cache ExperimentCache) readHeader(filename string) (CachedExperimentHeader, error) { + out := CachedExperimentHeader{} + f, err := os.Open(filepath.Join(cache.Path, filename)) + if err != nil { + return out, err + } + defer f.Close() + err = json.NewDecoder(f).Decode(&out) + return out, err +} + +/* +List returns a list of all Cached Experiments found in the cache. +*/ +func (cache ExperimentCache) List() ([]CachedExperimentHeader, error) { + out := []CachedExperimentHeader{} + infos, err := ioutil.ReadDir(cache.Path) + if err != nil { + return out, err + } + for _, info := range infos { + if filepath.Ext(info.Name()) != CACHE_EXT { + continue + } + header, err := cache.readHeader(info.Name()) + if err != nil { + return out, err + } + out = append(out, header) + } + return out, nil +} + +/* +Clear empties out the cache - this will delete any and all detected cache files in the cache directory. Use with caution! +*/ +func (cache ExperimentCache) Clear() error { + infos, err := ioutil.ReadDir(cache.Path) + if err != nil { + return err + } + for _, info := range infos { + if filepath.Ext(info.Name()) != CACHE_EXT { + continue + } + err := os.Remove(filepath.Join(cache.Path, info.Name())) + if err != nil { + return err + } + } + return nil +} + +/* +Load fetches an experiment from the cache. Lookup occurs by name. Load requires that the version numer in the cache is equal to or greater than the passed-in version. + +If an experiment with corresponding name and version >= the passed-in version is found, it is unmarshaled and returned. + +If no experiment is found, or the cached version is smaller than the passed-in version, Load will return nil. + +When paired with Ginkgo you can cache experiments and prevent potentially expensive recomputation with this pattern: + + const EXPERIMENT_VERSION = 1 //bump this to bust the cache and recompute _all_ experiments + + Describe("some experiments", func() { + var cache gmeasure.ExperimentCache + var experiment *gmeasure.Experiment + + BeforeEach(func() { + cache = gmeasure.NewExperimentCache("./gmeasure-cache") + name := CurrentSpecReport().LeafNodeText + experiment = cache.Load(name, EXPERIMENT_VERSION) + if experiment != nil { + AddReportEntry(experiment) + Skip("cached") + } + experiment = gmeasure.NewExperiment(name) + AddReportEntry(experiment) + }) + + It("foo runtime", func() { + experiment.SampleDuration("runtime", func() { + //do stuff + }, gmeasure.SamplingConfig{N:100}) + }) + + It("bar runtime", func() { + experiment.SampleDuration("runtime", func() { + //do stuff + }, gmeasure.SamplingConfig{N:100}) + }) + + AfterEach(func() { + if !CurrentSpecReport().State.Is(types.SpecStateSkipped) { + cache.Save(experiment.Name, EXPERIMENT_VERSION, experiment) + } + }) + }) +*/ +func (cache ExperimentCache) Load(name string, version int) *Experiment { + path := filepath.Join(cache.Path, cache.hashOf(name)+CACHE_EXT) + f, err := os.Open(path) + if err != nil { + return nil + } + defer f.Close() + dec := json.NewDecoder(f) + header := CachedExperimentHeader{} + dec.Decode(&header) + if header.Version < version { + return nil + } + out := NewExperiment("") + err = dec.Decode(out) + if err != nil { + return nil + } + return out +} + +/* +Save stores the passed-in experiment to the cache with the passed-in name and version. +*/ +func (cache ExperimentCache) Save(name string, version int, experiment *Experiment) error { + path := filepath.Join(cache.Path, cache.hashOf(name)+CACHE_EXT) + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + err = enc.Encode(CachedExperimentHeader{ + Name: name, + Version: version, + }) + if err != nil { + return err + } + return enc.Encode(experiment) +} + +/* +Delete removes the experiment with the passed-in name from the cache +*/ +func (cache ExperimentCache) Delete(name string) error { + path := filepath.Join(cache.Path, cache.hashOf(name)+CACHE_EXT) + return os.Remove(path) +} diff --git a/gmeasure/cache_test.go b/gmeasure/cache_test.go new file mode 100644 index 000000000..ca532e5b4 --- /dev/null +++ b/gmeasure/cache_test.go @@ -0,0 +1,109 @@ +package gmeasure_test + +import ( + "fmt" + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gmeasure" +) + +var _ = Describe("Cache", func() { + var path string + var cache gmeasure.ExperimentCache + var e1, e2 *gmeasure.Experiment + + BeforeEach(func() { + var err error + path = fmt.Sprintf("./cache-%d", GinkgoParallelNode()) + cache, err = gmeasure.NewExperimentCache(path) + Ω(err).ShouldNot(HaveOccurred()) + e1 = gmeasure.NewExperiment("Experiment-1") + e1.RecordValue("foo", 32) + e2 = gmeasure.NewExperiment("Experiment-2") + e2.RecordValue("bar", 64) + }) + + AfterEach(func() { + Ω(os.RemoveAll(path)).Should(Succeed()) + }) + + Describe("when creating a cache that points to a file", func() { + It("errors", func() { + f, err := os.Create("cache-temp-file") + Ω(err).ShouldNot(HaveOccurred()) + f.Close() + cache, err := gmeasure.NewExperimentCache("cache-temp-file") + Ω(err).Should(MatchError("cache-temp-file is not a directory")) + Ω(cache).Should(BeZero()) + Ω(os.RemoveAll("cache-temp-file")).Should(Succeed()) + }) + }) + + Describe("the happy path", func() { + It("can save, load, list, delete, and clear the cache", func() { + Ω(cache.Save("e1", 1, e1)).Should(Succeed()) + Ω(cache.Save("e2", 7, e2)).Should(Succeed()) + + Ω(cache.Load("e1", 1)).Should(Equal(e1)) + Ω(cache.Load("e2", 7)).Should(Equal(e2)) + + Ω(cache.List()).Should(ConsistOf( + gmeasure.CachedExperimentHeader{"e1", 1}, + gmeasure.CachedExperimentHeader{"e2", 7}, + )) + + Ω(cache.Delete("e2")).Should(Succeed()) + Ω(cache.Load("e1", 1)).Should(Equal(e1)) + Ω(cache.Load("e2", 7)).Should(BeNil()) + Ω(cache.List()).Should(ConsistOf( + gmeasure.CachedExperimentHeader{"e1", 1}, + )) + + Ω(cache.Clear()).Should(Succeed()) + Ω(cache.List()).Should(BeEmpty()) + Ω(cache.Load("e1", 1)).Should(BeNil()) + Ω(cache.Load("e2", 7)).Should(BeNil()) + }) + }) + + Context("with an empty cache", func() { + It("should list nothing", func() { + Ω(cache.List()).Should(BeEmpty()) + }) + + It("should not error when clearing", func() { + Ω(cache.Clear()).Should(Succeed()) + }) + + It("returs nil when loading a non-existing experiment", func() { + Ω(cache.Load("floop", 17)).Should(BeNil()) + }) + }) + + Describe("version management", func() { + BeforeEach(func() { + Ω(cache.Save("e1", 7, e1)).Should(Succeed()) + }) + + Context("when the cached version is older than the requested version", func() { + It("returns nil", func() { + Ω(cache.Load("e1", 8)).Should(BeNil()) + }) + }) + + Context("when the cached version equals the requested version", func() { + It("returns the cached version", func() { + Ω(cache.Load("e1", 7)).Should(Equal(e1)) + }) + }) + + Context("when the cached version is newer than the requested version", func() { + It("returns the cached version", func() { + Ω(cache.Load("e1", 6)).Should(Equal(e1)) + }) + }) + }) + +}) diff --git a/gmeasure/enum_support.go b/gmeasure/enum_support.go new file mode 100644 index 000000000..86a14180a --- /dev/null +++ b/gmeasure/enum_support.go @@ -0,0 +1,43 @@ +package gmeasure + +import "encoding/json" + +type enumSupport struct { + toString map[uint]string + toEnum map[string]uint + maxEnum uint +} + +func newEnumSupport(toString map[uint]string) enumSupport { + toEnum, maxEnum := map[string]uint{}, uint(0) + for k, v := range toString { + toEnum[v] = k + if maxEnum < k { + maxEnum = k + } + } + return enumSupport{toString: toString, toEnum: toEnum, maxEnum: maxEnum} +} + +func (es enumSupport) String(e uint) string { + if e > es.maxEnum { + return es.toString[0] + } + return es.toString[e] +} + +func (es enumSupport) UnmarshalJSON(b []byte) (uint, error) { + var dec string + if err := json.Unmarshal(b, &dec); err != nil { + return 0, err + } + out := es.toEnum[dec] // if we miss we get 0 which is what we want anyway + return out, nil +} + +func (es enumSupport) MarshalJSON(e uint) ([]byte, error) { + if e == 0 || e > es.maxEnum { + return json.Marshal(nil) + } + return json.Marshal(es.toString[e]) +} diff --git a/gmeasure/experiment.go b/gmeasure/experiment.go new file mode 100644 index 000000000..6081ca331 --- /dev/null +++ b/gmeasure/experiment.go @@ -0,0 +1,518 @@ +/* +Package gomega/gmeasure provides support for benchmarking and measuring code. It is intended as a more robust replacement for Ginkgo V1's Measure nodes. + +**gmeasure IS CURRENTLY IN BETA - THE API MAY CHANGE IN THE NEAR-FUTURE. gmeasure WILL BE CONSIDERED GA WHEN Ginkgo V2 IS GA. + +gmeasure is organized around the metaphor of an Experiment that can record multiple Measurements. A Measurement is a named collection of data points and gmeasure supports +measuring Values (of type float64) and Durations (of type time.Duration). + +Experiments allows the user to record Measurements directly by passing in Values (i.e. float64) or Durations (i.e. time.Duration) +or to measure measurements by passing in functions to measure. When measuring functions Experiments take care of timing the duration of functions (for Duration measurements) +and/or recording returned values (for Value measurements). Experiments also support sampling functions - when told to sample Experiments will run functions repeatedly +and measure and record results. The sampling behavior is configured by passing in a SamplingConfig that can control the maximum number of samples, the maximum duration for sampling (or both) +and the number of concurrent samples to take. + +Measurements can be decorated with additional information. This is supported by passing in special typed decorators when recording measurements. These include: + +- Units("any string") - to attach units to a Value Measurement (Duration Measurements always have units of "duration") +- Style("any Ginkgo color style string") - to attach styling to a Measurement. This styling is used when rendering console information about the measurement in reports. Color style strings are documented at TODO. +- Precision(integer or time.Duration) - to attach precision to a Measurement. This controls how many decimal places to show for Value Measurements and how to round Duration Measurements when rendering them to screen. + +In addition, individual data points in a Measurement can be annotated with an Annotation("any string"). The annotation is associated with the individual data point and is intended to convey additional context about the data point. + +Once measurements are complete, an Experiment can generate a comprehensive report by calling its String() or ColorableString() method. + +Users can also access and analyze the resulting Measurements directly. Use Experiment.Get(NAME) to fetch the Measurement named NAME. This returned struct will have fields containing +all the data points and annotations recorded by the experiment. You can subsequently fetch the Measurement.Stats() to get a Stats struct that contains basic statistical information about the +Measurement (min, max, median, mean, standard deviation). You can order these Stats objects using RankStats() to identify best/worst performers across multpile experiments or measurements. + +gmeasure also supports caching Experiments via an ExperimentCache. The cache supports storing and retreiving experiments by name and version. This allows you to rerun code without +repeating expensive experiments that may not have changed (which can be controlled by the cache version number). It also enables you to compare new experiment runs with older runs to detect +variations in performance/behavior. + +When used with Ginkgo, you can emit experiment reports and encode them in test reports easily using Ginkgo V2's support for Report Entries. +Simply pass your experiment to AddReportEntry to get a report every time the tests run. You can also use AddReportEntry with Measurements to emit all the captured data +and Rankings to emit measurement summaries in rank order. + +Finally, Experiments provide an additional mechanism to measure durations called a Stopwatch. The Stopwatch makes it easy to pepper code with statements that measure elapsed time across +different sections of code and can be useful when debugging or evaluating bottlenecks in a given codepath. +*/ +package gmeasure + +import ( + "fmt" + "math" + "reflect" + "sync" + "time" + + "github.com/onsi/gomega/gmeasure/table" +) + +/* +SamplingConfig configures the Sample family of experiment methods. +These methods invoke passed-in functions repeatedly to sample and record a given measurement. +SamplingConfig is used to control the maximum number of samples or time spent sampling (or both). When both are specified sampling ends as soon as one of the conditions is met. +SamplingConfig can also enable concurrent sampling. +*/ +type SamplingConfig struct { + // N - the maximum number of samples to record + N int + // Duration - the maximum amount of time to spend recording samples + Duration time.Duration + // NumParallel - the number of parallel workers to spin up to record samples. + NumParallel int +} + +// The Units decorator allows you to specify units (an arbitrary string) when recording values. It is ignored when recording durations. +// +// e := gmeasure.NewExperiment("My Experiment") +// e.RecordValue("length", 3.141, gmeasure.Units("inches")) +// +// Units are only set the first time a value of a given name is recorded. In the example above any subsequent calls to e.RecordValue("length", X) will maintain the "inches" units even if a new set of Units("UNIT") are passed in later. +type Units string + +// The Annotation decorator allows you to attach an annotation to a given recorded data-point: +// +// For example: +// +// e := gmeasure.NewExperiment("My Experiment") +// e.RecordValue("length", 3.141, gmeasure.Annotation("bob")) +// e.RecordValue("length", 2.71, gmeasure.Annotation("jane")) +// +// ...will result in a Measurement named "length" that records two values )[3.141, 2.71]) annotation with (["bob", "jane"]) +type Annotation string + +// The Style decorator allows you to associate a style with a measurement. This is used to generate colorful console reports using Ginkgo V2's +// console formatter. Styles are strings in curly brackets that correspond to a color or style. +// +// For example: +// +// e := gmeasure.NewExperiment("My Experiment") +// e.RecordValue("length", 3.141, gmeasure.Style("{{blue}}{{bold}}")) +// e.RecordValue("length", 2.71) +// e.RecordDuration("cooking time", 3 * time.Second, gmeasure.Style("{{red}}{{underline}}")) +// e.RecordDuration("cooking time", 2 * time.Second) +// +// will emit a report with blue bold entries for the length measurement and red underlined entries for the cooking time measurement. +// +// Units are only set the first time a value or duration of a given name is recorded. In the example above any subsequent calls to e.RecordValue("length", X) will maintain the "{{blue}}{{bold}}" style even if a new Style is passed in later. +type Style string + +// The PrecisionBundle decorator controls the rounding of value and duration measurements. See Precision(). +type PrecisionBundle struct { + Duration time.Duration + ValueFormat string +} + +// Precision() allows you to specify the precision of a value or duration measurement - this precision is used when rendering the measurement to screen. +// +// To control the precision of Value measurements, pass Precision an integer. This will denote the number of decimal places to render (equivalen to the format string "%.Nf") +// To control the precision of Duration measurements, pass Precision a time.Duration. Duration measurements will be rounded oo the nearest time.Duration when rendered. +// +// For example: +// +// e := gmeasure.NewExperiment("My Experiment") +// e.RecordValue("length", 3.141, gmeasure.Precision(2)) +// e.RecordValue("length", 2.71) +// e.RecordDuration("cooking time", 3214 * time.Millisecond, gmeasure.Precision(100*time.Millisecond)) +// e.RecordDuration("cooking time", 2623 * time.Millisecond) +func Precision(p interface{}) PrecisionBundle { + out := DefaultPrecisionBundle + switch reflect.TypeOf(p) { + case reflect.TypeOf(time.Duration(0)): + out.Duration = p.(time.Duration) + case reflect.TypeOf(int(0)): + out.ValueFormat = fmt.Sprintf("%%.%df", p.(int)) + default: + panic("invalid precision type, must be time.Duration or int") + } + return out +} + +// DefaultPrecisionBundle captures the default precisions for Vale and Duration measurements. +var DefaultPrecisionBundle = PrecisionBundle{ + Duration: 100 * time.Microsecond, + ValueFormat: "%.3f", +} + +type extractedDecorations struct { + annotation Annotation + units Units + precisionBundle PrecisionBundle + style Style +} + +func extractDecorations(args []interface{}) extractedDecorations { + var out extractedDecorations + out.precisionBundle = DefaultPrecisionBundle + + for _, arg := range args { + switch reflect.TypeOf(arg) { + case reflect.TypeOf(out.annotation): + out.annotation = arg.(Annotation) + case reflect.TypeOf(out.units): + out.units = arg.(Units) + case reflect.TypeOf(out.precisionBundle): + out.precisionBundle = arg.(PrecisionBundle) + case reflect.TypeOf(out.style): + out.style = arg.(Style) + default: + panic(fmt.Sprintf("unrecognized argument %#v", arg)) + } + } + + return out +} + +/* +Experiment is gmeasure's core data type. You use experiments to record Measurements and generate reports. +Experiments are thread-safe and all methods can be called from multiple goroutines. +*/ +type Experiment struct { + Name string + + // Measurements includes all Measurements recorded by this experiment. You should access them by name via Get() and GetStats() + Measurements Measurements + lock *sync.Mutex +} + +/* +NexExperiment creates a new experiment with the passed-in name. + +When using Ginkgo we recommend immediately registering the experiment as a ReportEntry: + + experiment = NewExperiment("My Experiment") + AddReportEntry(experiment.Name, experiment) + +this will ensure an experiment report is emitted as part of the test output and exported with any test reports. +*/ +func NewExperiment(name string) *Experiment { + experiment := &Experiment{ + Name: name, + lock: &sync.Mutex{}, + } + return experiment +} + +func (e *Experiment) report(enableStyling bool) string { + t := table.NewTable() + t.TableStyle.EnableTextStyling = enableStyling + t.AppendRow(table.R( + table.C("Name"), table.C("N"), table.C("Min"), table.C("Median"), table.C("Mean"), table.C("StdDev"), table.C("Max"), + table.Divider("="), + "{{bold}}", + )) + + for _, measurement := range e.Measurements { + r := table.R(measurement.Style) + t.AppendRow(r) + switch measurement.Type { + case MeasurementTypeNote: + r.AppendCell(table.C(measurement.Note)) + case MeasurementTypeValue, MeasurementTypeDuration: + name := measurement.Name + if measurement.Units != "" { + name += " [" + measurement.Units + "]" + } + r.AppendCell(table.C(name)) + r.AppendCell(measurement.Stats().cells()...) + } + } + + out := e.Name + "\n" + if enableStyling { + out = "{{bold}}" + out + "{{/}}" + } + out += t.Render() + return out +} + +/* +ColorableString returns a Ginkgo formatted summary of the experiment and all its Measurements. +It is called automatically by Ginkgo's reporting infrastructure when the Experiment is registered as a ReportEntry via AddReportEntry. +*/ +func (e *Experiment) ColorableString() string { + return e.report(true) +} + +/* +ColorableString returns an unformatted summary of the experiment and all its Measurements. +*/ +func (e *Experiment) String() string { + return e.report(false) +} + +/* +RecordNote records a Measurement of type MeasurementTypeNote - this is simply a textual note to annotate the experiment. It will be emitted in any experiment reports. + +RecordNote supports the Style() decoration. +*/ +func (e *Experiment) RecordNote(note string, args ...interface{}) { + decorations := extractDecorations(args) + + e.lock.Lock() + defer e.lock.Unlock() + e.Measurements = append(e.Measurements, Measurement{ + ExperimentName: e.Name, + Type: MeasurementTypeNote, + Note: note, + Style: string(decorations.style), + }) +} + +/* +RecordDuration records the passed-in duration on a Duration Measurement with the passed-in name. If the Measurement does not exist it is created. + +RecordDuration supports the Style(), Precision(), and Annotation() decorations. +*/ +func (e *Experiment) RecordDuration(name string, duration time.Duration, args ...interface{}) { + decorations := extractDecorations(args) + e.recordDuration(name, duration, decorations) +} + +/* +MeasureDuration runs the passed-in callback and times how long it takes to complete. The resulting duration is recorded on a Duration Measurement with the passed-in name. If the Measurement does not exist it is created. + +MeasureDuration supports the Style(), Precision(), and Annotation() decorations. +*/ +func (e *Experiment) MeasureDuration(name string, callback func(), args ...interface{}) time.Duration { + t := time.Now() + callback() + duration := time.Since(t) + e.RecordDuration(name, duration, args...) + return duration +} + +/* +SampleDuration samples the passed-in callback and times how long it takes to complete each sample. +The resulting durations are recorded on a Duration Measurement with the passed-in name. If the Measurement does not exist it is created. + +The callback is given a zero-based index that increments by one between samples. The Sampling is configured via the passed-in SamplingConfig + +SampleDuration supports the Style(), Precision(), and Annotation() decorations. When passed an Annotation() the same annotation is applied to all sample measurements. +*/ +func (e *Experiment) SampleDuration(name string, callback func(idx int), samplingConfig SamplingConfig, args ...interface{}) { + decorations := extractDecorations(args) + e.Sample(func(idx int) { + t := time.Now() + callback(idx) + duration := time.Since(t) + e.recordDuration(name, duration, decorations) + }, samplingConfig) +} + +/* +SampleDuration samples the passed-in callback and times how long it takes to complete each sample. +The resulting durations are recorded on a Duration Measurement with the passed-in name. If the Measurement does not exist it is created. + +The callback is given a zero-based index that increments by one between samples. The callback must return an Annotation - this annotation is attached to the measured duration. + +The Sampling is configured via the passed-in SamplingConfig + +SampleAnnotatedDuration supports the Style() and Precision() decorations. +*/ +func (e *Experiment) SampleAnnotatedDuration(name string, callback func(idx int) Annotation, samplingConfig SamplingConfig, args ...interface{}) { + decorations := extractDecorations(args) + e.Sample(func(idx int) { + t := time.Now() + decorations.annotation = callback(idx) + duration := time.Since(t) + e.recordDuration(name, duration, decorations) + }, samplingConfig) +} + +func (e *Experiment) recordDuration(name string, duration time.Duration, decorations extractedDecorations) { + e.lock.Lock() + defer e.lock.Unlock() + idx := e.Measurements.IdxWithName(name) + if idx == -1 { + measurement := Measurement{ + ExperimentName: e.Name, + Type: MeasurementTypeDuration, + Name: name, + Units: "duration", + Durations: []time.Duration{duration}, + PrecisionBundle: decorations.precisionBundle, + Style: string(decorations.style), + Annotations: []string{string(decorations.annotation)}, + } + e.Measurements = append(e.Measurements, measurement) + } else { + if e.Measurements[idx].Type != MeasurementTypeDuration { + panic(fmt.Sprintf("attempting to record duration with name '%s'. That name is already in-use for recording values.", name)) + } + e.Measurements[idx].Durations = append(e.Measurements[idx].Durations, duration) + e.Measurements[idx].Annotations = append(e.Measurements[idx].Annotations, string(decorations.annotation)) + } +} + +/* +NewStopwatch() returns a stopwatch configured to record duration measurements with this experiment. +*/ +func (e *Experiment) NewStopwatch() *Stopwatch { + return newStopwatch(e) +} + +/* +RecordValue records the passed-in value on a Value Measurement with the passed-in name. If the Measurement does not exist it is created. + +RecordValue supports the Style(), Units(), Precision(), and Annotation() decorations. +*/ +func (e *Experiment) RecordValue(name string, value float64, args ...interface{}) { + decorations := extractDecorations(args) + e.recordValue(name, value, decorations) +} + +/* +MeasureValue runs the passed-in callback and records the return value on a Value Measurement with the passed-in name. If the Measurement does not exist it is created. + +MeasureValue supports the Style(), Units(), Precision(), and Annotation() decorations. +*/ +func (e *Experiment) MeasureValue(name string, callback func() float64, args ...interface{}) float64 { + value := callback() + e.RecordValue(name, value, args...) + return value +} + +/* +SampleValue samples the passed-in callback and records the return value on a Value Measurement with the passed-in name. If the Measurement does not exist it is created. + +The callback is given a zero-based index that increments by one between samples. The callback must return a float64. The Sampling is configured via the passed-in SamplingConfig + +SampleValue supports the Style(), Units(), Precision(), and Annotation() decorations. When passed an Annotation() the same annotation is applied to all sample measurements. +*/ +func (e *Experiment) SampleValue(name string, callback func(idx int) float64, samplingConfig SamplingConfig, args ...interface{}) { + decorations := extractDecorations(args) + e.Sample(func(idx int) { + value := callback(idx) + e.recordValue(name, value, decorations) + }, samplingConfig) +} + +/* +SampleAnnotatedValue samples the passed-in callback and records the return value on a Value Measurement with the passed-in name. If the Measurement does not exist it is created. + +The callback is given a zero-based index that increments by one between samples. The callback must return a float64 and an Annotation - the annotation is attached to the recorded value. + +The Sampling is configured via the passed-in SamplingConfig + +SampleValue supports the Style(), Units(), and Precision() decorations. +*/ +func (e *Experiment) SampleAnnotatedValue(name string, callback func(idx int) (float64, Annotation), samplingConfig SamplingConfig, args ...interface{}) { + decorations := extractDecorations(args) + e.Sample(func(idx int) { + var value float64 + value, decorations.annotation = callback(idx) + e.recordValue(name, value, decorations) + }, samplingConfig) +} + +func (e *Experiment) recordValue(name string, value float64, decorations extractedDecorations) { + e.lock.Lock() + defer e.lock.Unlock() + idx := e.Measurements.IdxWithName(name) + if idx == -1 { + measurement := Measurement{ + ExperimentName: e.Name, + Type: MeasurementTypeValue, + Name: name, + Style: string(decorations.style), + Units: string(decorations.units), + PrecisionBundle: decorations.precisionBundle, + Values: []float64{value}, + Annotations: []string{string(decorations.annotation)}, + } + e.Measurements = append(e.Measurements, measurement) + } else { + if e.Measurements[idx].Type != MeasurementTypeValue { + panic(fmt.Sprintf("attempting to record value with name '%s'. That name is already in-use for recording durations.", name)) + } + e.Measurements[idx].Values = append(e.Measurements[idx].Values, value) + e.Measurements[idx].Annotations = append(e.Measurements[idx].Annotations, string(decorations.annotation)) + } +} + +/* +Sample samples the passed-in callback repeatedly. The sampling is governed by the passed in SamplingConfig. + +The SamplingConfig can limit the total number of samples and/or the total time spent sampling the callback. +The SamplingConfig can also instruct Sample to run with multiple concurrent workers. + +The callback is called with a zero-based index that incerements by one between samples. +*/ +func (e *Experiment) Sample(callback func(idx int), samplingConfig SamplingConfig) { + if samplingConfig.N == 0 && samplingConfig.Duration == 0 { + panic("you must specify at least one of SamplingConfig.N and SamplingConfig.Duration") + } + maxTime := time.Now().Add(100000 * time.Hour) + if samplingConfig.Duration > 0 { + maxTime = time.Now().Add(samplingConfig.Duration) + } + maxN := math.MaxInt64 + if samplingConfig.N > 0 { + maxN = samplingConfig.N + } + numParallel := 1 + if samplingConfig.NumParallel > numParallel { + numParallel = samplingConfig.NumParallel + } + + work := make(chan int) + if numParallel > 1 { + for worker := 0; worker < numParallel; worker++ { + go func() { + for idx := range work { + callback(idx) + } + }() + } + } + + idx := 0 + var avgDt time.Duration + for { + t := time.Now() + if numParallel > 1 { + work <- idx + } else { + callback(idx) + } + dt := time.Since(t) + if idx >= numParallel { + avgDt = (avgDt*time.Duration(idx-numParallel) + dt) / time.Duration(idx-numParallel+1) + } + idx += 1 + if idx >= maxN { + return + } + if time.Now().Add(avgDt).After(maxTime) { + return + } + } +} + +/* +Get returns the Measurement with the associated name. If no Measurement is found a zero Measurement{} is returned. +*/ +func (e *Experiment) Get(name string) Measurement { + e.lock.Lock() + defer e.lock.Unlock() + idx := e.Measurements.IdxWithName(name) + if idx == -1 { + return Measurement{} + } + return e.Measurements[idx] +} + +/* +GetStats returns the Stats for the Measurement with the associated name. If no Measurement is found a zero Stats{} is returned. + +experiment.GetStats(name) is equivalent to experiment.Get(name).Stats() +*/ +func (e *Experiment) GetStats(name string) Stats { + measurement := e.Get(name) + e.lock.Lock() + defer e.lock.Unlock() + return measurement.Stats() +} diff --git a/gmeasure/experiment_test.go b/gmeasure/experiment_test.go new file mode 100644 index 000000000..8408354b7 --- /dev/null +++ b/gmeasure/experiment_test.go @@ -0,0 +1,365 @@ +package gmeasure_test + +import ( + "fmt" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/onsi/gomega/gmeasure" +) + +var _ = Describe("Experiment", func() { + var e *gmeasure.Experiment + BeforeEach(func() { + e = gmeasure.NewExperiment("Test Experiment") + }) + + Describe("Recording Notes", func() { + It("creates a note Measurement", func() { + e.RecordNote("I'm a note", gmeasure.Style("{{blue}}")) + measurement := e.Measurements[0] + Ω(measurement.Type).Should(Equal(gmeasure.MeasurementTypeNote)) + Ω(measurement.ExperimentName).Should(Equal("Test Experiment")) + Ω(measurement.Note).Should(Equal("I'm a note")) + Ω(measurement.Style).Should(Equal("{{blue}}")) + }) + }) + + Describe("Recording Durations", func() { + commonMeasurementAssertions := func() gmeasure.Measurement { + measurement := e.Get("runtime") + Ω(measurement.Type).Should(Equal(gmeasure.MeasurementTypeDuration)) + Ω(measurement.ExperimentName).Should(Equal("Test Experiment")) + Ω(measurement.Name).Should(Equal("runtime")) + Ω(measurement.Units).Should(Equal("duration")) + Ω(measurement.Style).Should(Equal("{{red}}")) + Ω(measurement.PrecisionBundle.Duration).Should(Equal(time.Millisecond)) + return measurement + } + + BeforeEach(func() { + e.RecordDuration("runtime", time.Second, gmeasure.Annotation("first"), gmeasure.Style("{{red}}"), gmeasure.Precision(time.Millisecond), gmeasure.Units("ignored")) + }) + + Describe("RecordDuration", func() { + It("generates a measurement and records the passed-in duration along with any relevant decorations", func() { + e.RecordDuration("runtime", time.Minute, gmeasure.Annotation("second")) + measurement := commonMeasurementAssertions() + Ω(measurement.Durations).Should(Equal([]time.Duration{time.Second, time.Minute})) + Ω(measurement.Annotations).Should(Equal([]string{"first", "second"})) + }) + }) + + Describe("MeasureDuration", func() { + It("measure the duration of the passed-in function", func() { + e.MeasureDuration("runtime", func() { + time.Sleep(200 * time.Millisecond) + }, gmeasure.Annotation("second")) + measurement := commonMeasurementAssertions() + Ω(measurement.Durations[0]).Should(Equal(time.Second)) + Ω(measurement.Durations[1]).Should(BeNumerically("~", 200*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Annotations).Should(Equal([]string{"first", "second"})) + }) + }) + + Describe("SampleDuration", func() { + It("samples the passed-in function according to SampleConfig and records the measured durations", func() { + e.SampleDuration("runtime", func(_ int) { + time.Sleep(100 * time.Millisecond) + }, gmeasure.SamplingConfig{N: 3}, gmeasure.Annotation("sampled")) + measurement := commonMeasurementAssertions() + Ω(measurement.Durations[0]).Should(Equal(time.Second)) + Ω(measurement.Durations[1]).Should(BeNumerically("~", 100*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Durations[2]).Should(BeNumerically("~", 100*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Durations[3]).Should(BeNumerically("~", 100*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Annotations).Should(Equal([]string{"first", "sampled", "sampled", "sampled"})) + }) + }) + + Describe("SampleAnnotatedDuration", func() { + It("samples the passed-in function according to SampleConfig and records the measured durations and returned annotations", func() { + e.SampleAnnotatedDuration("runtime", func(idx int) gmeasure.Annotation { + time.Sleep(100 * time.Millisecond) + return gmeasure.Annotation(fmt.Sprintf("sampled-%d", idx+1)) + }, gmeasure.SamplingConfig{N: 3}, gmeasure.Annotation("ignored")) + measurement := commonMeasurementAssertions() + Ω(measurement.Durations[0]).Should(Equal(time.Second)) + Ω(measurement.Durations[1]).Should(BeNumerically("~", 100*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Durations[2]).Should(BeNumerically("~", 100*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Durations[3]).Should(BeNumerically("~", 100*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Annotations).Should(Equal([]string{"first", "sampled-1", "sampled-2", "sampled-3"})) + }) + }) + }) + + Describe("Stopwatch Support", func() { + It("can generate a new stopwatch tied to the experiment", func() { + s := e.NewStopwatch() + time.Sleep(50 * time.Millisecond) + s.Record("runtime", gmeasure.Annotation("first")).Reset() + time.Sleep(100 * time.Millisecond) + s.Record("runtime", gmeasure.Annotation("second")).Reset() + time.Sleep(150 * time.Millisecond) + s.Record("runtime", gmeasure.Annotation("third")) + measurement := e.Get("runtime") + Ω(measurement.Durations[0]).Should(BeNumerically("~", 50*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Durations[1]).Should(BeNumerically("~", 100*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Durations[2]).Should(BeNumerically("~", 150*time.Millisecond, 20*time.Millisecond)) + Ω(measurement.Annotations).Should(Equal([]string{"first", "second", "third"})) + }) + }) + + Describe("Recording Values", func() { + commonMeasurementAssertions := func() gmeasure.Measurement { + measurement := e.Get("sprockets") + Ω(measurement.Type).Should(Equal(gmeasure.MeasurementTypeValue)) + Ω(measurement.ExperimentName).Should(Equal("Test Experiment")) + Ω(measurement.Name).Should(Equal("sprockets")) + Ω(measurement.Units).Should(Equal("widgets")) + Ω(measurement.Style).Should(Equal("{{yellow}}")) + Ω(measurement.PrecisionBundle.ValueFormat).Should(Equal("%.0f")) + return measurement + } + + BeforeEach(func() { + e.RecordValue("sprockets", 3.2, gmeasure.Annotation("first"), gmeasure.Style("{{yellow}}"), gmeasure.Precision(0), gmeasure.Units("widgets")) + }) + + Describe("RecordValue", func() { + It("generates a measurement and records the passed-in value along with any relevant decorations", func() { + e.RecordValue("sprockets", 17.4, gmeasure.Annotation("second")) + measurement := commonMeasurementAssertions() + Ω(measurement.Values).Should(Equal([]float64{3.2, 17.4})) + Ω(measurement.Annotations).Should(Equal([]string{"first", "second"})) + }) + }) + + Describe("MeasureValue", func() { + It("records the value returned by the passed-in function", func() { + e.MeasureValue("sprockets", func() float64 { + return 17.4 + }, gmeasure.Annotation("second")) + measurement := commonMeasurementAssertions() + Ω(measurement.Values).Should(Equal([]float64{3.2, 17.4})) + Ω(measurement.Annotations).Should(Equal([]string{"first", "second"})) + }) + }) + + Describe("SampleValue", func() { + It("samples the passed-in function according to SampleConfig and records the resulting values", func() { + e.SampleValue("sprockets", func(idx int) float64 { + return 17.4 + float64(idx) + }, gmeasure.SamplingConfig{N: 3}, gmeasure.Annotation("sampled")) + measurement := commonMeasurementAssertions() + Ω(measurement.Values).Should(Equal([]float64{3.2, 17.4, 18.4, 19.4})) + Ω(measurement.Annotations).Should(Equal([]string{"first", "sampled", "sampled", "sampled"})) + }) + }) + + Describe("SampleAnnotatedValue", func() { + It("samples the passed-in function according to SampleConfig and records the returned values and annotations", func() { + e.SampleAnnotatedValue("sprockets", func(idx int) (float64, gmeasure.Annotation) { + return 17.4 + float64(idx), gmeasure.Annotation(fmt.Sprintf("sampled-%d", idx+1)) + }, gmeasure.SamplingConfig{N: 3}, gmeasure.Annotation("ignored")) + measurement := commonMeasurementAssertions() + Ω(measurement.Values).Should(Equal([]float64{3.2, 17.4, 18.4, 19.4})) + Ω(measurement.Annotations).Should(Equal([]string{"first", "sampled-1", "sampled-2", "sampled-3"})) + }) + }) + }) + + Describe("Sampling", func() { + var indices []int + BeforeEach(func() { + indices = []int{} + }) + + ints := func(n int) []int { + out := []int{} + for i := 0; i < n; i++ { + out = append(out, i) + } + return out + } + + It("calls the function repeatedly passing in an index", func() { + e.Sample(func(idx int) { + indices = append(indices, idx) + }, gmeasure.SamplingConfig{N: 3}) + + Ω(indices).Should(Equal(ints(3))) + }) + + It("can cap the maximum number of samples", func() { + e.Sample(func(idx int) { + indices = append(indices, idx) + }, gmeasure.SamplingConfig{N: 10, Duration: time.Minute}) + + Ω(indices).Should(Equal(ints(10))) + }) + + It("can cap the maximum sample time", func() { + e.Sample(func(idx int) { + indices = append(indices, idx) + time.Sleep(10 * time.Millisecond) + }, gmeasure.SamplingConfig{N: 100, Duration: 100 * time.Millisecond}) + + Ω(len(indices)).Should(BeNumerically("~", 10, 3)) + Ω(indices).Should(Equal(ints(len(indices)))) + }) + + It("can run samples in parallel", func() { + lock := &sync.Mutex{} + + e.Sample(func(idx int) { + lock.Lock() + indices = append(indices, idx) + lock.Unlock() + time.Sleep(10 * time.Millisecond) + }, gmeasure.SamplingConfig{N: 100, Duration: 100 * time.Millisecond, NumParallel: 3}) + + Ω(len(indices)).Should(BeNumerically("~", 30, 10)) + Ω(indices).Should(ConsistOf(ints(len(indices)))) + }) + + It("panics if the SamplingConfig is misconfigured", func() { + Expect(func() { + e.Sample(func(_ int) {}, gmeasure.SamplingConfig{}) + }).To(PanicWith("you must specify at least one of SamplingConfig.N and SamplingConfig.Duration")) + }) + }) + + Describe("recording multiple entries", func() { + It("always appends to the correct measurement (by name)", func() { + e.RecordDuration("alpha", time.Second) + e.RecordDuration("beta", time.Minute) + e.RecordValue("gamma", 1) + e.RecordValue("delta", 2.71) + e.RecordDuration("alpha", 2*time.Second) + e.RecordDuration("beta", 2*time.Minute) + e.RecordValue("gamma", 2) + e.RecordValue("delta", 3.141) + + Ω(e.Measurements).Should(HaveLen(4)) + Ω(e.Get("alpha").Durations).Should(Equal([]time.Duration{time.Second, 2 * time.Second})) + Ω(e.Get("beta").Durations).Should(Equal([]time.Duration{time.Minute, 2 * time.Minute})) + Ω(e.Get("gamma").Values).Should(Equal([]float64{1, 2})) + Ω(e.Get("delta").Values).Should(Equal([]float64{2.71, 3.141})) + }) + + It("panics if you incorrectly mix types", func() { + e.RecordDuration("runtime", time.Second) + Ω(func() { + e.RecordValue("runtime", 3.141) + }).Should(PanicWith("attempting to record value with name 'runtime'. That name is already in-use for recording durations.")) + + e.RecordValue("sprockets", 2) + Ω(func() { + e.RecordDuration("sprockets", time.Minute) + }).Should(PanicWith("attempting to record duration with name 'sprockets'. That name is already in-use for recording values.")) + }) + }) + + Describe("Decorators", func() { + It("uses the default precisions when none is specified", func() { + e.RecordValue("sprockets", 2) + e.RecordDuration("runtime", time.Minute) + + Ω(e.Get("sprockets").PrecisionBundle.ValueFormat).Should(Equal("%.3f")) + Ω(e.Get("runtime").PrecisionBundle.Duration).Should(Equal(100 * time.Microsecond)) + }) + + It("panics if an unsupported type is passed into Precision", func() { + Ω(func() { + gmeasure.Precision("aardvark") + }).Should(PanicWith("invalid precision type, must be time.Duration or int")) + }) + + It("panics if an unrecognized argumnet is passed in", func() { + Ω(func() { + e.RecordValue("sprockets", 2, "boom") + }).Should(PanicWith(`unrecognized argument "boom"`)) + }) + }) + + Describe("Getting Measurements", func() { + Context("when the Measurement does not exist", func() { + It("returns the zero Measurement", func() { + Ω(e.Get("not here")).Should(BeZero()) + }) + }) + }) + + Describe("Getting Stats", func() { + It("returns the Measurement's Stats", func() { + e.RecordValue("alpha", 1) + e.RecordValue("alpha", 2) + e.RecordValue("alpha", 3) + Ω(e.GetStats("alpha")).Should(Equal(e.Get("alpha").Stats())) + }) + }) + + Describe("Generating Reports", func() { + BeforeEach(func() { + e.RecordNote("A note") + e.RecordValue("sprockets", 7, gmeasure.Units("widgets"), gmeasure.Precision(0), gmeasure.Style("{{yellow}}"), gmeasure.Annotation("sprockets-1")) + e.RecordDuration("runtime", time.Second, gmeasure.Precision(100*time.Millisecond), gmeasure.Style("{{red}}"), gmeasure.Annotation("runtime-1")) + e.RecordNote("A blue note", gmeasure.Style("{{blue}}")) + e.RecordValue("gear ratio", 10.3, gmeasure.Precision(2), gmeasure.Style("{{green}}"), gmeasure.Annotation("ratio-1")) + + e.RecordValue("sprockets", 8, gmeasure.Annotation("sprockets-2")) + e.RecordValue("sprockets", 9, gmeasure.Annotation("sprockets-3")) + + e.RecordDuration("runtime", 2*time.Second, gmeasure.Annotation("runtime-2")) + e.RecordValue("gear ratio", 13.758, gmeasure.Precision(2), gmeasure.Annotation("ratio-2")) + }) + + It("emits a nicely formatted table", func() { + expected := strings.Join([]string{ + "Test Experiment", + "Name | N | Min | Median | Mean | StdDev | Max ", + "=============================================================================", + "A note ", + "-----------------------------------------------------------------------------", + "sprockets [widgets] | 3 | 7 | 8 | 8 | 1 | 9 ", + " | | sprockets-1 | | | | sprockets-3", + "-----------------------------------------------------------------------------", + "runtime [duration] | 2 | 1s | 1.5s | 1.5s | 500ms | 2s ", + " | | runtime-1 | | | | runtime-2 ", + "-----------------------------------------------------------------------------", + "A blue note ", + "-----------------------------------------------------------------------------", + "gear ratio | 2 | 10.30 | 12.03 | 12.03 | 1.73 | 13.76 ", + " | | ratio-1 | | | | ratio-2 ", + "", + }, "\n") + Ω(e.String()).Should(Equal(expected)) + }) + + It("can also emit a styled table", func() { + expected := strings.Join([]string{ + "{{bold}}Test Experiment", + "{{/}}{{bold}}Name {{/}} | {{bold}}N{{/}} | {{bold}}Min {{/}} | {{bold}}Median{{/}} | {{bold}}Mean {{/}} | {{bold}}StdDev{{/}} | {{bold}}Max {{/}}", + "=============================================================================", + "A note ", + "-----------------------------------------------------------------------------", + "{{yellow}}sprockets [widgets]{{/}} | {{yellow}}3{{/}} | {{yellow}}7 {{/}} | {{yellow}}8 {{/}} | {{yellow}}8 {{/}} | {{yellow}}1 {{/}} | {{yellow}}9 {{/}}", + " | | {{yellow}}sprockets-1{{/}} | | | | {{yellow}}sprockets-3{{/}}", + "-----------------------------------------------------------------------------", + "{{red}}runtime [duration] {{/}} | {{red}}2{{/}} | {{red}}1s {{/}} | {{red}}1.5s {{/}} | {{red}}1.5s {{/}} | {{red}}500ms {{/}} | {{red}}2s {{/}}", + " | | {{red}}runtime-1 {{/}} | | | | {{red}}runtime-2 {{/}}", + "-----------------------------------------------------------------------------", + "{{blue}}A blue note {{/}}", + "-----------------------------------------------------------------------------", + "{{green}}gear ratio {{/}} | {{green}}2{{/}} | {{green}}10.30 {{/}} | {{green}}12.03 {{/}} | {{green}}12.03{{/}} | {{green}}1.73 {{/}} | {{green}}13.76 {{/}}", + " | | {{green}}ratio-1 {{/}} | | | | {{green}}ratio-2 {{/}}", + "", + }, "\n") + Ω(e.ColorableString()).Should(Equal(expected)) + }) + }) +}) diff --git a/gmeasure/gmeasure_suite_test.go b/gmeasure/gmeasure_suite_test.go new file mode 100644 index 000000000..ab89fb2d0 --- /dev/null +++ b/gmeasure/gmeasure_suite_test.go @@ -0,0 +1,13 @@ +package gmeasure_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestGmeasure(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Gmeasure Suite") +} diff --git a/gmeasure/measurement.go b/gmeasure/measurement.go new file mode 100644 index 000000000..27bf8ae56 --- /dev/null +++ b/gmeasure/measurement.go @@ -0,0 +1,235 @@ +package gmeasure + +import ( + "fmt" + "math" + "sort" + "time" + + "github.com/onsi/gomega/gmeasure/table" +) + +type MeasurementType uint + +const ( + MeasurementTypeInvalid MeasurementType = iota + MeasurementTypeNote + MeasurementTypeDuration + MeasurementTypeValue +) + +var letEnumSupport = newEnumSupport(map[uint]string{uint(MeasurementTypeInvalid): "INVALID LOG ENTRY TYPE", uint(MeasurementTypeNote): "Note", uint(MeasurementTypeDuration): "Duration", uint(MeasurementTypeValue): "Value"}) + +func (s MeasurementType) String() string { return letEnumSupport.String(uint(s)) } +func (s *MeasurementType) UnmarshalJSON(b []byte) error { + out, err := letEnumSupport.UnmarshalJSON(b) + *s = MeasurementType(out) + return err +} +func (s MeasurementType) MarshalJSON() ([]byte, error) { return letEnumSupport.MarshalJSON(uint(s)) } + +/* +Measurement records all captured data for a given measurement. You generally don't make Measurements directly - but you can fetch them from Experiments using Get(). + +When using Ginkgo, you can register Measurements as Report Entries via AddReportEntry. This will emit all the captured data points when Ginkgo generates the report. +*/ +type Measurement struct { + // Type is the MeasurementType - one of MeasurementTypeNote, MeasurementTypeDuration, or MeasurementTypeValue + Type MeasurementType + + // ExperimentName is the name of the experiment that this Measurement is associated with + ExperimentName string + + // If Type is MeasurementTypeNote, Note is populated with the note text. + Note string + + // If Type is MeasurementTypeDuration or MeasurementTypeValue, Name is the name of the recorded measurement + Name string + + // Style captures the styling information (if any) for this Measurement + Style string + + // Units capture the units (if any) for this Measurement. Units is set to "duration" if the Type is MeasurementTypeDuration + Units string + + // PrecisionBundle captures the precision to use when rendering data for this Measurement. + // If Type is MeasurementTypeDuration then PrecisionBundle.Duration is used to round any durations before presentation. + // If Type is MeasurementTypeValue then PrecisionBundle.ValueFormat is used to format any values before presentation + PrecisionBundle PrecisionBundle + + // If Type is MeasurementTypeDuration, Durations will contain all durations recorded for this measurement + Durations []time.Duration + + // If Type is MeasurementTypeValue, Values will contain all float64s recorded for this measurement + Values []float64 + + // If Type is MeasurementTypeDuration or MeasurementTypeValue then Annotations will include string annotations for all recorded Durations or Values. + // If the user does not pass-in an Annotation() decoration for a particular value or duration, the corresponding entry in the Annotations slice will be the empty string "" + Annotations []string +} + +type Measurements []Measurement + +func (m Measurements) IdxWithName(name string) int { + for idx, measurement := range m { + if measurement.Name == name { + return idx + } + } + + return -1 +} + +func (m Measurement) report(enableStyling bool) string { + out := "" + style := m.Style + if !enableStyling { + style = "" + } + switch m.Type { + case MeasurementTypeNote: + out += fmt.Sprintf("%s - Note\n%s\n", m.ExperimentName, m.Note) + if style != "" { + out = style + out + "{{/}}" + } + return out + case MeasurementTypeValue, MeasurementTypeDuration: + out += fmt.Sprintf("%s - %s", m.ExperimentName, m.Name) + if m.Units != "" { + out += " [" + m.Units + "]" + } + if style != "" { + out = style + out + "{{/}}" + } + out += "\n" + out += m.Stats().String() + "\n" + } + t := table.NewTable() + t.TableStyle.EnableTextStyling = enableStyling + switch m.Type { + case MeasurementTypeValue: + t.AppendRow(table.R(table.C("Value", table.AlignTypeCenter), table.C("Annotation", table.AlignTypeCenter), table.Divider("="), style)) + for idx := range m.Values { + t.AppendRow(table.R( + table.C(fmt.Sprintf(m.PrecisionBundle.ValueFormat, m.Values[idx]), table.AlignTypeRight), + table.C(m.Annotations[idx], "{{gray}}", table.AlignTypeLeft), + )) + } + case MeasurementTypeDuration: + t.AppendRow(table.R(table.C("Duration", table.AlignTypeCenter), table.C("Annotation", table.AlignTypeCenter), table.Divider("="), style)) + for idx := range m.Durations { + t.AppendRow(table.R( + table.C(m.Durations[idx].Round(m.PrecisionBundle.Duration).String(), style, table.AlignTypeRight), + table.C(m.Annotations[idx], "{{gray}}", table.AlignTypeLeft), + )) + } + } + out += t.Render() + return out +} + +/* +ColorableString generates a styled report that includes all the data points for this Measurement. +It is called automatically by Ginkgo's reporting infrastructure when the Measurement is registered as a ReportEntry via AddReportEntry. +*/ +func (m Measurement) ColorableString() string { + return m.report(true) +} + +/* +String generates an unstyled report that includes all the data points for this Measurement. +*/ +func (m Measurement) String() string { + return m.report(false) +} + +/* +Stats returns a Stats struct summarizing the statistic of this measurement +*/ +func (m Measurement) Stats() Stats { + if m.Type == MeasurementTypeInvalid || m.Type == MeasurementTypeNote { + return Stats{} + } + + out := Stats{ + ExperimentName: m.ExperimentName, + MeasurementName: m.Name, + Style: m.Style, + Units: m.Units, + PrecisionBundle: m.PrecisionBundle, + } + + switch m.Type { + case MeasurementTypeValue: + out.Type = StatsTypeValue + out.N = len(m.Values) + if out.N == 0 { + return out + } + indices, sum := make([]int, len(m.Values)), 0.0 + for idx, v := range m.Values { + indices[idx] = idx + sum += v + } + sort.Slice(indices, func(i, j int) bool { + return m.Values[indices[i]] < m.Values[indices[j]] + }) + out.ValueBundle = map[Stat]float64{ + StatMin: m.Values[indices[0]], + StatMax: m.Values[indices[out.N-1]], + StatMean: sum / float64(out.N), + StatStdDev: 0.0, + } + out.AnnotationBundle = map[Stat]string{ + StatMin: m.Annotations[indices[0]], + StatMax: m.Annotations[indices[out.N-1]], + } + + if out.N%2 == 0 { + out.ValueBundle[StatMedian] = (m.Values[indices[out.N/2]] + m.Values[indices[out.N/2-1]]) / 2.0 + } else { + out.ValueBundle[StatMedian] = m.Values[indices[(out.N-1)/2]] + } + + for _, v := range m.Values { + out.ValueBundle[StatStdDev] += (v - out.ValueBundle[StatMean]) * (v - out.ValueBundle[StatMean]) + } + out.ValueBundle[StatStdDev] = math.Sqrt(out.ValueBundle[StatStdDev] / float64(out.N)) + case MeasurementTypeDuration: + out.Type = StatsTypeDuration + out.N = len(m.Durations) + if out.N == 0 { + return out + } + indices, sum := make([]int, len(m.Durations)), time.Duration(0) + for idx, v := range m.Durations { + indices[idx] = idx + sum += v + } + sort.Slice(indices, func(i, j int) bool { + return m.Durations[indices[i]] < m.Durations[indices[j]] + }) + out.DurationBundle = map[Stat]time.Duration{ + StatMin: m.Durations[indices[0]], + StatMax: m.Durations[indices[out.N-1]], + StatMean: sum / time.Duration(out.N), + } + out.AnnotationBundle = map[Stat]string{ + StatMin: m.Annotations[indices[0]], + StatMax: m.Annotations[indices[out.N-1]], + } + + if out.N%2 == 0 { + out.DurationBundle[StatMedian] = (m.Durations[indices[out.N/2]] + m.Durations[indices[out.N/2-1]]) / 2 + } else { + out.DurationBundle[StatMedian] = m.Durations[indices[(out.N-1)/2]] + } + stdDev := 0.0 + for _, v := range m.Durations { + stdDev += float64(v-out.DurationBundle[StatMean]) * float64(v-out.DurationBundle[StatMean]) + } + out.DurationBundle[StatStdDev] = time.Duration(math.Sqrt(stdDev / float64(out.N))) + } + + return out +} diff --git a/gmeasure/measurement_test.go b/gmeasure/measurement_test.go new file mode 100644 index 000000000..b54addd9f --- /dev/null +++ b/gmeasure/measurement_test.go @@ -0,0 +1,256 @@ +package gmeasure_test + +import ( + "math" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gmeasure" +) + +var _ = Describe("Measurement", func() { + var e *gmeasure.Experiment + var measurement gmeasure.Measurement + + BeforeEach(func() { + e = gmeasure.NewExperiment("Test Experiment") + }) + + Describe("Note Measurement", func() { + BeforeEach(func() { + e.RecordNote("I'm a red note", gmeasure.Style("{{red}}")) + measurement = e.Measurements[0] + }) + + Describe("Generating Stats", func() { + It("returns an empty stats", func() { + Ω(measurement.Stats()).Should(BeZero()) + }) + }) + + Describe("Emitting an unstyled report", func() { + It("does not include styling", func() { + Ω(measurement.String()).Should(Equal("Test Experiment - Note\nI'm a red note\n")) + }) + }) + + Describe("Emitting a styled report", func() { + It("does include styling", func() { + Ω(measurement.ColorableString()).Should(Equal("{{red}}Test Experiment - Note\nI'm a red note\n{{/}}")) + }) + }) + }) + + Describe("Value Measurement", func() { + var min, median, mean, stdDev, max float64 + BeforeEach(func() { + e.RecordValue("flange widths", 7.128, gmeasure.Annotation("A"), gmeasure.Precision(2), gmeasure.Units("inches"), gmeasure.Style("{{blue}}")) + e.RecordValue("flange widths", 3.141, gmeasure.Annotation("B")) + e.RecordValue("flange widths", 9.28223, gmeasure.Annotation("C")) + e.RecordValue("flange widths", 14.249, gmeasure.Annotation("D")) + e.RecordValue("flange widths", 8.975, gmeasure.Annotation("E")) + measurement = e.Measurements[0] + min = 3.141 + max = 14.249 + median = 8.975 + mean = (7.128 + 3.141 + 9.28223 + 14.249 + 8.975) / 5.0 + stdDev = (7.128-mean)*(7.128-mean) + (3.141-mean)*(3.141-mean) + (9.28223-mean)*(9.28223-mean) + (14.249-mean)*(14.249-mean) + (8.975-mean)*(8.975-mean) + stdDev = math.Sqrt(stdDev / 5.0) + }) + + Describe("Generating Stats", func() { + It("generates a correctly configured Stats with correct values", func() { + stats := measurement.Stats() + Ω(stats.ExperimentName).Should(Equal("Test Experiment")) + Ω(stats.MeasurementName).Should(Equal("flange widths")) + Ω(stats.Style).Should(Equal("{{blue}}")) + Ω(stats.Units).Should(Equal("inches")) + Ω(stats.PrecisionBundle.ValueFormat).Should(Equal("%.2f")) + + Ω(stats.ValueBundle[gmeasure.StatMin]).Should(Equal(min)) + Ω(stats.AnnotationBundle[gmeasure.StatMin]).Should(Equal("B")) + Ω(stats.ValueBundle[gmeasure.StatMax]).Should(Equal(max)) + Ω(stats.AnnotationBundle[gmeasure.StatMax]).Should(Equal("D")) + Ω(stats.ValueBundle[gmeasure.StatMedian]).Should(Equal(median)) + Ω(stats.ValueBundle[gmeasure.StatMean]).Should(Equal(mean)) + Ω(stats.ValueBundle[gmeasure.StatStdDev]).Should(Equal(stdDev)) + }) + }) + + Describe("Emitting an unstyled report", func() { + It("does not include styling", func() { + expected := strings.Join([]string{ + "Test Experiment - flange widths [inches]", + "3.14 < [8.97] | <8.56> ±3.59 < 14.25", + "Value | Annotation", + "==================", + " 7.13 | A ", + "------------------", + " 3.14 | B ", + "------------------", + " 9.28 | C ", + "------------------", + "14.25 | D ", + "------------------", + " 8.97 | E ", + "", + }, "\n") + Ω(measurement.String()).Should(Equal(expected)) + }) + }) + + Describe("Emitting a styled report", func() { + It("does include styling", func() { + expected := strings.Join([]string{ + "{{blue}}Test Experiment - flange widths [inches]{{/}}", + "3.14 < [8.97] | <8.56> ±3.59 < 14.25", + "{{blue}}Value{{/}} | {{blue}}Annotation{{/}}", + "==================", + " 7.13 | {{gray}}A {{/}}", + "------------------", + " 3.14 | {{gray}}B {{/}}", + "------------------", + " 9.28 | {{gray}}C {{/}}", + "------------------", + "14.25 | {{gray}}D {{/}}", + "------------------", + " 8.97 | {{gray}}E {{/}}", + "", + }, "\n") + Ω(measurement.ColorableString()).Should(Equal(expected)) + }) + }) + + Describe("Computing medians", func() { + Context("with an odd number of values", func() { + It("returns the middle element", func() { + e.RecordValue("odd", 5) + e.RecordValue("odd", 1) + e.RecordValue("odd", 2) + e.RecordValue("odd", 4) + e.RecordValue("odd", 3) + + Ω(e.GetStats("odd").ValueBundle[gmeasure.StatMedian]).Should(Equal(3.0)) + }) + }) + + Context("when an even number of values", func() { + It("returns the mean of the two middle elements", func() { + e.RecordValue("even", 1) + e.RecordValue("even", 2) + e.RecordValue("even", 4) + e.RecordValue("even", 3) + + Ω(e.GetStats("even").ValueBundle[gmeasure.StatMedian]).Should(Equal(2.5)) + }) + }) + }) + }) + + Describe("Duration Measurement", func() { + var min, median, mean, stdDev, max time.Duration + BeforeEach(func() { + e.RecordDuration("runtime", 7128*time.Millisecond, gmeasure.Annotation("A"), gmeasure.Precision(time.Millisecond*100), gmeasure.Style("{{blue}}")) + e.RecordDuration("runtime", 3141*time.Millisecond, gmeasure.Annotation("B")) + e.RecordDuration("runtime", 9282*time.Millisecond, gmeasure.Annotation("C")) + e.RecordDuration("runtime", 14249*time.Millisecond, gmeasure.Annotation("D")) + e.RecordDuration("runtime", 8975*time.Millisecond, gmeasure.Annotation("E")) + measurement = e.Measurements[0] + min = 3141 * time.Millisecond + max = 14249 * time.Millisecond + median = 8975 * time.Millisecond + mean = ((7128 + 3141 + 9282 + 14249 + 8975) * time.Millisecond) / 5 + stdDev = time.Duration(math.Sqrt((float64(7128*time.Millisecond-mean)*float64(7128*time.Millisecond-mean) + float64(3141*time.Millisecond-mean)*float64(3141*time.Millisecond-mean) + float64(9282*time.Millisecond-mean)*float64(9282*time.Millisecond-mean) + float64(14249*time.Millisecond-mean)*float64(14249*time.Millisecond-mean) + float64(8975*time.Millisecond-mean)*float64(8975*time.Millisecond-mean)) / 5.0)) + }) + + Describe("Generating Stats", func() { + It("generates a correctly configured Stats with correct values", func() { + stats := measurement.Stats() + Ω(stats.ExperimentName).Should(Equal("Test Experiment")) + Ω(stats.MeasurementName).Should(Equal("runtime")) + Ω(stats.Style).Should(Equal("{{blue}}")) + Ω(stats.Units).Should(Equal("duration")) + Ω(stats.PrecisionBundle.Duration).Should(Equal(time.Millisecond * 100)) + + Ω(stats.DurationBundle[gmeasure.StatMin]).Should(Equal(min)) + Ω(stats.AnnotationBundle[gmeasure.StatMin]).Should(Equal("B")) + Ω(stats.DurationBundle[gmeasure.StatMax]).Should(Equal(max)) + Ω(stats.AnnotationBundle[gmeasure.StatMax]).Should(Equal("D")) + Ω(stats.DurationBundle[gmeasure.StatMedian]).Should(Equal(median)) + Ω(stats.DurationBundle[gmeasure.StatMean]).Should(Equal(mean)) + Ω(stats.DurationBundle[gmeasure.StatStdDev]).Should(Equal(stdDev)) + }) + }) + + Describe("Emitting an unstyled report", func() { + It("does not include styling", func() { + expected := strings.Join([]string{ + "Test Experiment - runtime [duration]", + "3.1s < [9s] | <8.6s> ±3.6s < 14.2s", + "Duration | Annotation", + "=====================", + " 7.1s | A ", + "---------------------", + " 3.1s | B ", + "---------------------", + " 9.3s | C ", + "---------------------", + " 14.2s | D ", + "---------------------", + " 9s | E ", + "", + }, "\n") + Ω(measurement.String()).Should(Equal(expected)) + }) + }) + + Describe("Emitting a styled report", func() { + It("does include styling", func() { + expected := strings.Join([]string{ + "{{blue}}Test Experiment - runtime [duration]{{/}}", + "3.1s < [9s] | <8.6s> ±3.6s < 14.2s", + "{{blue}}Duration{{/}} | {{blue}}Annotation{{/}}", + "=====================", + "{{blue}} 7.1s{{/}} | {{gray}}A {{/}}", + "---------------------", + "{{blue}} 3.1s{{/}} | {{gray}}B {{/}}", + "---------------------", + "{{blue}} 9.3s{{/}} | {{gray}}C {{/}}", + "---------------------", + "{{blue}} 14.2s{{/}} | {{gray}}D {{/}}", + "---------------------", + "{{blue}} 9s{{/}} | {{gray}}E {{/}}", + "", + }, "\n") + Ω(measurement.ColorableString()).Should(Equal(expected)) + }) + }) + + Describe("Computing medians", func() { + Context("with an odd number of values", func() { + It("returns the middle element", func() { + e.RecordDuration("odd", 5*time.Second) + e.RecordDuration("odd", 1*time.Second) + e.RecordDuration("odd", 2*time.Second) + e.RecordDuration("odd", 4*time.Second) + e.RecordDuration("odd", 3*time.Second) + + Ω(e.GetStats("odd").DurationBundle[gmeasure.StatMedian]).Should(Equal(3 * time.Second)) + }) + }) + + Context("when an even number of values", func() { + It("returns the mean of the two middle elements", func() { + e.RecordDuration("even", 1*time.Second) + e.RecordDuration("even", 2*time.Second) + e.RecordDuration("even", 4*time.Second) + e.RecordDuration("even", 3*time.Second) + + Ω(e.GetStats("even").DurationBundle[gmeasure.StatMedian]).Should(Equal(2500 * time.Millisecond)) + }) + }) + }) + }) +}) diff --git a/gmeasure/rank.go b/gmeasure/rank.go new file mode 100644 index 000000000..5beee0fcd --- /dev/null +++ b/gmeasure/rank.go @@ -0,0 +1,141 @@ +package gmeasure + +import ( + "fmt" + "sort" + + "github.com/onsi/gomega/gmeasure/table" +) + +/* +RankingCriteria is an enum representing the criteria by which Stats should be ranked. The enum names should be self explanatory. e.g. LowerMeanIsBetter means that Stats with lower mean values are considered more beneficial, with the lowest mean being declared the "winner" . +*/ +type RankingCriteria uint + +const ( + LowerMeanIsBetter RankingCriteria = iota + HigherMeanIsBetter + LowerMedianIsBetter + HigherMedianIsBetter + LowerMinIsBetter + HigherMinIsBetter + LowerMaxIsBetter + HigherMaxIsBetter +) + +var rcEnumSupport = newEnumSupport(map[uint]string{uint(LowerMeanIsBetter): "Lower Mean is Better", uint(HigherMeanIsBetter): "Higher Mean is Better", uint(LowerMedianIsBetter): "Lower Median is Better", uint(HigherMedianIsBetter): "Higher Median is Better", uint(LowerMinIsBetter): "Lower Mins is Better", uint(HigherMinIsBetter): "Higher Min is Better", uint(LowerMaxIsBetter): "Lower Max is Better", uint(HigherMaxIsBetter): "Higher Max is Better"}) + +func (s RankingCriteria) String() string { return rcEnumSupport.String(uint(s)) } +func (s *RankingCriteria) UnmarshalJSON(b []byte) error { + out, err := rcEnumSupport.UnmarshalJSON(b) + *s = RankingCriteria(out) + return err +} +func (s RankingCriteria) MarshalJSON() ([]byte, error) { return rcEnumSupport.MarshalJSON(uint(s)) } + +/* +Ranking ranks a set of Stats by a specified RankingCritera. Use RankStats to create a Ranking. + +When using Ginkgo, you can register Rankings as Report Entries via AddReportEntry. This will emit a formatted table representing the Stats in rank-order when Ginkgo generates the report. +*/ +type Ranking struct { + Criteria RankingCriteria + Stats []Stats +} + +/* +RankStats creates a new ranking of the passed-in stats according to the passed-in criteria. +*/ +func RankStats(criteria RankingCriteria, stats ...Stats) Ranking { + sort.Slice(stats, func(i int, j int) bool { + switch criteria { + case LowerMeanIsBetter: + return stats[i].FloatFor(StatMean) < stats[j].FloatFor(StatMean) + case HigherMeanIsBetter: + return stats[i].FloatFor(StatMean) > stats[j].FloatFor(StatMean) + case LowerMedianIsBetter: + return stats[i].FloatFor(StatMedian) < stats[j].FloatFor(StatMedian) + case HigherMedianIsBetter: + return stats[i].FloatFor(StatMedian) > stats[j].FloatFor(StatMedian) + case LowerMinIsBetter: + return stats[i].FloatFor(StatMin) < stats[j].FloatFor(StatMin) + case HigherMinIsBetter: + return stats[i].FloatFor(StatMin) > stats[j].FloatFor(StatMin) + case LowerMaxIsBetter: + return stats[i].FloatFor(StatMax) < stats[j].FloatFor(StatMax) + case HigherMaxIsBetter: + return stats[i].FloatFor(StatMax) > stats[j].FloatFor(StatMax) + } + return false + }) + + out := Ranking{ + Criteria: criteria, + Stats: stats, + } + + return out +} + +/* +Winner returns the Stats with the most optimal rank based on the specified ranking criteria. For example, if the RankingCriteria is LowerMaxIsBetter then the Stats with the lowest value or duration for StatMax will be returned as the "winner" +*/ +func (c Ranking) Winner() Stats { + if len(c.Stats) == 0 { + return Stats{} + } + return c.Stats[0] +} + +func (c Ranking) report(enableStyling bool) string { + if len(c.Stats) == 0 { + return "Empty Ranking" + } + t := table.NewTable() + t.TableStyle.EnableTextStyling = enableStyling + t.AppendRow(table.R( + table.C("Experiment"), table.C("Name"), table.C("N"), table.C("Min"), table.C("Median"), table.C("Mean"), table.C("StdDev"), table.C("Max"), + table.Divider("="), + "{{bold}}", + )) + + for idx, stats := range c.Stats { + name := stats.MeasurementName + if stats.Units != "" { + name = name + " [" + stats.Units + "]" + } + experimentName := stats.ExperimentName + style := stats.Style + if idx == 0 { + style = "{{bold}}" + style + name += "\n*Winner*" + experimentName += "\n*Winner*" + } + r := table.R(style) + t.AppendRow(r) + r.AppendCell(table.C(experimentName), table.C(name)) + r.AppendCell(stats.cells()...) + + } + out := fmt.Sprintf("Ranking Criteria: %s\n", c.Criteria) + if enableStyling { + out = "{{bold}}" + out + "{{/}}" + } + out += t.Render() + return out +} + +/* +ColorableString generates a styled report that includes a table of the rank-ordered Stats +It is called automatically by Ginkgo's reporting infrastructure when the Ranking is registered as a ReportEntry via AddReportEntry. +*/ +func (c Ranking) ColorableString() string { + return c.report(true) +} + +/* +String generates an unstyled report that includes a table of the rank-ordered Stats +*/ +func (c Ranking) String() string { + return c.report(false) +} diff --git a/gmeasure/rank_test.go b/gmeasure/rank_test.go new file mode 100644 index 000000000..8699fd4ab --- /dev/null +++ b/gmeasure/rank_test.go @@ -0,0 +1,178 @@ +package gmeasure_test + +import ( + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gmeasure" +) + +var _ = Describe("Rank", func() { + var A, B, C, D gmeasure.Stats + + Describe("Ranking Values", func() { + makeStats := func(name string, min float64, max float64, mean float64, median float64) gmeasure.Stats { + return gmeasure.Stats{ + Type: gmeasure.StatsTypeValue, + ExperimentName: "Exp-" + name, + MeasurementName: name, + N: 100, + PrecisionBundle: gmeasure.Precision(2), + ValueBundle: map[gmeasure.Stat]float64{ + gmeasure.StatMin: min, + gmeasure.StatMax: max, + gmeasure.StatMean: mean, + gmeasure.StatMedian: median, + gmeasure.StatStdDev: 2.0, + }, + } + } + + BeforeEach(func() { + A = makeStats("A", 1, 2, 3, 4) + B = makeStats("B", 2, 3, 4, 1) + C = makeStats("C", 3, 4, 1, 2) + D = makeStats("D", 4, 1, 2, 3) + }) + + DescribeTable("ranking by criteria", + func(criteria gmeasure.RankingCriteria, expectedOrder func() []gmeasure.Stats) { + ranking := gmeasure.RankStats(criteria, A, B, C, D) + expected := expectedOrder() + Ω(ranking.Winner()).Should(Equal(expected[0])) + Ω(ranking.Stats).Should(Equal(expected)) + }, + Entry("entry", gmeasure.LowerMeanIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{C, D, A, B} }), + Entry("entry", gmeasure.HigherMeanIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{B, A, D, C} }), + Entry("entry", gmeasure.LowerMedianIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{B, C, D, A} }), + Entry("entry", gmeasure.HigherMedianIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{A, D, C, B} }), + Entry("entry", gmeasure.LowerMinIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{A, B, C, D} }), + Entry("entry", gmeasure.HigherMinIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{D, C, B, A} }), + Entry("entry", gmeasure.LowerMaxIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{D, A, B, C} }), + Entry("entry", gmeasure.HigherMaxIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{C, B, A, D} }), + ) + + Describe("Generating Reports", func() { + It("can generate an unstyled report", func() { + ranking := gmeasure.RankStats(gmeasure.LowerMeanIsBetter, A, B, C, D) + Ω(ranking.String()).Should(Equal(strings.Join([]string{ + "Ranking Criteria: Lower Mean is Better", + "Experiment | Name | N | Min | Median | Mean | StdDev | Max ", + "==================================================================", + "Exp-C | C | 100 | 3.00 | 2.00 | 1.00 | 2.00 | 4.00", + "*Winner* | *Winner* | | | | | | ", + "------------------------------------------------------------------", + "Exp-D | D | 100 | 4.00 | 3.00 | 2.00 | 2.00 | 1.00", + "------------------------------------------------------------------", + "Exp-A | A | 100 | 1.00 | 4.00 | 3.00 | 2.00 | 2.00", + "------------------------------------------------------------------", + "Exp-B | B | 100 | 2.00 | 1.00 | 4.00 | 2.00 | 3.00", + "", + }, "\n"))) + }) + + It("can generate a styled report", func() { + ranking := gmeasure.RankStats(gmeasure.LowerMeanIsBetter, A, B, C, D) + Ω(ranking.ColorableString()).Should(Equal(strings.Join([]string{ + "{{bold}}Ranking Criteria: Lower Mean is Better", + "{{/}}{{bold}}Experiment{{/}} | {{bold}}Name {{/}} | {{bold}}N {{/}} | {{bold}}Min {{/}} | {{bold}}Median{{/}} | {{bold}}Mean{{/}} | {{bold}}StdDev{{/}} | {{bold}}Max {{/}}", + "==================================================================", + "{{bold}}Exp-C {{/}} | {{bold}}C {{/}} | {{bold}}100{{/}} | {{bold}}3.00{{/}} | {{bold}}2.00 {{/}} | {{bold}}1.00{{/}} | {{bold}}2.00 {{/}} | {{bold}}4.00{{/}}", + "{{bold}}*Winner* {{/}} | {{bold}}*Winner*{{/}} | | | | | | ", + "------------------------------------------------------------------", + "Exp-D | D | 100 | 4.00 | 3.00 | 2.00 | 2.00 | 1.00", + "------------------------------------------------------------------", + "Exp-A | A | 100 | 1.00 | 4.00 | 3.00 | 2.00 | 2.00", + "------------------------------------------------------------------", + "Exp-B | B | 100 | 2.00 | 1.00 | 4.00 | 2.00 | 3.00", + "", + }, "\n"))) + }) + }) + }) + + Describe("Ranking Durations", func() { + makeStats := func(name string, min time.Duration, max time.Duration, mean time.Duration, median time.Duration) gmeasure.Stats { + return gmeasure.Stats{ + Type: gmeasure.StatsTypeDuration, + ExperimentName: "Exp-" + name, + MeasurementName: name, + N: 100, + PrecisionBundle: gmeasure.Precision(time.Millisecond * 100), + DurationBundle: map[gmeasure.Stat]time.Duration{ + gmeasure.StatMin: min, + gmeasure.StatMax: max, + gmeasure.StatMean: mean, + gmeasure.StatMedian: median, + gmeasure.StatStdDev: 2.0, + }, + } + } + + BeforeEach(func() { + A = makeStats("A", 1*time.Second, 2*time.Second, 3*time.Second, 4*time.Second) + B = makeStats("B", 2*time.Second, 3*time.Second, 4*time.Second, 1*time.Second) + C = makeStats("C", 3*time.Second, 4*time.Second, 1*time.Second, 2*time.Second) + D = makeStats("D", 4*time.Second, 1*time.Second, 2*time.Second, 3*time.Second) + }) + + DescribeTable("ranking by criteria", + func(criteria gmeasure.RankingCriteria, expectedOrder func() []gmeasure.Stats) { + ranking := gmeasure.RankStats(criteria, A, B, C, D) + expected := expectedOrder() + Ω(ranking.Winner()).Should(Equal(expected[0])) + Ω(ranking.Stats).Should(Equal(expected)) + }, + Entry("entry", gmeasure.LowerMeanIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{C, D, A, B} }), + Entry("entry", gmeasure.HigherMeanIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{B, A, D, C} }), + Entry("entry", gmeasure.LowerMedianIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{B, C, D, A} }), + Entry("entry", gmeasure.HigherMedianIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{A, D, C, B} }), + Entry("entry", gmeasure.LowerMinIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{A, B, C, D} }), + Entry("entry", gmeasure.HigherMinIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{D, C, B, A} }), + Entry("entry", gmeasure.LowerMaxIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{D, A, B, C} }), + Entry("entry", gmeasure.HigherMaxIsBetter, func() []gmeasure.Stats { return []gmeasure.Stats{C, B, A, D} }), + ) + + Describe("Generating Reports", func() { + It("can generate an unstyled report", func() { + ranking := gmeasure.RankStats(gmeasure.LowerMeanIsBetter, A, B, C, D) + Ω(ranking.String()).Should(Equal(strings.Join([]string{ + "Ranking Criteria: Lower Mean is Better", + "Experiment | Name | N | Min | Median | Mean | StdDev | Max", + "================================================================", + "Exp-C | C | 100 | 3s | 2s | 1s | 0s | 4s ", + "*Winner* | *Winner* | | | | | | ", + "----------------------------------------------------------------", + "Exp-D | D | 100 | 4s | 3s | 2s | 0s | 1s ", + "----------------------------------------------------------------", + "Exp-A | A | 100 | 1s | 4s | 3s | 0s | 2s ", + "----------------------------------------------------------------", + "Exp-B | B | 100 | 2s | 1s | 4s | 0s | 3s ", + "", + }, "\n"))) + }) + + It("can generate a styled report", func() { + ranking := gmeasure.RankStats(gmeasure.LowerMeanIsBetter, A, B, C, D) + Ω(ranking.ColorableString()).Should(Equal(strings.Join([]string{ + "{{bold}}Ranking Criteria: Lower Mean is Better", + "{{/}}{{bold}}Experiment{{/}} | {{bold}}Name {{/}} | {{bold}}N {{/}} | {{bold}}Min{{/}} | {{bold}}Median{{/}} | {{bold}}Mean{{/}} | {{bold}}StdDev{{/}} | {{bold}}Max{{/}}", + "================================================================", + "{{bold}}Exp-C {{/}} | {{bold}}C {{/}} | {{bold}}100{{/}} | {{bold}}3s {{/}} | {{bold}}2s {{/}} | {{bold}}1s {{/}} | {{bold}}0s {{/}} | {{bold}}4s {{/}}", + "{{bold}}*Winner* {{/}} | {{bold}}*Winner*{{/}} | | | | | | ", + "----------------------------------------------------------------", + "Exp-D | D | 100 | 4s | 3s | 2s | 0s | 1s ", + "----------------------------------------------------------------", + "Exp-A | A | 100 | 1s | 4s | 3s | 0s | 2s ", + "----------------------------------------------------------------", + "Exp-B | B | 100 | 2s | 1s | 4s | 0s | 3s ", + "", + }, "\n"))) + }) + }) + }) + +}) diff --git a/gmeasure/stats.go b/gmeasure/stats.go new file mode 100644 index 000000000..18d501cd9 --- /dev/null +++ b/gmeasure/stats.go @@ -0,0 +1,153 @@ +package gmeasure + +import ( + "fmt" + "time" + + "github.com/onsi/gomega/gmeasure/table" +) + +/* +Stat is an enum representing the statistics you can request of a Stats struct +*/ +type Stat uint + +const ( + StatInvalid Stat = iota + StatMin + StatMax + StatMean + StatMedian + StatStdDev +) + +var statEnumSupport = newEnumSupport(map[uint]string{uint(StatInvalid): "INVALID STAT", uint(StatMin): "Min", uint(StatMax): "Max", uint(StatMean): "Mean", uint(StatMedian): "Median", uint(StatStdDev): "StdDev"}) + +func (s Stat) String() string { return statEnumSupport.String(uint(s)) } +func (s *Stat) UnmarshalJSON(b []byte) error { + out, err := statEnumSupport.UnmarshalJSON(b) + *s = Stat(out) + return err +} +func (s Stat) MarshalJSON() ([]byte, error) { return statEnumSupport.MarshalJSON(uint(s)) } + +type StatsType uint + +const ( + StatsTypeInvalid StatsType = iota + StatsTypeValue + StatsTypeDuration +) + +var statsTypeEnumSupport = newEnumSupport(map[uint]string{uint(StatsTypeInvalid): "INVALID STATS TYPE", uint(StatsTypeValue): "StatsTypeValue", uint(StatsTypeDuration): "StatsTypeDuration"}) + +func (s StatsType) String() string { return statsTypeEnumSupport.String(uint(s)) } +func (s *StatsType) UnmarshalJSON(b []byte) error { + out, err := statsTypeEnumSupport.UnmarshalJSON(b) + *s = StatsType(out) + return err +} +func (s StatsType) MarshalJSON() ([]byte, error) { return statsTypeEnumSupport.MarshalJSON(uint(s)) } + +/* +Stats records the key statistics for a given measurement. You generally don't make Stats directly - but you can fetch them from Experiments using GetStats() and from Measurements using Stats(). + +When using Ginkgo, you can register Measurements as Report Entries via AddReportEntry. This will emit all the captured data points when Ginkgo generates the report. +*/ +type Stats struct { + // Type is the StatType - one of StatTypeDuration or StatTypeValue + Type StatsType + + // ExperimentName is the name of the Experiment that recorded the Measurement from which this Stat is derived + ExperimentName string + + // MeasurementName is the name of the Measurement from which this Stat is derived + MeasurementName string + + // Units captures the Units of the Measurement from which this Stat is derived + Units string + + // Style captures the Style of the Measurement from which this Stat is derived + Style string + + // PrecisionBundle captures the precision to use when rendering data for this Measurement. + // If Type is StatTypeDuration then PrecisionBundle.Duration is used to round any durations before presentation. + // If Type is StatTypeValue then PrecisionBundle.ValueFormat is used to format any values before presentation + PrecisionBundle PrecisionBundle + + // N represents the total number of data points in the Meassurement from which this Stat is derived + N int + + // If Type is StatTypeValue, ValueBundle will be populated with float64s representing this Stat's statistics + ValueBundle map[Stat]float64 + + // If Type is StatTypeDuration, DurationBundle will be populated with float64s representing this Stat's statistics + DurationBundle map[Stat]time.Duration + + // AnnotationBundle is populated with Annotations corresponding to the data points that can be associated with a Stat. + // For example AnnotationBundle[StatMin] will return the Annotation for the data point that has the minimum value/duration. + AnnotationBundle map[Stat]string +} + +// String returns a minimal summary of the stats of the form "MIN < [MEDIAN] | ±STDDEV < MAX" +func (s Stats) String() string { + return fmt.Sprintf("%s < [%s] | <%s> ±%s < %s", s.StringFor(StatMin), s.StringFor(StatMedian), s.StringFor(StatMean), s.StringFor(StatStdDev), s.StringFor(StatMax)) +} + +// ValueFor returns the float64 value for a particular Stat. You should only use this if the Stats has Type StatsTypeValue +// For example: +// +// median := experiment.GetStats("length").ValueFor(gmeasure.StatMedian) +// +// will return the median data point for the "length" Measurement. +func (s Stats) ValueFor(stat Stat) float64 { + return s.ValueBundle[stat] +} + +// DurationFor returns the time.Duration for a particular Stat. You should only use this if the Stats has Type StatsTypeDuration +// For example: +// +// mean := experiment.GetStats("runtime").ValueFor(gmeasure.StatMean) +// +// will return the mean duration for the "runtime" Measurement. +func (s Stats) DurationFor(stat Stat) time.Duration { + return s.DurationBundle[stat] +} + +// FloatFor returns a float64 representation of the passed-in Stat. +// When Type is StatsTypeValue this is equivalent to s.ValueFor(stat). +// When Type is StatsTypeDuration this is equivalent to float64(s.DurationFor(stat)) +func (s Stats) FloatFor(stat Stat) float64 { + switch s.Type { + case StatsTypeValue: + return s.ValueFor(stat) + case StatsTypeDuration: + return float64(s.DurationFor(stat)) + } + return 0 +} + +// StringFor returns a formatted string representation of the passed-in Stat. +// The formatting honors the precision directives provided in stats.PrecisionBundle +func (s Stats) StringFor(stat Stat) string { + switch s.Type { + case StatsTypeValue: + return fmt.Sprintf(s.PrecisionBundle.ValueFormat, s.ValueFor(stat)) + case StatsTypeDuration: + return s.DurationFor(stat).Round(s.PrecisionBundle.Duration).String() + } + return "" +} + +func (s Stats) cells() []table.Cell { + out := []table.Cell{} + out = append(out, table.C(fmt.Sprintf("%d", s.N))) + for _, stat := range []Stat{StatMin, StatMedian, StatMean, StatStdDev, StatMax} { + content := s.StringFor(stat) + if s.AnnotationBundle[stat] != "" { + content += "\n" + s.AnnotationBundle[stat] + } + out = append(out, table.C(content)) + } + return out +} diff --git a/gmeasure/stats_test.go b/gmeasure/stats_test.go new file mode 100644 index 000000000..6c466b348 --- /dev/null +++ b/gmeasure/stats_test.go @@ -0,0 +1,104 @@ +package gmeasure_test + +import ( + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gmeasure" +) + +var _ = Describe("Stats", func() { + var stats gmeasure.Stats + + Describe("Stats representing values", func() { + BeforeEach(func() { + stats = gmeasure.Stats{ + Type: gmeasure.StatsTypeValue, + ExperimentName: "My Test Experiment", + MeasurementName: "Sprockets", + Units: "widgets", + N: 100, + PrecisionBundle: gmeasure.Precision(2), + ValueBundle: map[gmeasure.Stat]float64{ + gmeasure.StatMin: 17.48992, + gmeasure.StatMax: 293.4820, + gmeasure.StatMean: 187.3023, + gmeasure.StatMedian: 87.2235, + gmeasure.StatStdDev: 73.6394, + }, + } + }) + + Describe("String()", func() { + It("returns a one-line summary", func() { + Ω(stats.String()).Should(Equal("17.49 < [87.22] | <187.30> ±73.64 < 293.48")) + }) + }) + + Describe("ValueFor()", func() { + It("returns the value for the requested stat", func() { + Ω(stats.ValueFor(gmeasure.StatMin)).Should(Equal(17.48992)) + Ω(stats.ValueFor(gmeasure.StatMean)).Should(Equal(187.3023)) + }) + }) + + Describe("FloatFor", func() { + It("returns the requested stat as a float", func() { + Ω(stats.FloatFor(gmeasure.StatMin)).Should(Equal(17.48992)) + Ω(stats.FloatFor(gmeasure.StatMean)).Should(Equal(187.3023)) + }) + }) + + Describe("StringFor", func() { + It("returns the requested stat rendered with the configured precision", func() { + Ω(stats.StringFor(gmeasure.StatMin)).Should(Equal("17.49")) + Ω(stats.StringFor(gmeasure.StatMean)).Should(Equal("187.30")) + }) + }) + }) + + Describe("Stats representing durations", func() { + BeforeEach(func() { + stats = gmeasure.Stats{ + Type: gmeasure.StatsTypeDuration, + ExperimentName: "My Test Experiment", + MeasurementName: "Runtime", + N: 100, + PrecisionBundle: gmeasure.Precision(time.Millisecond * 100), + DurationBundle: map[gmeasure.Stat]time.Duration{ + gmeasure.StatMin: 17375 * time.Millisecond, + gmeasure.StatMax: 890321 * time.Millisecond, + gmeasure.StatMean: 328712 * time.Millisecond, + gmeasure.StatMedian: 552390 * time.Millisecond, + gmeasure.StatStdDev: 186259 * time.Millisecond, + }, + } + }) + Describe("String()", func() { + It("returns a one-line summary", func() { + Ω(stats.String()).Should(Equal("17.4s < [9m12.4s] | <5m28.7s> ±3m6.3s < 14m50.3s")) + }) + }) + Describe("DurationFor()", func() { + It("returns the duration for the requested stat", func() { + Ω(stats.DurationFor(gmeasure.StatMin)).Should(Equal(17375 * time.Millisecond)) + Ω(stats.DurationFor(gmeasure.StatMean)).Should(Equal(328712 * time.Millisecond)) + }) + }) + + Describe("FloatFor", func() { + It("returns the float64 representation for the requested duration stat", func() { + Ω(stats.FloatFor(gmeasure.StatMin)).Should(Equal(float64(17375 * time.Millisecond))) + Ω(stats.FloatFor(gmeasure.StatMean)).Should(Equal(float64(328712 * time.Millisecond))) + }) + }) + + Describe("StringFor", func() { + It("returns the requested stat rendered with the configured precision", func() { + Ω(stats.StringFor(gmeasure.StatMin)).Should(Equal("17.4s")) + Ω(stats.StringFor(gmeasure.StatMean)).Should(Equal("5m28.7s")) + }) + }) + }) +}) diff --git a/gmeasure/stopwatch.go b/gmeasure/stopwatch.go new file mode 100644 index 000000000..634f11f2a --- /dev/null +++ b/gmeasure/stopwatch.go @@ -0,0 +1,117 @@ +package gmeasure + +import "time" + +/* +Stopwatch provides a convenient abstraction for recording durations. There are two ways to make a Stopwatch: + +You can make a Stopwatch from an Experiment via experiment.NewStopwatch(). This is how you first get a hold of a Stopwatch. + +You can subsequently call stopwatch.NewStopwatch() to get a fresh Stopwatch. +This is only necessary if you need to record durations on a different goroutine as a single Stopwatch is not considered thread-safe. + +The Stopwatch starts as soon as it is created. You can Pause() the stopwatch and Reset() it as needed. + +Stopwatches refer back to their parent Experiment. They use this reference to record any measured durations back with the Experiment. +*/ +type Stopwatch struct { + Experiment *Experiment + t time.Time + pauseT time.Time + pauseDuration time.Duration + running bool +} + +func newStopwatch(experiment *Experiment) *Stopwatch { + return &Stopwatch{ + Experiment: experiment, + t: time.Now(), + running: true, + } +} + +/* +NewStopwatch returns a new Stopwatch pointing to the same Experiment as this Stopwatch +*/ +func (s *Stopwatch) NewStopwatch() *Stopwatch { + return newStopwatch(s.Experiment) +} + +/* +Record captures the amount of time that has passed since the Stopwatch was created or most recently Reset(). It records the duration on it's associated Experiment in a Measurement with the passed-in name. + +Record takes all the decorators that experiment.RecordDuration takes (e.g. Annotation("...") can be used to annotate this duration) + +Note that Record does not Reset the Stopwatch. It does, however, return the Stopwatch so the following pattern is common: + + stopwatch := experiment.NewStopwatch() + // first expensive operation + stopwatch.Record("first operation").Reset() //records the duration of the first operation and resets the stopwatch. + // second expensive operation + stopwatch.Record("second operation").Reset() //records the duration of the second operation and resets the stopwatch. + +omitting the Reset() after the first operation would cause the duration recorded for the second operation to include the time elapsed by both the first _and_ second operations. + +The Stopwatch must be running (i.e. not paused) when Record is called. +*/ +func (s *Stopwatch) Record(name string, args ...interface{}) *Stopwatch { + if !s.running { + panic("stopwatch is not running - call Resume or Reset before calling Record") + } + duration := time.Since(s.t) - s.pauseDuration + s.Experiment.RecordDuration(name, duration, args...) + return s +} + +/* +Reset resets the Stopwatch. Subsequent recorded durations will measure the time elapsed from the moment Reset was called. +If the Stopwatch was Paused it is unpaused after calling Reset. +*/ +func (s *Stopwatch) Reset() *Stopwatch { + s.running = true + s.t = time.Now() + s.pauseDuration = 0 + return s +} + +/* +Pause pauses the Stopwatch. While pasued the Stopwatch does not accumulate elapsed time. This is useful for ignoring expensive operations that are incidental to the behavior you are attempting to characterize. +Note: You must call Resume() before you can Record() subsequent measurements. + +For example: + + stopwatch := experiment.NewStopwatch() + // first expensive operation + stopwatch.Record("first operation").Reset() + // second expensive operation - part 1 + stopwatch.Pause() + // something expensive that we don't care about + stopwatch.Resume() + // second expensive operation - part 2 + stopwatch.Record("second operation").Reset() // the recorded duration captures the time elapsed during parts 1 and 2 of the second expensive operation, but not the bit in between + + +The Stopwatch must be running when Pause is called. +*/ +func (s *Stopwatch) Pause() *Stopwatch { + if !s.running { + panic("stopwatch is not running - call Resume or Reset before calling Pause") + } + s.running = false + s.pauseT = time.Now() + return s +} + +/* +Resume resumes a paused Stopwatch. Any time that elapses after Resume is called will be accumulated as elapsed time when a subsequent duration is Recorded. + +The Stopwatch must be Paused when Resume is called +*/ +func (s *Stopwatch) Resume() *Stopwatch { + if s.running { + panic("stopwatch is running - call Pause before calling Resume") + } + s.running = true + s.pauseDuration = s.pauseDuration + time.Since(s.pauseT) + return s +} diff --git a/gmeasure/stopwatch_test.go b/gmeasure/stopwatch_test.go new file mode 100644 index 000000000..5dc44ba5e --- /dev/null +++ b/gmeasure/stopwatch_test.go @@ -0,0 +1,66 @@ +package gmeasure_test + +import ( + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gmeasure" +) + +var _ = Describe("Stopwatch", func() { + var e *gmeasure.Experiment + var stopwatch *gmeasure.Stopwatch + + BeforeEach(func() { + e = gmeasure.NewExperiment("My Test Experiment") + stopwatch = e.NewStopwatch() + }) + + It("records durations", func() { + time.Sleep(100 * time.Millisecond) + stopwatch.Record("recordings", gmeasure.Annotation("A")) + time.Sleep(100 * time.Millisecond) + stopwatch.Record("recordings", gmeasure.Annotation("B")).Reset() + time.Sleep(100 * time.Millisecond) + stopwatch.Record("recordings", gmeasure.Annotation("C")).Reset() + time.Sleep(100 * time.Millisecond) + stopwatch.Pause() + time.Sleep(100 * time.Millisecond) + stopwatch.Resume() + time.Sleep(100 * time.Millisecond) + stopwatch.Pause() + time.Sleep(100 * time.Millisecond) + stopwatch.Resume() + time.Sleep(100 * time.Millisecond) + stopwatch.Record("recordings", gmeasure.Annotation("D")) + durations := e.Get("recordings").Durations + annotations := e.Get("recordings").Annotations + Ω(annotations).Should(Equal([]string{"A", "B", "C", "D"})) + Ω(durations[0]).Should(BeNumerically("~", 100*time.Millisecond, 50*time.Millisecond)) + Ω(durations[1]).Should(BeNumerically("~", 200*time.Millisecond, 50*time.Millisecond)) + Ω(durations[2]).Should(BeNumerically("~", 100*time.Millisecond, 50*time.Millisecond)) + Ω(durations[3]).Should(BeNumerically("~", 300*time.Millisecond, 50*time.Millisecond)) + + }) + + It("panics when asked to record but not running", func() { + stopwatch.Pause() + Ω(func() { + stopwatch.Record("A") + }).Should(PanicWith("stopwatch is not running - call Resume or Reset before calling Record")) + }) + + It("panics when paused but not running", func() { + stopwatch.Pause() + Ω(func() { + stopwatch.Pause() + }).Should(PanicWith("stopwatch is not running - call Resume or Reset before calling Pause")) + }) + + It("panics when asked to resume but not paused", func() { + Ω(func() { + stopwatch.Resume() + }).Should(PanicWith("stopwatch is running - call Pause before calling Resume")) + }) +}) diff --git a/gmeasure/table/table.go b/gmeasure/table/table.go new file mode 100644 index 000000000..f980b9c7a --- /dev/null +++ b/gmeasure/table/table.go @@ -0,0 +1,370 @@ +package table + +// This is a temporary package - Table will move to github.com/onsi/consolable once some more dust settles + +import ( + "reflect" + "strings" + "unicode/utf8" +) + +type AlignType uint + +const ( + AlignTypeLeft AlignType = iota + AlignTypeCenter + AlignTypeRight +) + +type Divider string + +type Row struct { + Cells []Cell + Divider string + Style string +} + +func R(args ...interface{}) *Row { + r := &Row{ + Divider: "-", + } + for _, arg := range args { + switch reflect.TypeOf(arg) { + case reflect.TypeOf(Divider("")): + r.Divider = string(arg.(Divider)) + case reflect.TypeOf(r.Style): + r.Style = arg.(string) + case reflect.TypeOf(Cell{}): + r.Cells = append(r.Cells, arg.(Cell)) + } + } + return r +} + +func (r *Row) AppendCell(cells ...Cell) *Row { + r.Cells = append(r.Cells, cells...) + return r +} + +func (r *Row) Render(widths []int, totalWidth int, tableStyle TableStyle, isLastRow bool) string { + out := "" + if len(r.Cells) == 1 { + out += strings.Join(r.Cells[0].render(totalWidth, r.Style, tableStyle), "\n") + "\n" + } else { + if len(r.Cells) != len(widths) { + panic("row vs width mismatch") + } + renderedCells := make([][]string, len(r.Cells)) + maxHeight := 0 + for colIdx, cell := range r.Cells { + renderedCells[colIdx] = cell.render(widths[colIdx], r.Style, tableStyle) + if len(renderedCells[colIdx]) > maxHeight { + maxHeight = len(renderedCells[colIdx]) + } + } + for colIdx := range r.Cells { + for len(renderedCells[colIdx]) < maxHeight { + renderedCells[colIdx] = append(renderedCells[colIdx], strings.Repeat(" ", widths[colIdx])) + } + } + border := strings.Repeat(" ", tableStyle.Padding) + if tableStyle.VerticalBorders { + border += "|" + border + } + for lineIdx := 0; lineIdx < maxHeight; lineIdx++ { + for colIdx := range r.Cells { + out += renderedCells[colIdx][lineIdx] + if colIdx < len(r.Cells)-1 { + out += border + } + } + out += "\n" + } + } + if tableStyle.HorizontalBorders && !isLastRow && r.Divider != "" { + out += strings.Repeat(string(r.Divider), totalWidth) + "\n" + } + + return out +} + +type Cell struct { + Contents []string + Style string + Align AlignType +} + +func C(contents string, args ...interface{}) Cell { + c := Cell{ + Contents: strings.Split(contents, "\n"), + } + for _, arg := range args { + switch reflect.TypeOf(arg) { + case reflect.TypeOf(c.Style): + c.Style = arg.(string) + case reflect.TypeOf(c.Align): + c.Align = arg.(AlignType) + } + } + return c +} + +func (c Cell) Width() (int, int) { + w, minW := 0, 0 + for _, line := range c.Contents { + lineWidth := utf8.RuneCountInString(line) + if lineWidth > w { + w = lineWidth + } + for _, word := range strings.Split(line, " ") { + wordWidth := utf8.RuneCountInString(word) + if wordWidth > minW { + minW = wordWidth + } + } + } + return w, minW +} + +func (c Cell) alignLine(line string, width int) string { + lineWidth := utf8.RuneCountInString(line) + if lineWidth == width { + return line + } + if lineWidth < width { + gap := width - lineWidth + switch c.Align { + case AlignTypeLeft: + return line + strings.Repeat(" ", gap) + case AlignTypeRight: + return strings.Repeat(" ", gap) + line + case AlignTypeCenter: + leftGap := gap / 2 + rightGap := gap - leftGap + return strings.Repeat(" ", leftGap) + line + strings.Repeat(" ", rightGap) + } + } + return line +} + +func (c Cell) splitWordToWidth(word string, width int) []string { + out := []string{} + n, subWord := 0, "" + for _, c := range word { + subWord += string(c) + n += 1 + if n == width-1 { + out = append(out, subWord+"-") + n, subWord = 0, "" + } + } + return out +} + +func (c Cell) splitToWidth(line string, width int) []string { + lineWidth := utf8.RuneCountInString(line) + if lineWidth <= width { + return []string{line} + } + + outLines := []string{} + words := strings.Split(line, " ") + outWords := []string{words[0]} + length := utf8.RuneCountInString(words[0]) + if length > width { + splitWord := c.splitWordToWidth(words[0], width) + lastIdx := len(splitWord) - 1 + outLines = append(outLines, splitWord[:lastIdx]...) + outWords = []string{splitWord[lastIdx]} + length = utf8.RuneCountInString(splitWord[lastIdx]) + } + + for _, word := range words[1:] { + wordLength := utf8.RuneCountInString(word) + if length+wordLength+1 <= width { + length += wordLength + 1 + outWords = append(outWords, word) + continue + } + outLines = append(outLines, strings.Join(outWords, " ")) + + outWords = []string{word} + length = wordLength + if length > width { + splitWord := c.splitWordToWidth(word, width) + lastIdx := len(splitWord) - 1 + outLines = append(outLines, splitWord[:lastIdx]...) + outWords = []string{splitWord[lastIdx]} + length = utf8.RuneCountInString(splitWord[lastIdx]) + } + } + if len(outWords) > 0 { + outLines = append(outLines, strings.Join(outWords, " ")) + } + + return outLines +} + +func (c Cell) render(width int, style string, tableStyle TableStyle) []string { + out := []string{} + for _, line := range c.Contents { + out = append(out, c.splitToWidth(line, width)...) + } + for idx := range out { + out[idx] = c.alignLine(out[idx], width) + } + + if tableStyle.EnableTextStyling { + style = style + c.Style + if style != "" { + for idx := range out { + out[idx] = style + out[idx] + "{{/}}" + } + } + } + + return out +} + +type TableStyle struct { + Padding int + VerticalBorders bool + HorizontalBorders bool + MaxTableWidth int + MaxColWidth int + EnableTextStyling bool +} + +var DefaultTableStyle = TableStyle{ + Padding: 1, + VerticalBorders: true, + HorizontalBorders: true, + MaxTableWidth: 120, + MaxColWidth: 40, + EnableTextStyling: true, +} + +type Table struct { + Rows []*Row + + TableStyle TableStyle +} + +func NewTable() *Table { + return &Table{ + TableStyle: DefaultTableStyle, + } +} + +func (t *Table) AppendRow(row *Row) *Table { + t.Rows = append(t.Rows, row) + return t +} + +func (t *Table) Render() string { + out := "" + totalWidth, widths := t.computeWidths() + for rowIdx, row := range t.Rows { + out += row.Render(widths, totalWidth, t.TableStyle, rowIdx == len(t.Rows)-1) + } + return out +} + +func (t *Table) computeWidths() (int, []int) { + nCol := 0 + for _, row := range t.Rows { + if len(row.Cells) > nCol { + nCol = len(row.Cells) + } + } + + // lets compute the contribution to width from the borders + padding + borderWidth := t.TableStyle.Padding + if t.TableStyle.VerticalBorders { + borderWidth += 1 + t.TableStyle.Padding + } + totalBorderWidth := borderWidth * (nCol - 1) + + // lets compute the width of each column + widths := make([]int, nCol) + minWidths := make([]int, nCol) + for colIdx := range widths { + for _, row := range t.Rows { + if colIdx >= len(row.Cells) { + // ignore rows with fewer columns + continue + } + w, minWid := row.Cells[colIdx].Width() + if w > widths[colIdx] { + widths[colIdx] = w + } + if minWid > minWidths[colIdx] { + minWidths[colIdx] = minWid + } + } + } + + // do we already fit? + if sum(widths)+totalBorderWidth <= t.TableStyle.MaxTableWidth { + // yes! we're done + return sum(widths) + totalBorderWidth, widths + } + + // clamp the widths and minWidths to MaxColWidth + for colIdx := range widths { + widths[colIdx] = min(widths[colIdx], t.TableStyle.MaxColWidth) + minWidths[colIdx] = min(minWidths[colIdx], t.TableStyle.MaxColWidth) + } + + // do we fit now? + if sum(widths)+totalBorderWidth <= t.TableStyle.MaxTableWidth { + // yes! we're done + return sum(widths) + totalBorderWidth, widths + } + + // hmm... still no... can we possibly squeeze the table in without violating minWidths? + if sum(minWidths)+totalBorderWidth >= t.TableStyle.MaxTableWidth { + // nope - we're just going to have to exceed MaxTableWidth + return sum(minWidths) + totalBorderWidth, minWidths + } + + // looks like we don't fit yet, but we should be able to fit without violating minWidths + // lets start scaling down + n := 0 + for sum(widths)+totalBorderWidth > t.TableStyle.MaxTableWidth { + budget := t.TableStyle.MaxTableWidth - totalBorderWidth + baseline := sum(widths) + + for colIdx := range widths { + widths[colIdx] = max((widths[colIdx]*budget)/baseline, minWidths[colIdx]) + } + n += 1 + if n > 100 { + break // in case we somehow fail to converge + } + } + + return sum(widths) + totalBorderWidth, widths +} + +func sum(s []int) int { + out := 0 + for _, v := range s { + out += v + } + return out +} + +func min(a int, b int) int { + if a < b { + return a + } + return b +} + +func max(a int, b int) int { + if a > b { + return a + } + return b +}