Merging custom-emitter with main

.github/workflows/dart.yml

@@ -7,7 +7,7 @@ name: Dart
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, custom-emitter ]
     paths:
     - '**.dart'
   pull_request:
@@ -34,8 +34,8 @@ jobs:
         run: dart pub get
 
       # Uncomment this step to verify the use of 'dart format' on each commit.
-      # - name: Verify formatting
-      #   run: dart format --output=none --set-exit-if-changed .
+      - name: Verify formatting
+        run: dart format --output=none --set-exit-if-changed .
 
       # Consider passing '--fatal-infos' for slightly stricter analysis.
       - name: Analyze project source

.gitignore

@@ -26,3 +26,6 @@ doc/api/
 
 # Compiled executables
 bin/*.exe
+
+# Exported benchmark scores
+benchmark/*.txt

CHANGELOG.md

@@ -1,7 +1,27 @@
+## 2.0.0
+Breaking changes:
+- The classes `Benchmark` and `AsyncBenchmark` are now solely responsible for
+generating benchmark scores. The constructor parameters `description` and
+`emitter` have been removed. Generating reports is delegated to `ScoreEmitter`.
+- The functions `benchmark` and `asyncBenchmark` are not longer generic and
+the only optional parameter is: `scoreEmitter`. A custom `ScoreEmitter` can be
+used to generate a custom benchmark report.
+
+
+## 1.0.0
+Breaking changes:
+- The command `benchmark_runner` now has subcommands `report` and `export`.
+- The functions `benchmark` and `asyncBenchmark` are now generic and
+  accept the optional parameters:
+    * `E extents ColorPrintEmitter emitter`,
+    * `report(instance, emitter){}`, where `instance` is an instance of
+       `Benchmark` or `AsyncBenchmark`, respectively.
+       The callback can be can be used to pass benchmark scores to the emitter.
+
 ## 0.1.9
 - Changed default encoding of standard output of benchmark processes
 to `Utf8Codec()`. This enables the correct output of histograms in windows
-terminals with utf8 support. 
+terminals with utf8 support.
 
 ## 0.1.8
 - Updated docs.
@@ -19,8 +39,9 @@ inter-quartile-range `iqr` of the score sample is zero.
 
 ## 0.1.5
 - Made [BenchmarkHelper.sampleSize][sampleSize] a variable assignable with
-defined function. This allows changing the benchmark runtime by customizing
-the relation between score estimate and score sample size.
+a user defined function. This allows changing the score sample generation
+customizing the relation between the single score estimate and the
+score sample size.
 
 ## 0.1.4
 - Fixed bugs in runner (results were listed twice, exit code was always 0).

README.md

@@ -6,8 +6,7 @@
 
 Benchmarking is used to estimate and compare the execution speed of
 numerical algorithms and programs.
-The package [`benchmark_runner`][benchmark_runner] is based on
-[`benchmark_harness`][benchmark_harness] and includes helper
+The package [`benchmark_runner`][benchmark_runner] includes helper
 functions for writing *inline* micro-benchmarks with the option of
 printing a score **histogram** and reporting the score **mean** ±
 **standard deviation**, and score **median** ± **inter quartile range**.
@@ -36,8 +35,6 @@ Write inline benchmarks using the functions:
  asynchronous benchmarks.
 
   ```Dart
-  // ignore_for_file: unused_local_variable
-
   import 'package:benchmark_runner/benchmark_runner.dart';
 
   /// Returns the value [t] after waiting for [duration].
@@ -53,7 +50,7 @@ Write inline benchmarks using the functions:
 
       await asyncBenchmark('5ms', () async {
         await later<int>(27, Duration(milliseconds: 5));
-      }, emitStats: false);
+      }, scoreEmitter: MeanEmitter());
     });
 
     group('Set', () async {
@@ -72,13 +69,12 @@ Write inline benchmarks using the functions:
   ```
 ### 1. Running a Single Benchmark File
 A *single* benchmark file may be run as a Dart executable:
-```Console
-$ dart benchmark/example_async_benchmark.dart
-```
-![Console Output Single](https://raw.githubusercontent.com/simphotonics/benchmark_runner/main/images/console_output_single.png)
+![Console Output Single](https://raw.githubusercontent.com/simphotonics/benchmark_runner/custom-emitter/images/single_report.png)
 
-The console output is shown above. The following colours and coding
-are used:
+The console output is shown above. By default,
+the functions [`benchmark`][benchmark] and
+[`asyncBenchmark`][asyncBenchmark]
+emit benchmark score statistics.
 * The first column shows the micro-benchmark runtime, followed by the group
   name and the benchmark name.
 * The labels of asynchronous groups and benchmarks are marked with an hour-glass
@@ -91,30 +87,67 @@ using <span style="color:#2370C4">*blue*</span> foreground.
 using <span style="color:#28B5D7">*cyan*</span> foreground.
 * Errors are printed using <span style="color:#CB605E"> *red* </span> foreground.
 
-### 2. Running Several Benchmark Files
+### 2. Using the Benchmark Runner
 To run *several* benchmark files (with the format`*_benchmark.dart`)
-invoke the benchmark_runner and specify a directory.
+and print a report, invoke the sub-command `report` and specify a directory.
 If no directory is specified, it defaults to `benchmark`:
 
-```Console
-$ dart run benchmark_runner
-```
-
-![Console Output](https://raw.githubusercontent.com/simphotonics/benchmark_runner/main/images/console_output.png)
+![Runner Report](https://raw.githubusercontent.com/simphotonics/benchmark_runner/custom-emitter/images/runner_report.png)
 
 A typical console output is shown above. In this example, the benchmark_runner
 detected two benchmark files, ran the micro-benchmarks and produced a report.
 
-
 * The summary shows the total number of completed benchmarks, the number of
 benchmarks with errors and the number of groups with errors (that do not
 occur within the scope of a benchmark function).
-* To show a stack trace for each error, run the benchmark_runner using
-the option ``-v`` or `--verbose`.
+* To show a stack trace for each error, use the option ``-v`` or `--verbose`.
 * The total benchmark run time may be shorter than the sum of the
 micro-benchmark run times since each executable benchmark file is run in
 a separate process.
 
+### 3. Exporting Benchmark Scores
+
+To export benchmark scores use the sub-command `export`:
+```
+$ dart run benchmark_runner export --outputDir=scores --extension=csv searchDirectory
+```
+In the example above, `searchDirectory` is scanned for `*_benchmark.dart`
+files. For each benchmark file, a corresponding file `*_benchmark.csv` is
+written to the directory `scores`.
+
+Note: The directory must exist and the user
+must have write access. When exporting benchmark scores to a file
+and the emitter output is colorized,
+it is recommended to use the option `--isMonochrome`, to
+avoid spurious characters due to the use of Ansi modifiers.
+
+The functions [`benchmark`][benchmark] and
+[`asyncBenchmark`][asyncBenchmark] accept the optional parameters `scoreEmitter`.
+The parameter expects an object of type `ScoreEmitter` and
+can be used to customize the score reports e.g.
+to make the score format more suitable for writing to a file:
+
+```Dart
+import 'package:benchmark_runner/benchmark_runner.dart';
+
+class CustomEmitter implements ScoreEmitter {
+  @override
+  void emit({required description, required Score score}) {
+    print('# Mean               Standard Deviation');
+    print('${score.stats.mean}  ${score.stats.stdDev}');
+  }
+}
+
+void main(){
+  benchmark(
+      'construct list | use custom emitter',
+      () {
+        var list = <int>[for (var i = 0; i < 1000; ++i) i];
+      },
+      scoreEmitter: CustomEmitter(),
+  );
+}
+```
 
 ## Tips and Tricks
 
@@ -133,57 +166,63 @@ as reported by [`benchmark_harness`][benchmark_harness] and the
 score statistics.
 
 - By default, [`benchmark`][benchmark] and
-[`asyncBenchmark`][asyncBenchmark] report score statistics. In order to generate
-the report provided by [`benchmark_harness`][benchmark_harness] use the
-optional argument `emitStats: false`.
+[`asyncBenchmark`][asyncBenchmark] report score statistics. In order to print
+the report similar to that produced by
+[`benchmark_harness`][benchmark_harness], use the
+optional argument `emitter: MeanEmitter()`.
 
-- Color output can be switched off by using the option: `--isMonochrome` when
-calling the benchmark runner. When executing a single benchmark file the
+- Color output can be switched off by using the option: `--isMonochrome` or `-m`
+when calling the benchmark runner. When executing a single benchmark file the
 corresponding option is `--define=isMonochrome=true`.
 
 - The default colors used to style benchmark reports are best suited
 for a dark terminal background.
-They can, however, be altered by setting the static variables defined by
+They can, however, be altered by setting the *static* variables defined by
 the class [`ColorProfile`][ColorProfile]. In the example below, the styling of
 error messages and the mean value is altered.
   ```Dart
   import 'package:ansi_modifier/ansi_modifier.dart';
   import 'package:benchmark_runner/benchmark_runner.dart';
 
-  void customColorProfile() {
+  void adjustColorProfile() {
     ColorProfile.error = Ansi.red + Ansi.bold;
     ColorProfile.mean = Ansi.green + Ansi.italic;
   }
 
   void main(List<String> args) {
     // Call function to apply the new custom color profile.
-    customProfile();
+    adjustColorProfile();
   }
   ```
 
 - When running **asynchronous** benchmarks, the scores are printed in order of
-completion. The print the scores in sequential order (as they are listed in the
+completion. To print the scores in sequential order (as they are listed in the
 benchmark executable) it is required to *await* the completion
 of the async benchmark functions and
 the enclosing group.
 
 ## Score Sampling
 
-In order to calculate benchmark score statistics a sample of scores is
+In order to calculate benchmark score *statistics* a sample of scores is
 required. The question is how to generate the score sample while minimizing
 systematic errors (like overheads) and keeping the
-benchmark run times within acceptable limits.
+total benchmark run times within acceptable limits.
+
+<details> <summary> Click to show details. </summary>
 
-To estimate the benchmark score the functions [`warmup`][warmup]
-or [`warmupAsync`][warmupAsync] are run for 200 milliseconds.
+In a first step, benchmark scores are estimated using the
+functions [`warmUp`][warmUp]
+or [`warmUpAsync`][warmUpAsync]. The function [`BenchmarkHelper.sampleSize`][sampleSize]
+uses the score estimate to determine the sample size and the number of inner
+iterations (for short run times each sample entry is averaged).
 
 ### 1. Default Sampling Method
 The graph below shows the sample size (orange curve) as calculated by the function
 [`BenchmarkHelper.sampleSize`][sampleSize].
 The green curve shows the lower limit of the total microbenchmark duration and
 represents the value: `clockTicks * sampleSize * innerIterations`.
 
-![Sample Size](https://raw.githubusercontent.com/simphotonics/benchmark_runner/main/images/sample_size.png)
+![Sample Size](https://raw.githubusercontent.com/simphotonics/benchmark_runner/custom-emitter/images/sample_size.png)
 
 For short run times below 100000 clock ticks each sample score is generated
 using the functions [`measure`][measure] or the equivalent asynchronous method [`measureAsync`][measureAsync].
@@ -197,9 +236,10 @@ averaged over (see the cyan curve in the graph above):
 * ticks > 1e5 => No preliminary averaging of sample scores.
 
 ### 2. Custom Sampling Method
-To amend the score sampling process the static function
+To custominze the score sampling process, the static function
 [`BenchmarkHelper.sampleSize`][sampleSize] can be replaced with a custom function:
 ```Dart
+/// Generates a sample containing 100 benchmark scores.
 BenchmarkHelper.sampleSize = (int clockTicks) {
   return (outer: 100, inner: 1)
 }
@@ -219,6 +259,7 @@ The command above lauches a process and runs a [`gnuplot`][gnuplot] script.
 For this reason, the program [`gnuplot`][gnuplot] must be installed (with
 the `qt` terminal enabled).
 
+</details>
 
 ## Contributions
 
@@ -260,6 +301,6 @@ Please file feature requests and bugs at the [issue tracker][tracker].
 
 [sampleSize]: https://pub.dev/documentation/benchmark_runner/latest/benchmark_runner/BenchmarkHelper/sampleSize.html
 
-[warmup]: https://pub.dev/documentation/benchmark_runner/latest/benchmark_runner/BenchmarkHelper/warmup.html
+[warmUp]: https://pub.dev/documentation/benchmark_runner/latest/benchmark_runner/BenchmarkHelper/warmUp.html
 
-[warmupAsync]: https://pub.dev/documentation/benchmark_runner/latest/benchmark_runner/BenchmarkHelper/warmupAsync.html
+[warmUpAsync]: https://pub.dev/documentation/benchmark_runner/latest/benchmark_runner/BenchmarkHelper/warmUpAsync.html

analysis_options.yaml

@@ -1 +1,5 @@
-include: package:lints/recommended.yaml
+include: package:lints/recommended.yaml
+
+linter:
+  rules:
+    - prefer_relative_imports

benchmark/custom_emitter_benchmark.dart

@@ -0,0 +1,16 @@
+// ignore_for_file: unused_local_variable
+import 'package:benchmark_runner/benchmark_runner.dart';
+
+class CustomEmitter implements ScoreEmitter {
+  @override
+  void emit({required description, required Score score}) {
+    print('Mean               Standard Deviation');
+    print('${score.stats.mean}  ${score.stats.stdDev}');
+  }
+}
+
+void main(List<String> args) {
+  benchmark('construct | Custom emitter', () {
+    var list = <int>[for (var i = 0; i < 1000; ++i) i];
+  }, scoreEmitter: CustomEmitter());
+}

benchmark/example_async_benchmark.dart

@@ -15,7 +15,7 @@ void main(List<String> args) async {
 
     await asyncBenchmark('5ms', () async {
       await later<int>(27, Duration(milliseconds: 5));
-    }, emitStats: false);
+    }, scoreEmitter: MeanEmitter());
   });
 
   group('Set', () async {

benchmark/example_benchmark.dart

@@ -1,6 +1,4 @@
 // ignore_for_file: unused_local_variable
-import 'dart:collection';
-
 import 'package:benchmark_runner/benchmark_runner.dart';
 
 void main(List<String> args) {
@@ -23,10 +21,6 @@ void main(List<String> args) {
 
     benchmark('construct', () {
       var list = <int>[for (var i = 0; i < 1000; ++i) i];
-    }, emitStats: false);
-
-    benchmark('construct list view', () {
-      final listView = UnmodifiableListView(originalList);
-    });
+    }, scoreEmitter: MeanEmitter());
   });
 }