changed
.formatter.exs
|
|
@@ -3,6 +3,7 @@
|
|
3
3
|
"lib/**/*.{ex,exs}",
|
|
4
4
|
"test/**/*.{ex,exs}",
|
|
5
5
|
"mix/**/*.{ex,exs}",
|
|
6
|
- "./mix.exs"
|
|
6
|
+ "./mix.exs",
|
|
7
|
+ "samples/**/*.{ex,exs}"
|
|
7
8
|
]
|
|
8
9
|
]
|
changed
CHANGELOG.md
|
|
@@ -1,5 +1,41 @@
|
|
1
|
+ ## 1.0.0
|
|
2
|
+
|
|
3
|
+ It's 0.99.0 without the deprecation warnings. Specifically:
|
|
4
|
+
|
|
5
|
+ * Old way of passing formatters (`:formatter_options`) vs. new `:formatters` with modules, tuples or functions with one arg
|
|
6
|
+ * The configuration needs to be passed as the second argument to `Benchee.run/2`
|
|
7
|
+ * `Benchee.collect/1` replaces `Benchee.measure/1`
|
|
8
|
+ * `unit_scaling` is a top level configuration option, not for the console formatter
|
|
9
|
+ * the warning for memory measurements not working on OTP <= 18 will also be dropped (we already officially dropped OTP 18 support in 0.14.0)
|
|
10
|
+
|
|
11
|
+ We're aiming to follow Semantic Versioning as we go forward. That means formatters should be safe to use `~> 1.0` (or even `>= 0.99.0 and < 2.0.0`).
|
|
12
|
+
|
|
13
|
+ ## 0.99.0 (2019-03-28)
|
|
14
|
+
|
|
15
|
+ The "we're almost 1.0!" release - all the last small features, a bag of polish and deprecation warnings. If you run this release succesfully without deprecation warnings you should be safe to upgrade to 1.0.0, if not - it's a bug :)
|
|
16
|
+
|
|
17
|
+ ### Breaking Changes (User Facing)
|
|
18
|
+ * changed official Elixir compatibility to `~> 1.6`, 1.4+ should still work but aren't guaranteed or tested against.
|
|
19
|
+
|
|
20
|
+ ### Features (User Facing)
|
|
21
|
+ * the console comparison now also displays the absolute difference in the average (like +12 ms) so that you have an idea to how much time that translates to in your applications not just that it's 100x faster
|
|
22
|
+ * Overhaul of README, documentation, update samples etc. - a whole lot of things have also been marked `@doc false` as they're considered internal
|
|
23
|
+
|
|
24
|
+ ### Bugfixes (User Facing)
|
|
25
|
+ * Remove double empty line after configuration display
|
|
26
|
+ * Fix some wrong type specs
|
|
27
|
+
|
|
28
|
+ ### Breaking Changes (Plugins)
|
|
29
|
+ * `Scenario` made it to the big leagues, it's no longer `Benchee.Benchmark.Scenario` but `Benchee.Scenario` - as it is arguably one of our most important data structures.
|
|
30
|
+ * The `Scenario` struct had some keys changed (last time before 2.0 I promise!) - instead of `:run_times`/`:run_time_statistics` you now have one `run_time_data` key that contains `Benchee.CollectionData` which has the keys `:samples` and `:statistics`. Same for `memory_usage`. This was done to be able to handle different kinds of measurements more uniformly as we will add more of them.
|
|
31
|
+
|
|
32
|
+ ### Features (Plugins)
|
|
33
|
+ * `Benchee.Statistics` comes with 3 new values: `:relative_more`, `:relative_less`, `:absolute_difference` so that you don't have to calculate these relative values yourself :)
|
|
34
|
+
|
|
1
35
|
## 0.14.0 (2019-02-10)
|
|
2
36
|
|
|
37
|
+ Highlights of this release are a new way to specify formatter options closer to the formatters themselves as well as maximum precision measurements.
|
|
38
|
+
|
|
3
39
|
### Breaking Changes (User Facing)
|
|
4
40
|
* dropped support for Erlang 18.x
|
|
5
41
|
* Formatters no longer have an `output/1` method, instead use `Formatter.output/3` please
|
changed
README.md
|
|
@@ -1,31 +1,35 @@
|
|
1
|
- # Benchee [](https://hex.pm/packages/benchee) [](https://hexdocs.pm/benchee/) [](https://travis-ci.org/PragTob/benchee) [](https://ci.appveyor.com/project/PragTob/benchee/branch/master) [](https://coveralls.io/github/PragTob/benchee?branch=master) [](https://inch-ci.org/github/PragTob/benchee)
|
|
1
|
+ # Benchee [](https://hex.pm/packages/benchee) [](https://hexdocs.pm/benchee/) [](https://travis-ci.org/bencheeorg/benchee) [](https://ci.appveyor.com/project/PragTob/benchee-8e4gw) [](https://coveralls.io/github/bencheeorg/benchee?branch=master)
|
|
2
2
|
|
|
3
|
- Library for easy and nice (micro) benchmarking in Elixir. It allows you to compare the performance of different pieces of code at a glance. Benchee is also versatile and extensible, relying only on functions! There are also a bunch of [plugins](#plugins) to draw pretty graphs and more!
|
|
3
|
+ Library for easy and nice (micro) benchmarking in Elixir. Benchee allows you to compare the performance of different pieces of code at a glance. It is also versatile and extensible, relying only on functions. There are also a bunch of [plugins](#plugins) to draw pretty graphs and more!
|
|
4
4
|
|
|
5
|
- Benchee runs each of your functions for a given amount of time after an initial warmup, it then measures their run time and optionally memory consumption. It then shows different statistical values like average, iterations per second and the standard deviation.
|
|
5
|
+ benchee runs each of your functions for a given amount of time after an initial warmup, it then measures their run time and optionally memory consumption. It then shows different statistical values like average, standard deviation etc. See [features](#features)
|
|
6
6
|
|
|
7
|
- Benchee has a nice and concise main interface, its behavior can be altered through lots of [configuration options](#configuration):
|
|
7
|
+ benchee has a nice and concise main interface, its behavior can be altered through lots of [configuration options](#configuration):
|
|
8
8
|
|
|
9
9
|
```elixir
|
|
10
10
|
list = Enum.to_list(1..10_000)
|
|
11
|
- map_fun = fn(i) -> [i, i * i] end
|
|
11
|
+ map_fun = fn i -> [i, i * i] end
|
|
12
12
|
|
|
13
|
- Benchee.run(%{
|
|
14
|
- "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
|
|
15
|
- "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten end
|
|
16
|
- }, time: 10, memory_time: 2)
|
|
13
|
+ Benchee.run(
|
|
14
|
+ %{
|
|
15
|
+ "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
|
|
16
|
+ "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten() end
|
|
17
|
+ },
|
|
18
|
+ time: 10,
|
|
19
|
+ memory_time: 2
|
|
20
|
+ )
|
|
17
21
|
```
|
|
18
22
|
|
|
19
23
|
Produces the following output on the console:
|
|
20
24
|
|
|
21
25
|
```
|
|
22
|
- tobi@speedy:~/github/benchee(master)$ mix run samples/run.exs
|
|
23
|
- Operating System: Linux"
|
|
26
|
+ tobi@speedy:$ mix run samples/run.exs
|
|
27
|
+ Operating System: Linux
|
|
24
28
|
CPU Information: Intel(R) Core(TM) i7-4790 CPU @ 3.60GHz
|
|
25
29
|
Number of Available Cores: 8
|
|
26
30
|
Available memory: 15.61 GB
|
|
27
|
- Elixir 1.6.4
|
|
28
|
- Erlang 20.3
|
|
31
|
+ Elixir 1.8.1
|
|
32
|
+ Erlang 21.2.7
|
|
29
33
|
|
|
30
34
|
Benchmark suite executing with the following configuration:
|
|
31
35
|
warmup: 2 s
|
|
|
@@ -35,58 +39,56 @@ parallel: 1
|
|
35
39
|
inputs: none specified
|
|
36
40
|
Estimated total run time: 28 s
|
|
37
41
|
|
|
38
|
-
|
|
39
42
|
Benchmarking flat_map...
|
|
40
43
|
Benchmarking map.flatten...
|
|
41
44
|
|
|
42
45
|
Name ips average deviation median 99th %
|
|
43
|
- flat_map 2.31 K 433.25 μs ±8.64% 428 μs 729 μs
|
|
44
|
- map.flatten 1.22 K 822.22 μs ±16.43% 787 μs 1203 μs
|
|
46
|
+ flat_map 2.34 K 426.84 μs ±9.88% 418.72 μs 720.20 μs
|
|
47
|
+ map.flatten 1.18 K 844.08 μs ±19.73% 778.10 μs 1314.87 μs
|
|
45
48
|
|
|
46
49
|
Comparison:
|
|
47
|
- flat_map 2.31 K
|
|
48
|
- map.flatten 1.22 K - 1.90x slower
|
|
50
|
+ flat_map 2.34 K
|
|
51
|
+ map.flatten 1.18 K - 1.98x slower +417.24 μs
|
|
49
52
|
|
|
50
53
|
Memory usage statistics:
|
|
51
54
|
|
|
52
55
|
Name Memory usage
|
|
53
|
- flat_map 625.54 KB
|
|
54
|
- map.flatten 781.85 KB - 1.25x memory usage
|
|
56
|
+ flat_map 624.97 KB
|
|
57
|
+ map.flatten 781.25 KB - 1.25x memory usage +156.28 KB
|
|
55
58
|
|
|
56
59
|
**All measurements for memory usage were the same**
|
|
57
|
-
|
|
58
60
|
```
|
|
59
61
|
|
|
60
|
- The aforementioned [plugins](#plugins) like [benchee_html](https://github.com/PragTob/benchee_html) make it possible to generate nice looking [html reports](https://www.pragtob.info/benchee/flat_map.html), where individual graphs can also be exported as PNG images:
|
|
62
|
+ The aforementioned [plugins](#plugins) like [benchee_html](https://github.com/bencheeorg/benchee_html) make it possible to generate nice looking [html reports](https://www.pragtob.info/benchee/flat_map.html), where individual graphs can also be exported as PNG images:
|
|
61
63
|
|
|
62
64
|
|
|
63
65
|
|
|
64
66
|
## Features
|
|
65
67
|
|
|
66
|
- * first runs the functions for a given warmup time without recording the results, to simulate a _"warm"_ running system
|
|
67
|
- * [measures memory](#measuring-memory-consumption)
|
|
68
|
+ * first runs the functions for a given warmup time without recording the results, to simulate a _"warm"/running_ system
|
|
69
|
+ * [measures memory usage](#measuring-memory-consumption)
|
|
68
70
|
* provides you with lots of statistics - check the next list
|
|
69
|
- * plugin/extensible friendly architecture so you can use different formatters to generate [CSV, HTML and more](#plugins)
|
|
70
|
- * nicely formatted console output with units scaled to appropriate units
|
|
71
|
- * [hooks](#hooks-setup-teardown-etc) to execute something before/after a benchmark
|
|
71
|
+ * plugin/extensible friendly architecture so you can use different formatters to display benchmarking results as [HTML, markdown, JSON and more](#plugins)
|
|
72
|
+ * nicely formatted console output with units scaled to appropriately (nanoseconds to minutes)
|
|
73
|
+ * measures the overhead of function calls so that the measured/reported times really are the execution time of _your_code_ without that overhead.
|
|
74
|
+ * [hooks](#hooks-setup-teardown-etc) to execute something before/after a benchmarking invocation
|
|
72
75
|
* execute benchmark jobs in parallel to gather more results in the same time, or simulate a system under load
|
|
73
|
- * well tested
|
|
74
|
- * well documented
|
|
76
|
+ * well documented & well tested
|
|
75
77
|
|
|
76
78
|
Provides you with the following **statistical data**:
|
|
77
79
|
|
|
78
|
- * **average** - average execution time (the lower the better)
|
|
79
|
- * **ips** - iterations per second, aka how often can the given function be executed within one second (the higher the better)
|
|
80
|
+ * **average** - average execution time/memory usage (the lower the better)
|
|
81
|
+ * **ips** - iterations per second, aka how often can the given function be executed within one second (the higher the better - good for graphing), only for run times
|
|
80
82
|
* **deviation** - standard deviation (how much do the results vary), given as a percentage of the average (raw absolute values also available)
|
|
81
|
- * **median** - when all measured times are sorted, this is the middle value (or average of the two middle values when the number of samples is even). More stable than the average and somewhat more likely to be a typical value you see. (the lower the better)
|
|
82
|
- * **99th %** - 99th percentile, 99% of all run times are less than this
|
|
83
|
+ * **median** - when all measured values are sorted, this is the middle value. More stable than the average and somewhat more likely to be a typical value you see, for the msot typical value see mode. (the lower the better)
|
|
84
|
+ * **99th %** - 99th percentile, 99% of all measured values are less than this - worst case performance-ish
|
|
83
85
|
|
|
84
86
|
In addition, you can optionally output an extended set of statistics:
|
|
85
87
|
|
|
86
|
- * **minimum** - the smallest (fastest) run time measured for the job
|
|
87
|
- * **maximum** - the biggest (slowest) run time measured for the job
|
|
88
|
- * **sample size** - the number of run time measurements taken
|
|
89
|
- * **mode** - the run time(s) that occur the most. Often one value, but can be multiple values if they occur the same amount of times. If no value occurs at least twice, this value will be nil.
|
|
88
|
+ * **minimum** - the smallest value measured for the job (fastest/least consumption)
|
|
89
|
+ * **maximum** - the biggest run time measured for the job (slowest/most consumption)
|
|
90
|
+ * **sample size** - the number of measurements taken
|
|
91
|
+ * **mode** - the measured values that occur the most. Often one value, but can be multiple values if they occur the same amount of times. If no value occurs at least twice, this value will be nil.
|
|
90
92
|
|
|
91
93
|
## Installation
|
|
92
94
|
|
|
|
@@ -94,13 +96,13 @@ Add benchee to your list of dependencies in `mix.exs`:
|
|
94
96
|
|
|
95
97
|
```elixir
|
|
96
98
|
defp deps do
|
|
97
|
- [{:benchee, "~> 0.13", only: :dev}]
|
|
99
|
+ [{:benchee, "~> 0.14", only: :dev}]
|
|
98
100
|
end
|
|
99
101
|
```
|
|
100
102
|
|
|
101
103
|
Install via `mix deps.get` and then happy benchmarking as described in [Usage](#usage) :)
|
|
102
104
|
|
|
103
|
- Elixir versions supported are 1.4+.
|
|
105
|
+ Elixir versions supported/tested against are 1.6+. It _should_ theoretically still work with 1.4 & 1.5 but we don't actively test/support it.
|
|
104
106
|
|
|
105
107
|
## Usage
|
|
106
108
|
|
|
|
@@ -108,12 +110,14 @@ After installing just write a little Elixir benchmarking script:
|
|
108
110
|
|
|
109
111
|
```elixir
|
|
110
112
|
list = Enum.to_list(1..10_000)
|
|
111
|
- map_fun = fn(i) -> [i, i * i] end
|
|
113
|
+ map_fun = fn i -> [i, i * i] end
|
|
112
114
|
|
|
113
|
- Benchee.run(%{
|
|
114
|
- "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
|
|
115
|
- "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten end
|
|
116
|
- })
|
|
115
|
+ Benchee.run(
|
|
116
|
+ %{
|
|
117
|
+ "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
|
|
118
|
+ "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten() end
|
|
119
|
+ }
|
|
120
|
+ )
|
|
117
121
|
```
|
|
118
122
|
|
|
119
123
|
(names can also be specified as `:atoms` if you want to)
|
|
|
@@ -121,37 +125,43 @@ Benchee.run(%{
|
|
121
125
|
This produces the following output:
|
|
122
126
|
|
|
123
127
|
```
|
|
124
|
- tobi@speedy ~/github/benchee $ mix run samples/run.exs
|
|
125
|
- Elixir 1.4.0
|
|
126
|
- Erlang 19.1
|
|
128
|
+ tobi@speedy:$ mix run samples/run_defaults.exs
|
|
129
|
+ Operating System: Linux
|
|
130
|
+ CPU Information: Intel(R) Core(TM) i7-4790 CPU @ 3.60GHz
|
|
131
|
+ Number of Available Cores: 8
|
|
132
|
+ Available memory: 15.61 GB
|
|
133
|
+ Elixir 1.8.1
|
|
134
|
+ Erlang 21.2.7
|
|
135
|
+
|
|
127
136
|
Benchmark suite executing with the following configuration:
|
|
128
|
- warmup: 2.0s
|
|
129
|
- time: 5.0s
|
|
137
|
+ warmup: 2 s
|
|
138
|
+ time: 5 s
|
|
139
|
+ memory time: 0 ns
|
|
130
140
|
parallel: 1
|
|
131
141
|
inputs: none specified
|
|
132
|
- Estimated total run time: 14.0s
|
|
142
|
+ Estimated total run time: 14 s
|
|
133
143
|
|
|
134
144
|
Benchmarking flat_map...
|
|
135
145
|
Benchmarking map.flatten...
|
|
136
146
|
|
|
137
|
- Name ips average deviation median
|
|
138
|
- flat_map 2.28 K 438.07 μs ±16.66% 419 μs
|
|
139
|
- map.flatten 1.25 K 802.99 μs ±13.40% 782 μs
|
|
147
|
+ Name ips average deviation median 99th %
|
|
148
|
+ flat_map 2.34 K 427.78 μs ±16.02% 406.29 μs 743.01 μs
|
|
149
|
+ map.flatten 1.22 K 820.87 μs ±19.29% 772.61 μs 1286.35 μs
|
|
140
150
|
|
|
141
151
|
Comparison:
|
|
142
|
- flat_map 2.28 K
|
|
143
|
- map.flatten 1.25 K - 1.83x slower
|
|
152
|
+ flat_map 2.34 K
|
|
153
|
+ map.flatten 1.22 K - 1.92x slower +393.09 μs
|
|
144
154
|
```
|
|
145
155
|
|
|
146
156
|
See [Features](#features) for a description of the different statistical values and what they mean.
|
|
147
157
|
|
|
148
|
- If you're looking to see how to make something specific work, please refer to the [samples](https://github.com/PragTob/benchee/tree/master/samples) directory. Also, especially when wanting to extend benchee check out the [hexdocs](https://hexdocs.pm/benchee/api-reference.html).
|
|
158
|
+ If you're looking to see how to make something specific work, please refer to the [samples](https://github.com/PragTob/benchee/tree/master/samples) directory. Also, especially when wanting to extend benchee, check out the [hexdocs](https://hexdocs.pm/benchee/api-reference.html).
|
|
149
159
|
|
|
150
160
|
### Configuration
|
|
151
161
|
|
|
152
162
|
Benchee takes a wealth of configuration options, however those are entirely optional. Benchee ships with sensible defaults for all of these.
|
|
153
163
|
|
|
154
|
- In the most common `Benchee.run/2` interface configuration options are passed as the second argument in the form of an optional keyword list:
|
|
164
|
+ In the most common `Benchee.run/2` interface configuration options are passed as the second argument in the form of an keyword list:
|
|
155
165
|
|
|
156
166
|
```elixir
|
|
157
167
|
Benchee.run(%{"some function" => fn -> magic end}, print: [benchmarking: false])
|
|
|
@@ -159,56 +169,54 @@ Benchee.run(%{"some function" => fn -> magic end}, print: [benchmarking: false])
|
|
159
169
|
|
|
160
170
|
The available options are the following (also documented in [hexdocs](https://hexdocs.pm/benchee/Benchee.Configuration.html#init/1)).
|
|
161
171
|
|
|
162
|
- * `warmup` - the time in seconds for which a benchmarking job should be run without measuring times before "real" measurements start. This simulates a _"warm"_ running system. Defaults to 2.
|
|
163
|
- * `time` - the time in seconds for how long each individual benchmarking job should be run for measuring the execution times (run time performance). Defaults to 5.
|
|
172
|
+ * `warmup` - the time in seconds for which a benchmarking job should be run without measuring anything before "real" measurements start. This simulates a _"warm"/running_ system. Defaults to 2.
|
|
173
|
+ * `time` - the time in seconds for how long each individual scenario (benchmarking job x input) should be run for measuring the execution times (run time performance). Defaults to 5.
|
|
164
174
|
* `memory_time` - the time in seconds for how long [memory measurements](measuring-memory-consumption) should be conducted. Defaults to 0 (turned off).
|
|
165
175
|
* `inputs` - a map or list of two element tuples. If a map, they keys are descriptive input names and values are the actual input values. If a list of tuples, the first element in each tuple is the input name, and the second element in each tuple is the actual input value. Your benchmarking jobs will then be run with each of these inputs. For this to work your benchmarking function gets the current input passed in as an argument into the function. Defaults to `nil`, aka no input specified and functions are called without an argument. See [Inputs](#inputs).
|
|
166
|
- * `formatters` - list of formatters either as a module implementing the formatter behaviour, a tuple of said module and options it should take or formatter functions. They are run when using `Benchee.run/2` or you can invoktem them through `Benchee.Formatter.output/1`. Functions need to accept one argument (which is the benchmarking suite with all data) and then use that to produce output. Used for plugins. Defaults to the builtin console formatter `Benchee.Formatters.Console`. See [Formatters](#formatters).
|
|
176
|
+ * `formatters` - list of formatters either as a module implementing the formatter behaviour, a tuple of said module and options it should take or formatter functions. They are run when using `Benchee.run/2` or you can invoke them through `Benchee.Formatter.output/1`. Functions need to accept one argument (which is the benchmarking suite with all data) and then use that to produce output. Used for plugins & configuration. Also allows the configuration of the console formatter to print extended statistics. Defaults to the builtin console formatter `Benchee.Formatters.Console`. See [Formatters](#formatters).
|
|
177
|
+ * `:measure_function_call_overhead` - Measure how long an empty function call takes and deduct this from each measured run time. Defaults to true.
|
|
167
178
|
* `pre_check` - whether or not to run each job with each input - including all given before or after scenario or each hooks - before the benchmarks are measured to ensure that your code executes without error. This can save time while developing your suites. Defaults to `false`.
|
|
168
|
- * `parallel` - the function of each benchmarking job will be executed in `parallel` number processes. If `parallel: 4` then 4 processes will be spawned that all execute the _same_ function for the given time. When these finish/the time is up 4 new processes will be spawned for the next job/function. This gives you more data in the same time, but also puts a load on the system interfering with benchmark results. For more on the pros and cons of parallel benchmarking [check the wiki](https://github.com/PragTob/benchee/wiki/Parallel-Benchmarking). Defaults to 1 (no parallel execution).
|
|
179
|
+ * `parallel` - the function of each benchmarking job will be executed in `parallel` number processes. If `parallel: 4` then 4 processes will be spawned that all execute the _same_ function for the given time. When these finish/the time is up 4 new processes will be spawned for the next scenario. This gives you more data in the same time, but also puts a load on the system interfering with benchmark results. For more on the pros and cons of parallel benchmarking [check the wiki](https://github.com/bencheeorg/benchee/wiki/Parallel-Benchmarking). Defaults to 1 (no parallel execution).
|
|
169
180
|
* `save` - specify a `path` where to store the results of the current benchmarking suite, tagged with the specified `tag`. See [Saving & Loading](#saving-loading-and-comparing-previous-runs).
|
|
170
181
|
* `load` - load saved suit or suits to compare your current benchmarks against. Can be a string or a list of strings or patterns. See [Saving & Loading](#saving-loading-and-comparing-previous-runs).
|
|
171
|
- * `print` - a map from atoms to `true` or `false` to configure if the output identified by the atom will be printed during the standard Benchee benchmarking process. All options are enabled by default (true). Options are:
|
|
172
|
- * `:benchmarking` - print when Benchee starts benchmarking a new job (Benchmarking name ..)
|
|
182
|
+ * `print` - a map or keyword list from atoms to `true` or `false` to configure if the output identified by the atom will be printed during the standard benchee benchmarking process. All options are enabled by default (true). Options are:
|
|
183
|
+ * `:benchmarking` - print when Benchee starts benchmarking a new job (`Benchmarking name ...`)
|
|
173
184
|
* `:configuration` - a summary of configured benchmarking options including estimated total run time is printed before benchmarking starts
|
|
174
185
|
* `:fast_warning` - warnings are displayed if functions are executed too fast leading to inaccurate measures
|
|
175
|
- * `console` - options for the built-in console formatter:
|
|
176
|
- * `:comparison` - if the comparison of the different benchmarking jobs (x times slower than) is shown. Enabled by default.
|
|
177
|
- * `extended_statistics` - display more statistics, aka `minimum`, `maximum`, `sample_size` and `mode`. Disabled by default.
|
|
178
|
- * `:unit_scaling` - the strategy for choosing a unit for durations and
|
|
179
|
- counts. May or may not be implemented by a given formatter (The console
|
|
180
|
- formatter implements it). When scaling a value, Benchee finds the "best fit"
|
|
186
|
+ * `:unit_scaling` - the strategy for choosing a unit for durations,
|
|
187
|
+ counts & memory measurements. May or may not be implemented by a given formatter (The console formatter implements it).
|
|
188
|
+ When scaling a value, benchee finds the "best fit"
|
|
181
189
|
unit (the largest unit for which the result is at least 1). For example,
|
|
182
190
|
1_200_000 scales to `1.2 M`, while `800_000` scales to `800 K`. The
|
|
183
|
- `unit_scaling` strategy determines how Benchee chooses the best fit unit for
|
|
191
|
+ `unit_scaling` strategy determines how benchee chooses the best fit unit for
|
|
184
192
|
an entire list of values, when the individual values in the list may have
|
|
185
193
|
different best fit units. There are four strategies, defaulting to `:best`:
|
|
186
194
|
* `:best` - the most frequent best fit unit will be used, a tie
|
|
187
195
|
will result in the larger unit being selected.
|
|
188
|
- * `:largest` - the largest best fit unit will be used (i.e. thousand
|
|
189
|
- and seconds if values are large enough).
|
|
190
|
- * `:smallest` - the smallest best fit unit will be used (i.e.
|
|
191
|
- millisecond and one)
|
|
192
|
- * `:none` - no unit scaling will occur. Durations will be displayed
|
|
193
|
- in microseconds, and counts will be displayed in ones (this is
|
|
194
|
- equivalent to the behaviour Benchee had pre 0.5.0)
|
|
196
|
+ * `:largest` - the largest best fit unit will be used
|
|
197
|
+ * `:smallest` - the smallest best fit unit will be used
|
|
198
|
+ * `:none` - no unit scaling will occur.
|
|
195
199
|
* `:before_scenario`/`after_scenario`/`before_each`/`after_each` - read up on them in the [hooks section](#hooks-setup-teardown-etc)
|
|
196
|
- * `:measure_function_call_overhead` - Measure how long an empty function call takes and deduct this from each measure run time. Defaults to true.
|
|
197
200
|
|
|
198
201
|
### Measuring memory consumption
|
|
199
202
|
|
|
200
|
- Starting with version 0.13, users can now get measurements of how much memory their benchmarks use. This measurement is **not** the actual effect on the size of the BEAM VM size, but the total amount of memory that was allocated during the execution of a given scenario. This includes all memory that was garbage collected during the execution of that scenario. It **does not** include any memory used in any process other than the original one in which the scenario is run.
|
|
203
|
+ Starting with version 0.13, users can now get measurements of how much memory their benchmarked scenarios use. The measurement is **limited to the process that benche executes your provided code in** - i.e. other processes (like worker pools)/the whole BEAM isn't taken into account.
|
|
204
|
+
|
|
205
|
+ This measurement is **not** the actual effect on the size of the BEAM VM size, but the total amount of memory that was allocated during the execution of a given scenario. This includes all memory that was garbage collected during the execution of that scenario.
|
|
201
206
|
|
|
202
207
|
This measurement of memory does not affect the measurement of run times.
|
|
203
208
|
|
|
204
|
- In cases where all measurements of memory consumption are identical, which happens very frequently, the full statistics will be omitted from the standard console formatter. If your function is deterministic, this will always be the case. Only in functions with some amount of randomness will there be variation in memory usage.
|
|
209
|
+ In cases where all measurements of memory consumption are identical, which happens very frequently, the full statistics will be omitted from the standard console formatter. If your function is deterministic, this should always be the case. Only in functions with some amount of randomness will there be variation in memory usage.
|
|
205
210
|
|
|
206
211
|
Memory measurement is disabled by default, and you can choose to enable it by passing `memory_time: your_seconds` option to `Benchee.run/2`:
|
|
207
212
|
|
|
208
213
|
```elixir
|
|
209
|
- Benchee.run(%{
|
|
210
|
- "something_great" => fn -> cool_stuff end
|
|
211
|
- }, memory_time: 2)
|
|
214
|
+ Benchee.run(
|
|
215
|
+ %{
|
|
216
|
+ "something_great" => fn -> cool_stuff end
|
|
217
|
+ },
|
|
218
|
+ memory_time: 2
|
|
219
|
+ )
|
|
212
220
|
```
|
|
213
221
|
|
|
214
222
|
Memory time can be specified separately as it will often be constant - so it might not need as much measuring time.
|
|
|
@@ -218,41 +226,64 @@ A full example, including an example of the console output, can be found
|
|
218
226
|
|
|
219
227
|
### Inputs
|
|
220
228
|
|
|
221
|
- `:inputs` is a very useful configuration that allows you to run the same benchmarking jobs with different inputs. You specify the inputs as either a map from name (String or atom) to the actual input value or a list of tuples where the first element in each tuple is the name and the second element in the tuple is the value. Functions can have different performance characteristics on differently shaped inputs - be that structure or input size.
|
|
229
|
+ `:inputs` is a very useful configuration that allows you to run the same benchmarking jobs with different inputs. We call this combination a _"scenario"_. You specify the inputs as either a map from name (String or atom) to the actual input value or a list of tuples where the first element in each tuple is the name and the second element in the tuple is the value.
|
|
222
230
|
|
|
223
|
- One of such cases is comparing tail-recursive and body-recursive implementations of `map`. More information in the [repository with the benchmark](https://github.com/PragTob/elixir_playground/blob/master/bench/tco_blog_post_focussed_inputs.exs) and the [blog post](https://pragtob.wordpress.com/2016/06/16/tail-call-optimization-in-elixir-erlang-not-as-efficient-and-important-as-you-probably-think/).
|
|
231
|
+ Why do this? Functions can have different performance characteristics on differently shaped inputs - be that structure or input size. One of such cases is comparing tail-recursive and body-recursive implementations of `map`. More information in the [repository with the benchmark](https://github.com/PragTob/elixir_playground/blob/master/bench/tco_blog_post_focussed_inputs.exs) and the [blog post](https://pragtob.wordpress.com/2016/06/16/tail-call-optimization-in-elixir-erlang-not-as-efficient-and-important-as-you-probably-think/).
|
|
232
|
+
|
|
233
|
+ As a little sample:
|
|
224
234
|
|
|
225
235
|
```elixir
|
|
226
|
- map_fun = fn(i) -> i + 1 end
|
|
227
|
- inputs = %{
|
|
228
|
- "Small (1 Thousand)" => Enum.to_list(1..1_000),
|
|
229
|
- "Middle (100 Thousand)" => Enum.to_list(1..100_000),
|
|
230
|
- "Big (10 Million)" => Enum.to_list(1..10_000_000)
|
|
231
|
- }
|
|
236
|
+ map_fun = fn i -> [i, i * i] end
|
|
232
237
|
|
|
233
|
- # Or inputs could also look like this:
|
|
234
|
- #
|
|
235
|
- # inputs = [
|
|
236
|
- # {"Small (1 Thousand)", Enum.to_list(1..1_000)},
|
|
237
|
- # {"Middle (100 Thousand)", Enum.to_list(1..100_000)},
|
|
238
|
- # {"Big (10 Million)", Enum.to_list(1..10_000_000)}
|
|
239
|
- # ]
|
|
240
|
-
|
|
241
|
- Benchee.run %{
|
|
242
|
- "map tail-recursive" =>
|
|
243
|
- fn(list) -> MyMap.map_tco(list, map_fun) end,
|
|
244
|
- "stdlib map" =>
|
|
245
|
- fn(list) -> Enum.map(list, map_fun) end,
|
|
246
|
- "map simple body-recursive" =>
|
|
247
|
- fn(list) -> MyMap.map_body(list, map_fun) end,
|
|
248
|
- "map tail-recursive different argument order" =>
|
|
249
|
- fn(list) -> MyMap.map_tco_arg_order(list, map_fun) end
|
|
250
|
- }, time: 15, warmup: 5, inputs: inputs
|
|
238
|
+ Benchee.run(
|
|
239
|
+ %{
|
|
240
|
+ "flat_map" => fn input -> Enum.flat_map(input, map_fun) end,
|
|
241
|
+ "map.flatten" => fn input -> input |> Enum.map(map_fun) |> List.flatten() end
|
|
242
|
+ },
|
|
243
|
+ inputs: %{
|
|
244
|
+ "Small" => Enum.to_list(1..1_000),
|
|
245
|
+ "Medium" => Enum.to_list(1..10_000),
|
|
246
|
+ "Bigger" => Enum.to_list(1..100_000)
|
|
247
|
+ }
|
|
248
|
+ )
|
|
251
249
|
```
|
|
252
250
|
|
|
253
|
- This means each function will be benchmarked with each input that is specified in the inputs. Then you'll get the output divided by input so you can see which function is fastest for which input.
|
|
251
|
+ This means each function will be benchmarked with each input that is specified in the inputs. Then you'll get the output divided by input so you can see which function is fastest for which input, like so:
|
|
254
252
|
|
|
255
|
- Therefore, I **highly recommend** using this feature and checking different realistically structured and sized inputs for the functions you benchmark!
|
|
253
|
+ ```
|
|
254
|
+ tobi@speedy:~/github/benchee(readme-overhaul)$ mix run samples/multiple_inputs.exs
|
|
255
|
+
|
|
256
|
+ (... general information ...)
|
|
257
|
+
|
|
258
|
+ ##### With input Bigger #####
|
|
259
|
+ Name ips average deviation median 99th %
|
|
260
|
+ flat_map 150.81 6.63 ms ±12.65% 6.57 ms 8.74 ms
|
|
261
|
+ map.flatten 114.05 8.77 ms ±16.22% 8.42 ms 12.76 ms
|
|
262
|
+
|
|
263
|
+ Comparison:
|
|
264
|
+ flat_map 150.81
|
|
265
|
+ map.flatten 114.05 - 1.32x slower +2.14 ms
|
|
266
|
+
|
|
267
|
+ ##### With input Medium #####
|
|
268
|
+ Name ips average deviation median 99th %
|
|
269
|
+ flat_map 2.28 K 437.80 μs ±10.72% 425.63 μs 725.09 μs
|
|
270
|
+ map.flatten 1.78 K 561.18 μs ±5.55% 553.98 μs 675.98 μs
|
|
271
|
+
|
|
272
|
+ Comparison:
|
|
273
|
+ flat_map 2.28 K
|
|
274
|
+ map.flatten 1.78 K - 1.28x slower +123.37 μs
|
|
275
|
+
|
|
276
|
+ ##### With input Small #####
|
|
277
|
+ Name ips average deviation median 99th %
|
|
278
|
+ flat_map 26.31 K 38.01 μs ±15.47% 36.69 μs 67.08 μs
|
|
279
|
+ map.flatten 18.65 K 53.61 μs ±11.32% 52.79 μs 70.17 μs
|
|
280
|
+
|
|
281
|
+ Comparison:
|
|
282
|
+ flat_map 26.31 K
|
|
283
|
+ map.flatten 18.65 K - 1.41x slower +15.61 μs
|
|
284
|
+ ```
|
|
285
|
+
|
|
286
|
+ Therefore, we **highly recommend** using this feature and checking different realistically structured and sized inputs for the functions you benchmark!
|
|
256
287
|
|
|
257
288
|
### Formatters
|
|
258
289
|
|
|
|
@@ -263,16 +294,17 @@ The `:formatters` option is specified a list of:
|
|
263
294
|
* a tuple of a module specified above and options for it `{module, options}`
|
|
264
295
|
* functions that take one argument (the benchmarking suite with all its results) and then do whatever you want them to
|
|
265
296
|
|
|
266
|
- So if you are using the [HTML plugin](https://github.com/PragTob/benchee_html) and you want to run both the console formatter and the HTML formatter this looks like this (after you installed it of course):
|
|
297
|
+ So if you are using the [HTML plugin](https://github.com/bencheeorg/benchee_html) and you want to run both the console formatter and the HTML formatter this looks like this (after you installed it of course):
|
|
267
298
|
|
|
268
299
|
```elixir
|
|
269
300
|
list = Enum.to_list(1..10_000)
|
|
270
|
- map_fun = fn(i) -> [i, i * i] end
|
|
301
|
+ map_fun = fn i -> [i, i * i] end
|
|
271
302
|
|
|
272
|
- Benchee.run(%{
|
|
273
|
- "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
|
|
274
|
- "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten end
|
|
275
|
- },
|
|
303
|
+ Benchee.run(
|
|
304
|
+ %{
|
|
305
|
+ "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
|
|
306
|
+ "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten end
|
|
307
|
+ },
|
|
276
308
|
formatters: [
|
|
277
309
|
{Benchee.Formatters.HTML, file: "samples_output/my.html"},
|
|
278
310
|
Benchee.Formatters.Console
|
|
|
@@ -280,29 +312,39 @@ Benchee.run(%{
|
|
280
312
|
)
|
|
281
313
|
```
|
|
282
314
|
|
|
283
|
- ### Extended Console Formatter Statistics
|
|
315
|
+ #### Console Formatter options
|
|
284
316
|
|
|
285
|
- Showing more statistics such as `minimum`, `maximum`, `sample_size` and `mode` is as simple as passing `extended_statistics: true` to the console formatter.
|
|
317
|
+ The console formatter supports 2 configuration options:
|
|
318
|
+
|
|
319
|
+ * `:comparison` - if the comparison of the different benchmarking jobs (x times slower than) is shown. Enabled by default.
|
|
320
|
+ * `extended_statistics` - display more statistics, aka `minimum`, `maximum`, `sample_size` and `mode`. Disabled by default.
|
|
321
|
+
|
|
322
|
+ So if you want to see more statistics you simple pass `extended_statistics: true` to the console formatter:
|
|
286
323
|
|
|
287
324
|
```elixir
|
|
288
325
|
list = Enum.to_list(1..10_000)
|
|
289
|
- map_fun = fn(i) -> [i, i * i] end
|
|
326
|
+ map_fun = fn i -> [i, i * i] end
|
|
290
327
|
|
|
291
|
- Benchee.run(%{
|
|
292
|
- "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
|
|
293
|
- "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten end
|
|
294
|
- }, time: 10, formatters: [{Benchee.Formatters.Console, extended_statistics: true}])
|
|
328
|
+ Benchee.run(
|
|
329
|
+ %{
|
|
330
|
+ "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
|
|
331
|
+ "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten() end
|
|
332
|
+ },
|
|
333
|
+ time: 10,
|
|
334
|
+ formatters: [{Benchee.Formatters.Console, extended_statistics: true}]
|
|
335
|
+ )
|
|
295
336
|
```
|
|
296
337
|
|
|
297
338
|
Which produces:
|
|
298
339
|
|
|
299
340
|
```
|
|
300
|
- # your normal output...
|
|
341
|
+ (... normal output ...)
|
|
301
342
|
|
|
302
343
|
Extended statistics:
|
|
344
|
+
|
|
303
345
|
Name minimum maximum sample size mode
|
|
304
|
- flat_map 365 μs 1371 μs 22.88 K 430 μs
|
|
305
|
- map.flatten 514 μs 1926 μs 13.01 K 517 μs
|
|
346
|
+ flat_map 345.43 μs 1195.89 μs 23.73 K 405.64 μs
|
|
347
|
+ map.flatten 522.99 μs 2105.56 μs 12.03 K 767.83 μs, 768.44 μs
|
|
306
348
|
```
|
|
307
349
|
|
|
308
350
|
(Btw. notice how the modes of both are much closer and for `map.flatten` much less than the average of `766.99`, see `samples/run_extended_statistics`)
|
|
|
@@ -314,12 +356,13 @@ Benchee can store the results of previous runs in a file and then load them agai
|
|
314
356
|
**Saving** is done through the `save` configuration option. You can specify a `path` where results are saved, or you can use the default option of`"benchmark.benchee"` if you don't pass a `path`. You can also pass a `tag` option which annotates these results (for instance with a branch name). The default option for the `tag` is a timestamp of when the benchmark was run.
|
|
315
357
|
|
|
316
358
|
**Loading** is done through the `load` option specifying a path to the file to
|
|
317
|
- load (for instance `benchmark.benchee`). You can also specify multiple files to load through a list of paths (`["my.benchee", "master_save.benchee"]`) - each one of those can also be a glob expression to match even more files glob (`"save_number*.benchee"`).
|
|
359
|
+ load (for instance `"benchmark.benchee"`). You can also specify multiple files to load through a list of paths (`["my.benchee", "master_save.benchee"]`) - each one of those can also be a glob expression to match even more files glob (`"save_number*.benchee"`).
|
|
318
360
|
|
|
319
361
|
```elixir
|
|
320
|
- Benchee.run(%{
|
|
321
|
- "something_great" => fn -> cool_stuff end
|
|
322
|
- },
|
|
362
|
+ Benchee.run(
|
|
363
|
+ %{
|
|
364
|
+ "something_great" => fn -> cool_stuff end
|
|
365
|
+ },
|
|
323
366
|
save: [path: "save.benchee", tag: "first-try"]
|
|
324
367
|
)
|
|
325
368
|
|
|
|
@@ -330,7 +373,7 @@ In the more verbose API this is triggered via `Benchee.load/1`.
|
|
330
373
|
|
|
331
374
|
### Hooks (Setup, Teardown etc.)
|
|
332
375
|
|
|
333
|
- Most of the time, it's best to keep your benchmarks as simple as possible: plain old immutable functions work best. But sometimes you need other things to happen. When you want to add before or after hooks to your benchmarks, we've got you covered! Before you dig into this section, **you usually don't need hooks**.
|
|
376
|
+ Most of the time, it's best to keep your benchmarks as simple as possible: plain old immutable functions work best. But sometimes you need other things to happen. When you want to add before or after hooks to your benchmarks, we've got you covered! Before you dig into this section though remember one thing: **you usually don't need hooks!**
|
|
334
377
|
|
|
335
378
|
Benchee has three types of hooks:
|
|
336
379
|
|
|
|
@@ -339,7 +382,7 @@ Benchee has three types of hooks:
|
|
339
382
|
* [Benchmarking function hooks](#benchmarking-function-hooks)
|
|
340
383
|
|
|
341
384
|
|
|
342
|
- Of course, **hooks are not included in the measured run times**. So they are there especially if you want to do something and want it to **not be included in the measured times**. Sadly there is the notable exception of _too_fast_functions_ (the ones that execute in less than 10 microseconds). As we need to measure their repeated invocations to get halfway good measurements `before_each` and `after_each` hooks are included there.
|
|
385
|
+ Of course, **hooks are not included in the measurements**. So they are there especially if you want to do something and want it to **not be included in the measurements**. Sadly there is the notable exception of _too_fast_functions_ (the ones that execute faster than we can measure in _native_ resolution). As we need to measure their repeated invocations to get halfway good measurements `before_each` and `after_each` hooks are included there. However, to the best of our knowledge this should only ever happen on Windows (because of the bad run time measurement accuracy).
|
|
343
386
|
|
|
344
387
|
#### Suite hooks
|
|
345
388
|
|
|
|
@@ -348,7 +391,7 @@ It is very easy in benchee to do setup and teardown for the whole benchmarking s
|
|
348
391
|
```elixir
|
|
349
392
|
your_setup()
|
|
350
393
|
|
|
351
|
- Benchee.run %{"Awesome stuff" => fn -> magic end }
|
|
394
|
+ Benchee.run(%{"Awesome stuff" => fn -> magic end})
|
|
352
395
|
|
|
353
396
|
your_teardown()
|
|
354
397
|
```
|
|
|
@@ -367,13 +410,16 @@ For the following discussions, it's important to know what benchee considers a "
|
|
367
410
|
A scenario is the combination of one benchmarking function and one input. So, given this benchmark:
|
|
368
411
|
|
|
369
412
|
```elixir
|
|
370
|
- Benchee.run %{
|
|
371
|
- "foo" => fn(input) -> ... end,
|
|
372
|
- "bar" => fn(input) -> ... end
|
|
373
|
- }, inputs: %{
|
|
374
|
- "input 1" => 1,
|
|
375
|
- "input 2" => 2
|
|
376
|
- }
|
|
413
|
+ Benchee.run(
|
|
414
|
+ %{
|
|
415
|
+ "foo" => fn input -> ... end,
|
|
416
|
+ "bar" => fn input -> ... end
|
|
417
|
+ },
|
|
418
|
+ inputs: %{
|
|
419
|
+ "input 1" => 1,
|
|
420
|
+ "input 2" => 2
|
|
421
|
+ }
|
|
422
|
+ )
|
|
377
423
|
```
|
|
378
424
|
|
|
379
425
|
there are 4 scenarios:
|
|
|
@@ -383,11 +429,11 @@ there are 4 scenarios:
|
|
383
429
|
3. bar with input 1
|
|
384
430
|
4. bar with input 2
|
|
385
431
|
|
|
386
|
- A scenario includes warmup and actual run time.
|
|
432
|
+ A scenario includes warmup and actual run time (+ other measurements like memory).
|
|
387
433
|
|
|
388
434
|
##### before_scenario
|
|
389
435
|
|
|
390
|
- Is executed before every [scenario](#what-is-a-scenario) that it applies to (see [hook configuration](#hook-configuration-global-versus-local)). Before scenario hooks take the input of the scenario as an argument.
|
|
436
|
+ Is executed before every [scenario](#what-is-a-scenario) that it applies to (see [hook configuration](#hook-configuration-global-versus-local)). `before_scenario` hooks take the input of the scenario as an argument.
|
|
391
437
|
|
|
392
438
|
Since the return value of a `before_scenario` becomes the input for next steps (see [hook arguments and return values](#hook-arguments-and-return-values)), there usually are 3 kinds of before scenarios:
|
|
393
439
|
|
|
|
@@ -400,22 +446,25 @@ For before scenario hooks, the _global_ hook is invoked first, then the _local_
|
|
400
446
|
Usage:
|
|
401
447
|
|
|
402
448
|
```elixir
|
|
403
|
- Benchee.run %{
|
|
404
|
- "foo" =>
|
|
405
|
- {
|
|
406
|
- fn({input, resource}) -> foo(input, resource) end,
|
|
407
|
- before_scenario: fn({input, resource}) ->
|
|
408
|
- resource = alter_resource(resource)
|
|
409
|
- {input, resource}
|
|
410
|
- end
|
|
411
|
- },
|
|
412
|
- "bar" =>
|
|
413
|
- fn({input, resource}) -> bar(input, resource) end
|
|
414
|
- }, inputs: %{"input 1" => 1},
|
|
415
|
- before_scenario: fn(input) ->
|
|
416
|
- resource = start_expensive_resource()
|
|
417
|
- {input, resource}
|
|
418
|
- end
|
|
449
|
+ Benchee.run(
|
|
450
|
+ %{
|
|
451
|
+ "foo" =>
|
|
452
|
+ {
|
|
453
|
+ fn {input, resource} -> foo(input, resource) end,
|
|
454
|
+ before_scenario: fn {input, resource} ->
|
|
455
|
+ resource = alter_resource(resource)
|
|
456
|
+ {input, resource}
|
|
457
|
+ end
|
|
458
|
+ },
|
|
459
|
+ "bar" =>
|
|
460
|
+ fn {input, resource} -> bar(input, resource) end
|
|
461
|
+ },
|
|
462
|
+ inputs: %{"input 1" => 1},
|
|
463
|
+ before_scenario: fn input ->
|
|
464
|
+ resource = start_expensive_resource()
|
|
465
|
+ {input, resource}
|
|
466
|
+ end
|
|
467
|
+ )
|
|
419
468
|
```
|
|
420
469
|
|
|
421
470
|
_When might this be useful?_
|
|
|
@@ -433,9 +482,12 @@ For after scenario hooks, the _local_ hook is invoked first, then the _global_ (
|
|
433
482
|
Usage:
|
|
434
483
|
|
|
435
484
|
```elixir
|
|
436
|
- Benchee.run %{
|
|
437
|
- "bar" => fn -> bar() end
|
|
438
|
- }, after_scenario: fn(_input) -> bust_my_cache() end
|
|
485
|
+ Benchee.run(
|
|
486
|
+ %{
|
|
487
|
+ "bar" => fn -> bar() end
|
|
488
|
+ },
|
|
489
|
+ after_scenario: fn _input -> bust_my_cache() end
|
|
490
|
+ )
|
|
439
491
|
```
|
|
440
492
|
|
|
441
493
|
_When might this be useful?_
|
|
|
@@ -457,17 +509,21 @@ For before each hooks, the _global_ hook is invoked first, then the _local_ (see
|
|
457
509
|
Usage:
|
|
458
510
|
|
|
459
511
|
```elixir
|
|
460
|
- Benchee.run %{
|
|
461
|
- "bar" => fn(record) -> bar(record) end
|
|
462
|
- }, inputs: %{ "record id" => 1},
|
|
463
|
- before_each: fn(input) -> get_from_db(input) end
|
|
512
|
+ Benchee.run(
|
|
513
|
+ %{
|
|
514
|
+ "bar" => fn record -> bar(record) end
|
|
515
|
+ },
|
|
516
|
+ inputs: %{ "record id" => 1},
|
|
517
|
+ before_each: fn input -> get_from_db(input) end
|
|
518
|
+ )
|
|
464
519
|
```
|
|
465
520
|
|
|
466
521
|
_When might this be useful?_
|
|
467
522
|
|
|
468
|
- * Retrieving a record from the database and passing it on to the benchmarking function to do something(tm) without the retrieval from the database adding to the benchmark measurement
|
|
523
|
+ * Retrieving a record from the database and passing it on to the benchmarking function to do _something(tm)_ without the retrieval from the database adding to the benchmark measurement
|
|
469
524
|
* Busting caches so that all measurements are taken in an uncached state
|
|
470
525
|
* Picking a random value from a collection and passing it to the benchmarking function for measuring performance with a wider spread of values
|
|
526
|
+ * you could also use this to benchmark with random data like `StreamData`, [devon shows how it's done here](https://devonestes.herokuapp.com/benchmarking-with-stream-data)
|
|
471
527
|
|
|
472
528
|
##### after_each
|
|
473
529
|
|
|
|
@@ -478,10 +534,13 @@ For after each hooks, the _local_ hook is invoked first, then the _global_ (see
|
|
478
534
|
Usage:
|
|
479
535
|
|
|
480
536
|
```elixir
|
|
481
|
- Benchee.run %{
|
|
482
|
- "bar" => fn(input) -> bar(input) end
|
|
483
|
- }, inputs: %{ "input 1" => 1}.
|
|
484
|
- after_each: fn(result) -> assert result == 42 end
|
|
537
|
+ Benchee.run(
|
|
538
|
+ %{
|
|
539
|
+ "bar" => fn input -> bar(input) end
|
|
540
|
+ },
|
|
541
|
+ inputs: %{ "input 1" => 1},
|
|
542
|
+ after_each: fn result -> assert result == 42 end
|
|
543
|
+ )
|
|
485
544
|
```
|
|
486
545
|
|
|
487
546
|
_When might this be useful?_
|
|
|
@@ -507,14 +566,17 @@ Hooks can be defined either _globally_ as part of the configuration or _locally_
|
|
507
566
|
Global hooks are specified as part of the general benchee configuration:
|
|
508
567
|
|
|
509
568
|
```elixir
|
|
510
|
- Benchee.run %{
|
|
511
|
- "foo" => fn(input) -> ... end,
|
|
512
|
- "bar" => fn(input) -> ... end
|
|
513
|
- }, inputs: %{
|
|
514
|
- "input 1" => 1,
|
|
515
|
- "input 2" => 2,
|
|
569
|
+ Benchee.run(
|
|
570
|
+ %{
|
|
571
|
+ "foo" => fn input -> ... end,
|
|
572
|
+ "bar" => fn input -> ... end
|
|
516
573
|
},
|
|
517
|
- before_scenario: fn(input) -> ... end
|
|
574
|
+ inputs: %{
|
|
575
|
+ "input 1" => 1,
|
|
576
|
+ "input 2" => 2,
|
|
577
|
+ },
|
|
578
|
+ before_scenario: fn input -> ... end
|
|
579
|
+ )
|
|
518
580
|
```
|
|
519
581
|
|
|
520
582
|
Here the `before_scenario` function will be executed for all 4 scenarios present in this benchmarking suite.
|
|
|
@@ -524,13 +586,16 @@ Here the `before_scenario` function will be executed for all 4 scenarios present
|
|
524
586
|
Local hooks are defined alongside the benchmarking function. To define a local hook, pass a tuple in the initial map, instead of just a single function. The benchmarking function comes first, followed by a keyword list specifying the hooks to run:
|
|
525
587
|
|
|
526
588
|
```elixir
|
|
527
|
- Benchee.run %{
|
|
528
|
- "foo" => {fn(input) -> ... end, before_scenario: fn(input) -> ... end},
|
|
529
|
- "bar" => fn(input) -> ... end
|
|
530
|
- }, inputs: %{
|
|
531
|
- "input 1" => 1,
|
|
532
|
- "input 2" => 2
|
|
533
|
- }
|
|
589
|
+ Benchee.run(
|
|
590
|
+ %{
|
|
591
|
+ "foo" => {fn input -> ... end, before_scenario: fn input -> ... end},
|
|
592
|
+ "bar" => fn input -> ... end
|
|
593
|
+ },
|
|
594
|
+ inputs: %{
|
|
595
|
+ "input 1" => 1,
|
|
596
|
+ "input 2" => 2
|
|
597
|
+ }
|
|
598
|
+ )
|
|
534
599
|
```
|
|
535
600
|
|
|
536
601
|
Here `before_scenario` is only run for the 2 scenarios associated with `"foo"`, i.e. foo with input 1 and foo with input 2. It is _not_ run for any `"bar"` benchmarks.
|
|
|
@@ -546,42 +611,45 @@ Given the following code:
|
|
546
611
|
|
|
547
612
|
suite_set_up()
|
|
548
613
|
|
|
549
|
- Benchee.run %{
|
|
550
|
- "foo" =>
|
|
551
|
- {
|
|
552
|
- fn(input) -> foo(input) end,
|
|
553
|
- before_scenario: fn(input) ->
|
|
554
|
- local_before_scenario(input)
|
|
555
|
- input + 1
|
|
556
|
- end,
|
|
557
|
- before_each: fn(input) ->
|
|
558
|
- local_before_each(input)
|
|
559
|
- input + 1
|
|
560
|
- end,
|
|
561
|
- after_each: fn(value) ->
|
|
562
|
- local_after_each(value)
|
|
563
|
- end,
|
|
564
|
- after_scenario: fn(input) ->
|
|
565
|
- local_after_scenario(input)
|
|
566
|
- end
|
|
567
|
- },
|
|
568
|
- "bar" =>
|
|
569
|
- fn(input) -> bar(input) end
|
|
570
|
- }, inputs: %{"input 1" => 1},
|
|
571
|
- before_scenario: fn(input) ->
|
|
572
|
- global_before_scenario(input)
|
|
573
|
- input + 1
|
|
574
|
- end,
|
|
575
|
- before_each: fn(input) ->
|
|
576
|
- global_before_each(input)
|
|
577
|
- input + 1
|
|
578
|
- end,
|
|
579
|
- after_each: fn(value) ->
|
|
580
|
- global_after_each(value)
|
|
581
|
- end,
|
|
582
|
- after_scenario: fn(input) ->
|
|
583
|
- global_after_scenario(input)
|
|
584
|
- end
|
|
614
|
+ Benchee.run(
|
|
615
|
+ %{
|
|
616
|
+ "foo" =>
|
|
617
|
+ {
|
|
618
|
+ fn input -> foo(input) end,
|
|
619
|
+ before_scenario: fn input ->
|
|
620
|
+ local_before_scenario(input)
|
|
621
|
+ input + 1
|
|
622
|
+ end,
|
|
623
|
+ before_each: fn input ->
|
|
624
|
+ local_before_each(input)
|
|
625
|
+ input + 1
|
|
626
|
+ end,
|
|
627
|
+ after_each: fn value ->
|
|
628
|
+ local_after_each(value)
|
|
629
|
+ end,
|
|
630
|
+ after_scenario: fn input ->
|
|
631
|
+ local_after_scenario(input)
|
|
632
|
+ end
|
|
633
|
+ },
|
|
634
|
+ "bar" =>
|
|
635
|
+ fn input -> bar(input) end
|
|
636
|
+ },
|
|
637
|
+ inputs: %{"input 1" => 1},
|
|
638
|
+ before_scenario: fn input ->
|
|
639
|
+ global_before_scenario(input)
|
|
640
|
+ input + 1
|
|
641
|
+ end,
|
|
642
|
+ before_each: fn input ->
|
|
643
|
+ global_before_each(input)
|
|
644
|
+ input + 1
|
|
645
|
+ end,
|
|
646
|
+ after_each: fn value ->
|
|
647
|
+ global_after_each(value)
|
|
648
|
+ end,
|
|
649
|
+ after_scenario: fn input ->
|
|
650
|
+ global_after_scenario(input)
|
|
651
|
+ end
|
|
652
|
+ )
|
|
585
653
|
|
|
586
654
|
suite_tear_down()
|
|
587
655
|
```
|
|
|
@@ -638,17 +706,20 @@ It is important to note that the benchmarking code shown in the beginning is the
|
|
638
706
|
|
|
639
707
|
```elixir
|
|
640
708
|
list = Enum.to_list(1..10_000)
|
|
641
|
- map_fun = fn(i) -> [i, i * i] end
|
|
709
|
+ map_fun = fn i -> [i, i * i] end
|
|
642
710
|
|
|
643
|
- Benchee.init(time: 3)
|
|
644
|
- |> Benchee.system
|
|
711
|
+ [time: 3]
|
|
712
|
+ |> Benchee.init()
|
|
713
|
+ |> Benchee.system()
|
|
645
714
|
|> Benchee.benchmark("flat_map", fn -> Enum.flat_map(list, map_fun) end)
|
|
646
|
- |> Benchee.benchmark("map.flatten",
|
|
647
|
- fn -> list |> Enum.map(map_fun) |> List.flatten end)
|
|
648
|
- |> Benchee.measure
|
|
649
|
- |> Benchee.statistics
|
|
650
|
- |> Benchee.load # can be omitted when you don't want to/need to load scenarios
|
|
651
|
- |> Benchee.Formatter.output(Benchee.Formatters.Console, %{})
|
|
715
|
+ |> Benchee.benchmark(
|
|
716
|
+ "map.flatten",
|
|
717
|
+ fn -> list |> Enum.map(map_fun) |> List.flatten() end
|
|
718
|
+ )
|
|
719
|
+ |> Benchee.collect()
|
|
720
|
+ |> Benchee.statistics()
|
|
721
|
+ |> Benchee.relative_statistics()
|
|
722
|
+ |> Benchee.Formatter.output(Benchee.Formatters.Console)
|
|
652
723
|
# Instead of the last call you could also just use Benchee.Formatter.output()
|
|
653
724
|
# to just output all configured formatters
|
|
654
725
|
```
|
|
|
@@ -659,12 +730,13 @@ This is a take on the _functional transformation_ of data applied to benchmarks:
|
|
659
730
|
2. Gather System data
|
|
660
731
|
3. Define the functions to be benchmarked
|
|
661
732
|
4. Run benchmarks with the given configuration gathering raw run times per function
|
|
662
|
- 5. Generate statistics based on the raw run times
|
|
663
|
- 6. Format the statistics in a suitable way and print them out
|
|
733
|
+ 5. Calculate statistics based on the raw run times
|
|
734
|
+ 6. Calculate statistics between the scenarios (faster/slower...)
|
|
735
|
+ 7. Format the statistics in a suitable way and print them out
|
|
664
736
|
|
|
665
737
|
This is also part of the **official API** and allows for more **fine grained control**. (It's also what benchee does internally when you use `Benchee.run/2`).
|
|
666
738
|
|
|
667
|
- Do you just want to have all the raw run times? Just work with the result of `Benchee.measure/1`! Just want to have the calculated statistics and use your own formatting? Grab the result of `Benchee.statistics/1`! Or, maybe you want to write to a file or send an HTTP post to some online service? Just grab the complete suite after statistics were generated.
|
|
739
|
+ Do you just want to have all the raw run times? Just work with the result of `Benchee.collect/1`! Just want to have the calculated statistics and use your own formatting? Grab the result of `Benchee.statistics/1`! Or, maybe you want to write to a file or send an HTTP post to some online service? Just grab the complete suite after statistics were generated.
|
|
668
740
|
|
|
669
741
|
It also allows you to alter behaviour, normally `Benchee.load/1` is called right before the formatters so that neither the benchmarks are run again or statistics are computed again. However, you might want to run the benchmarks again or recompute the statistics. Then you can call `Benchee.load/1` right at the start.
|
|
670
742
|
|
|
|
@@ -737,8 +809,9 @@ This doesn't seem to be too reliable right now, so suggestions and input are ver
|
|
737
809
|
Benchee only has small runtime dependencies that were initially extracted from it. Further functionality is provided through plugins that then pull in dependencies, such as HTML generation and CSV export. They help provide excellent visualization or interoperability.
|
|
738
810
|
|
|
739
811
|
* [benchee_html](//github.com/PragTob/benchee_html) - generate HTML including a data table and many different graphs with the possibility to export individual graphs as PNG :)
|
|
740
|
- * [benchee_csv](//github.com/PragTob/benchee_csv) - generate CSV from your Benchee benchmark results so you can import them into your favorite spreadsheet tool and make fancy graphs
|
|
812
|
+ * [benchee_csv](//github.com/PragTob/benchee_csv) - generate CSV from your benchee benchmark results so you can import them into your favorite spreadsheet tool and make fancy graphs
|
|
741
813
|
* [benchee_json](//github.com/PragTob/benchee_json) - export suite results as JSON to feed anywhere or feed it to your JavaScript and make magic happen :)
|
|
814
|
+ * [benchee_markdown](//github.com/hrzndhrn/benchee_markdown) - write markdown files containing your benchmarking results
|
|
742
815
|
|
|
743
816
|
With the HTML plugin for instance you can get fancy graphs like this boxplot:
|
|
744
817
|
|
|
|
@@ -748,23 +821,17 @@ Of course there also are normal bar charts including standard deviation:
|
|
748
821
|
|
|
749
822
|
|
|
750
823
|
|
|
751
|
- ## Presentation + general benchmarking advice
|
|
752
|
-
|
|
753
|
- If you're into watching videos of conference talks and also want to learn more about benchmarking in general I can recommend watching my talk from [ElixirLive 2016](https://www.elixirlive.com/). [Slides can be found here](https://pragtob.wordpress.com/2016/12/03/slides-how-fast-is-it-really-benchmarking-in-elixir/), video - click the washed out image below ;)
|
|
754
|
-
|
|
755
|
- [](https://www.youtube.com/watch?v=7-mE5CKXjkw)
|
|
756
|
-
|
|
757
824
|
## Contributing [](https://www.codetriage.com/pragtob/benchee)
|
|
758
825
|
|
|
759
826
|
Contributions to benchee are **very welcome**! Bug reports, documentation, spelling corrections, whole features, feature ideas, bugfixes, new plugins, fancy graphics... all of those (and probably more) are much appreciated contributions!
|
|
760
827
|
|
|
761
828
|
Keep in mind that the [plugins](#plugins) live in their own repositories with their own issue tracker and they also like to get contributions :)
|
|
762
829
|
|
|
763
|
- Please respect the [Code of Conduct](//github.com/PragTob/benchee/blob/master/CODE_OF_CONDUCT.md).
|
|
830
|
+ Please respect the [Code of Conduct](//github.com/bencheeorg/benchee/blob/master/CODE_OF_CONDUCT.md).
|
|
764
831
|
|
|
765
832
|
In addition to contributing code, you can help to triage issues. This can include reproducing bug reports, or asking for vital information such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to pragtob/benchee on CodeTriage](https://www.codetriage.com/pragtob/benchee).
|
|
766
833
|
|
|
767
|
- You can also look directly at the [open issues](https://github.com/PragTob/benchee/issues). There are `help wanted` and `good first issue` labels - those are meant as guidance, of course other issues can be tackled :)
|
|
834
|
+ You can also look directly at the [open issues](https://github.com/bencheeorg/benchee/issues). There are `help wanted` and `good first issue` labels - those are meant as guidance, of course other issues can be tackled :)
|
|
768
835
|
|
|
769
836
|
A couple of (hopefully) helpful points:
|
changed
hex_metadata.config
|
|
@@ -2,10 +2,11 @@
|
|
2
2
|
{<<"build_tools">>,[<<"mix">>]}.
|
|
3
3
|
{<<"description">>,
|
|
4
4
|
<<"Versatile (micro) benchmarking that is extensible. Get statistics such as:\naverage, iterations per second, standard deviation and the median.">>}.
|
|
5
|
- {<<"elixir">>,<<"~> 1.4">>}.
|
|
5
|
+ {<<"elixir">>,<<"~> 1.6">>}.
|
|
6
6
|
{<<"files">>,
|
|
7
7
|
[<<"lib">>,<<"lib/benchee.ex">>,<<"lib/benchee">>,
|
|
8
|
- <<"lib/benchee/formatters">>,<<"lib/benchee/formatters/console">>,
|
|
8
|
+ <<"lib/benchee/scenario.ex">>,<<"lib/benchee/formatters">>,
|
|
9
|
+ <<"lib/benchee/formatters/console">>,
|
|
9
10
|
<<"lib/benchee/formatters/console/memory.ex">>,
|
|
10
11
|
<<"lib/benchee/formatters/console/run_time.ex">>,
|
|
11
12
|
<<"lib/benchee/formatters/console/helpers.ex">>,
|
|
|
@@ -13,19 +14,17 @@
|
|
13
14
|
<<"lib/benchee/formatters/tagged_save.ex">>,<<"lib/benchee/output">>,
|
|
14
15
|
<<"lib/benchee/output/benchmark_printer.ex">>,<<"lib/benchee/system.ex">>,
|
|
15
16
|
<<"lib/benchee/statistics.ex">>,<<"lib/benchee/benchmark">>,
|
|
16
|
- <<"lib/benchee/benchmark/scenario.ex">>,
|
|
17
|
- <<"lib/benchee/benchmark/measure.ex">>,
|
|
18
17
|
<<"lib/benchee/benchmark/runner.ex">>,
|
|
19
18
|
<<"lib/benchee/benchmark/repeated_measurement.ex">>,
|
|
20
|
- <<"lib/benchee/benchmark/hooks.ex">>,
|
|
19
|
+ <<"lib/benchee/benchmark/collect.ex">>,<<"lib/benchee/benchmark/hooks.ex">>,
|
|
20
|
+ <<"lib/benchee/benchmark/collect">>,
|
|
21
|
+ <<"lib/benchee/benchmark/collect/time.ex">>,
|
|
22
|
+ <<"lib/benchee/benchmark/collect/native_time.ex">>,
|
|
23
|
+ <<"lib/benchee/benchmark/collect/memory.ex">>,
|
|
21
24
|
<<"lib/benchee/benchmark/scenario_context.ex">>,
|
|
22
|
- <<"lib/benchee/benchmark/measure">>,
|
|
23
|
- <<"lib/benchee/benchmark/measure/time.ex">>,
|
|
24
|
- <<"lib/benchee/benchmark/measure/native_time.ex">>,
|
|
25
|
- <<"lib/benchee/benchmark/measure/memory.ex">>,
|
|
26
|
- <<"lib/benchee/conversion.ex">>,<<"lib/benchee/configuration.ex">>,
|
|
27
|
- <<"lib/benchee/conversion">>,<<"lib/benchee/conversion/count.ex">>,
|
|
28
|
- <<"lib/benchee/conversion/scale.ex">>,
|
|
25
|
+ <<"lib/benchee/collection_data.ex">>,<<"lib/benchee/conversion.ex">>,
|
|
26
|
+ <<"lib/benchee/configuration.ex">>,<<"lib/benchee/conversion">>,
|
|
27
|
+ <<"lib/benchee/conversion/count.ex">>,<<"lib/benchee/conversion/scale.ex">>,
|
|
29
28
|
<<"lib/benchee/conversion/format.ex">>,
|
|
30
29
|
<<"lib/benchee/conversion/memory.ex">>,
|
|
31
30
|
<<"lib/benchee/conversion/deviation_percent.ex">>,
|
|
|
@@ -37,7 +36,8 @@
|
|
37
36
|
<<"lib/benchee/utility/file_creation.ex">>,
|
|
38
37
|
<<"lib/benchee/utility/parallel.ex">>,<<"lib/benchee/statistics">>,
|
|
39
38
|
<<"lib/benchee/statistics/mode.ex">>,
|
|
40
|
- <<"lib/benchee/statistics/percentile.ex">>,<<"lib/benchee/suite.ex">>,
|
|
39
|
+ <<"lib/benchee/statistics/percentile.ex">>,
|
|
40
|
+ <<"lib/benchee/relative_statistics.ex">>,<<"lib/benchee/suite.ex">>,
|
|
41
41
|
<<"lib/benchee/formatter.ex">>,<<"lib/benchee/benchmark.ex">>,
|
|
42
42
|
<<".formatter.exs">>,<<"mix.exs">>,<<"README.md">>,<<"LICENSE.md">>,
|
|
43
43
|
<<"CHANGELOG.md">>]}.
|
|
|
@@ -51,5 +51,5 @@
|
|
51
51
|
{<<"name">>,<<"deep_merge">>},
|
|
52
52
|
{<<"optional">>,false},
|
|
53
53
|
{<<"repository">>,<<"hexpm">>},
|
|
54
|
- {<<"requirement">>,<<"~> 0.1">>}]]}.
|
|
55
|
- {<<"version">>,<<"0.14.0">>}.
|
|
54
|
+ {<<"requirement">>,<<"~> 1.0">>}]]}.
|
|
55
|
+ {<<"version">>,<<"0.99.0">>}.
|
changed
lib/benchee.ex
|
|
@@ -3,7 +3,7 @@
|
|
3
3
|
# https://github.com/PragTob/benchee/commit/b3ddbc132e641cdf1eec0928b322ced1dab8553f#commitcomment-23381474
|
|
4
4
|
|
|
5
5
|
elixir_doc = """
|
|
6
|
- Top level module providing convenience access to needed functions as well
|
|
6
|
+ Top level module providing convenience access to needed functions as well
|
|
7
7
|
as the very high level `Benchee.run` API.
|
|
8
8
|
|
|
9
9
|
Intended Elixir interface.
|
|
|
@@ -44,6 +44,11 @@ for {module, moduledoc} <- [{Benchee, elixir_doc}, {:benchee, erlang_doc}] do
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
def run(config, jobs) when is_map(jobs) do
|
|
47
|
+ IO.puts("""
|
|
48
|
+ Passing configuration as the first argument to Benchee.run/2 is deprecated.
|
|
49
|
+ Please see the documentation for Benchee.run/2 for updated usage instructions.
|
|
50
|
+ """)
|
|
51
|
+
|
|
47
52
|
# pre 0.6.0 way of passing in the config first and as a map
|
|
48
53
|
do_run(jobs, config)
|
|
49
54
|
end
|
|
|
@@ -53,9 +58,10 @@ for {module, moduledoc} <- [{Benchee, elixir_doc}, {:benchee, erlang_doc}] do
|
|
53
58
|
|> Benchee.init()
|
|
54
59
|
|> Benchee.system()
|
|
55
60
|
|> add_benchmarking_jobs(jobs)
|
|
56
|
- |> Benchee.measure()
|
|
61
|
+ |> Benchee.collect()
|
|
57
62
|
|> Benchee.statistics()
|
|
58
63
|
|> Benchee.load()
|
|
64
|
+ |> Benchee.relative_statistics()
|
|
59
65
|
|> Formatter.output()
|
|
60
66
|
end
|
|
61
67
|
|
|
|
@@ -65,14 +71,58 @@ for {module, moduledoc} <- [{Benchee, elixir_doc}, {:benchee, erlang_doc}] do
|
|
65
71
|
end)
|
|
66
72
|
end
|
|
67
73
|
|
|
74
|
+ @doc false
|
|
75
|
+ def measure(suite) do
|
|
76
|
+ IO.puts("""
|
|
77
|
+ Benchee.measure/1 is deprecated, please use Benchee.collect/1.
|
|
78
|
+ """)
|
|
79
|
+
|
|
80
|
+ Benchee.Benchmark.collect(suite)
|
|
81
|
+ end
|
|
82
|
+
|
|
83
|
+ @doc """
|
|
84
|
+ See `Benchee.Configuration.init/1`
|
|
85
|
+ """
|
|
68
86
|
defdelegate init(), to: Benchee.Configuration
|
|
87
|
+
|
|
88
|
+ @doc """
|
|
89
|
+ See `Benchee.Configuration.init/1`
|
|
90
|
+ """
|
|
69
91
|
defdelegate init(config), to: Benchee.Configuration
|
|
92
|
+
|
|
93
|
+ @doc """
|
|
94
|
+ See `Benchee.System.system/1`
|
|
95
|
+ """
|
|
70
96
|
defdelegate system(suite), to: Benchee.System
|
|
97
|
+
|
|
98
|
+ @doc """
|
|
99
|
+ See `Benchee.Benchmark.benchmark/3`
|
|
100
|
+ """
|
|
71
101
|
defdelegate benchmark(suite, name, function), to: Benchee.Benchmark
|
|
102
|
+ @doc false
|
|
72
103
|
defdelegate benchmark(suite, name, function, printer), to: Benchee.Benchmark
|
|
73
|
- defdelegate measure(suite), to: Benchee.Benchmark
|
|
74
|
- defdelegate measure(suite, printer), to: Benchee.Benchmark
|
|
104
|
+
|
|
105
|
+ @doc """
|
|
106
|
+ See `Benchee.Benchmark.collect/1`
|
|
107
|
+ """
|
|
108
|
+ defdelegate collect(suite), to: Benchee.Benchmark
|
|
109
|
+
|
|
110
|
+ @doc false
|
|
111
|
+ defdelegate collect(suite, printer), to: Benchee.Benchmark
|
|
112
|
+
|
|
113
|
+ @doc """
|
|
114
|
+ See `Benchee.Statistics.statistics/1`
|
|
115
|
+ """
|
|
75
116
|
defdelegate statistics(suite), to: Benchee.Statistics
|
|
117
|
+
|
|
118
|
+ @doc """
|
|
119
|
+ See `Benchee.RelativeStatistics.relative_statistics/1`
|
|
120
|
+ """
|
|
121
|
+ defdelegate relative_statistics(suite), to: Benchee.RelativeStatistics
|
|
122
|
+
|
|
123
|
+ @doc """
|
|
124
|
+ See `Benchee.ScenarioLoader.load/1`
|
|
125
|
+ """
|
|
76
126
|
defdelegate load(suite), to: Benchee.ScenarioLoader
|
|
77
127
|
end
|
|
78
128
|
end
|
changed
lib/benchee/benchmark.ex
|
|
@@ -1,29 +1,29 @@
|
|
1
1
|
defmodule Benchee.Benchmark do
|
|
2
2
|
@moduledoc """
|
|
3
3
|
Functions related to building and running benchmarking scenarios.
|
|
4
|
- Exposes `benchmark/4` and `measure/3` functions.
|
|
4
|
+ Exposes `benchmark/4` and `collect/3` functions.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
- alias Benchee.Benchmark.{Runner, Scenario, ScenarioContext}
|
|
7
|
+ alias Benchee.Benchmark.{Runner, ScenarioContext}
|
|
8
8
|
alias Benchee.Output.BenchmarkPrinter, as: Printer
|
|
9
|
+ alias Benchee.Scenario
|
|
9
10
|
alias Benchee.Suite
|
|
10
11
|
alias Benchee.Utility.DeepConvert
|
|
11
12
|
|
|
12
|
- @type job_name :: String.t() | atom
|
|
13
13
|
@no_input :__no_input
|
|
14
14
|
|
|
15
15
|
@doc """
|
|
16
|
- Public access for the key representing no input for a scenario.
|
|
16
|
+ Public access for the special key representing no input for a scenario.
|
|
17
17
|
"""
|
|
18
18
|
def no_input, do: @no_input
|
|
19
19
|
|
|
20
20
|
@doc """
|
|
21
21
|
Takes the current suite and adds a new benchmarking scenario (represented by a
|
|
22
|
- %Scenario{} struct) to the suite's scenarios. If there are inputs in the
|
|
23
|
- suite's config, a scenario will be added for the given function for each
|
|
24
|
- input.
|
|
22
|
+ %Scenario{} struct) with the given name and function to the suite's scenarios.
|
|
23
|
+ If there are inputs in the suite's config, a scenario will be added for the given
|
|
24
|
+ function for each input.
|
|
25
25
|
"""
|
|
26
|
- @spec benchmark(Suite.t(), job_name, fun, module) :: Suite.t()
|
|
26
|
+ @spec benchmark(Suite.t(), Suite.key(), fun, module) :: Suite.t()
|
|
27
27
|
def benchmark(suite = %Suite{scenarios: scenarios}, job_name, function, printer \\ Printer) do
|
|
28
28
|
normalized_name = to_string(job_name)
|
|
29
29
|
|
|
|
@@ -99,13 +99,14 @@ defmodule Benchee.Benchmark do
|
|
99
99
|
end
|
|
100
100
|
|
|
101
101
|
@doc """
|
|
102
|
- Kicks off the benchmarking of all scenarios in the suite by passing the list
|
|
103
|
- of scenarios and a scenario context to our benchmark runner. For more
|
|
104
|
- information on how bencharmks are actually run, see
|
|
105
|
- `Benchee.Benchmark.Runner.run_scenarios/2`.
|
|
102
|
+ Kicks off the benchmarking of all scenarios defined in the given suite.
|
|
103
|
+
|
|
104
|
+ Hence, this might take a while ;) Passes a list of scenarios and a scenario context to our
|
|
105
|
+ benchmark runner. For more information on how benchmarks are actually run, see the
|
|
106
|
+ `Benchee.Benchmark.Runner` code (API considered private).
|
|
106
107
|
"""
|
|
107
|
- @spec measure(Suite.t(), module, module) :: Suite.t()
|
|
108
|
- def measure(
|
|
108
|
+ @spec collect(Suite.t(), module, module) :: Suite.t()
|
|
109
|
+ def collect(
|
|
109
110
|
suite = %Suite{scenarios: scenarios, configuration: config},
|
|
110
111
|
printer \\ Printer,
|
|
111
112
|
runner \\ Runner
|
added
lib/benchee/benchmark/collect.ex
|
|
@@ -0,0 +1,15 @@
|
|
1
|
+ defmodule Benchee.Benchmark.Collect do
|
|
2
|
+ @moduledoc false
|
|
3
|
+
|
|
4
|
+ # A thing that collects a data point about a function execution - like time
|
|
5
|
+ # or memory needed.
|
|
6
|
+
|
|
7
|
+ @doc """
|
|
8
|
+ Takes an anonymous 0 arity function to measure and returns the measurement
|
|
9
|
+ and the return value of the function in a tuple.
|
|
10
|
+
|
|
11
|
+ The returned measurement may be `nil` if the measurement failed for some
|
|
12
|
+ reason - it will then be ignored and not counted.
|
|
13
|
+ """
|
|
14
|
+ @callback collect((() -> any)) :: {non_neg_integer | nil, any}
|
|
15
|
+ end
|
added
lib/benchee/benchmark/collect/memory.ex
|
|
@@ -0,0 +1,149 @@
|
|
1
|
+ defmodule Benchee.Benchmark.Collect.Memory do
|
|
2
|
+ @moduledoc false
|
|
3
|
+
|
|
4
|
+ # Measure memory consumption of a function.
|
|
5
|
+ #
|
|
6
|
+ # This is somewhat tricky and hence some resources can be recommended reading alongside
|
|
7
|
+ # this code:
|
|
8
|
+ # * description of the approach: https://devonestes.herokuapp.com/using-erlang-trace-3
|
|
9
|
+ # * devon describing the journey that this feature put us through (includes remarks
|
|
10
|
+ # on why certain parts are very important: https://www.youtube.com/watch?v=aqLujfzvUgM)
|
|
11
|
+ # * erlang docs on the info data structure we use:
|
|
12
|
+ # https://erlang.org/doc/man/erlang.html#gc_minor_start
|
|
13
|
+ #
|
|
14
|
+ # Returns `{nil, return_value}` in case the memory measurement went bad.
|
|
15
|
+
|
|
16
|
+ @behaviour Benchee.Benchmark.Collect
|
|
17
|
+
|
|
18
|
+ def collect(fun) do
|
|
19
|
+ ref = make_ref()
|
|
20
|
+ Process.flag(:trap_exit, true)
|
|
21
|
+ start_runner(fun, ref)
|
|
22
|
+ await_results(nil, ref)
|
|
23
|
+ end
|
|
24
|
+
|
|
25
|
+ defp await_results(return_value, ref) do
|
|
26
|
+ receive do
|
|
27
|
+ {^ref, memory_usage} ->
|
|
28
|
+ return_memory({memory_usage, return_value})
|
|
29
|
+
|
|
30
|
+ {^ref, :shutdown} ->
|
|
31
|
+ nil
|
|
32
|
+
|
|
33
|
+ # we need a really basic pattern here because sending anything other than
|
|
34
|
+ # just what's returned from the function that we're benchmarking will
|
|
35
|
+ # involve allocating a new term, which will skew the measurements.
|
|
36
|
+ # We need to be very careful to always send the `ref` in every other
|
|
37
|
+ # message to this process.
|
|
38
|
+ new_result ->
|
|
39
|
+ await_results(new_result, ref)
|
|
40
|
+ end
|
|
41
|
+ end
|
|
42
|
+
|
|
43
|
+ defp start_runner(fun, ref) do
|
|
44
|
+ parent = self()
|
|
45
|
+
|
|
46
|
+ spawn_link(fn ->
|
|
47
|
+ tracer = start_tracer(self())
|
|
48
|
+
|
|
49
|
+ try do
|
|
50
|
+ _ = measure_memory(fun, tracer, parent)
|
|
51
|
+ word_size = :erlang.system_info(:wordsize)
|
|
52
|
+ memory_used = get_collected_memory(tracer)
|
|
53
|
+ send(parent, {ref, memory_used * word_size})
|
|
54
|
+ catch
|
|
55
|
+ kind, reason ->
|
|
56
|
+ # would love to have this in a separate function, but elixir 1.7 complains
|
|
57
|
+ send(tracer, :done)
|
|
58
|
+ send(parent, {ref, :shutdown})
|
|
59
|
+ stacktrace = System.stacktrace()
|
|
60
|
+ IO.puts(Exception.format(kind, reason, stacktrace))
|
|
61
|
+ exit(:normal)
|
|
62
|
+ after
|
|
63
|
+ send(tracer, :done)
|
|
64
|
+ end
|
|
65
|
+ end)
|
|
66
|
+ end
|
|
67
|
+
|
|
68
|
+ defp return_memory({memory_usage, return_value}) when memory_usage < 0, do: {nil, return_value}
|
|
69
|
+ defp return_memory(memory_usage_info), do: memory_usage_info
|
|
70
|
+
|
|
71
|
+ defp measure_memory(fun, tracer, parent) do
|
|
72
|
+ :erlang.garbage_collect()
|
|
73
|
+ send(tracer, :begin_collection)
|
|
74
|
+
|
|
75
|
+ receive do
|
|
76
|
+ :ready_to_begin -> nil
|
|
77
|
+ end
|
|
78
|
+
|
|
79
|
+ return_value = fun.()
|
|
80
|
+ send(parent, return_value)
|
|
81
|
+
|
|
82
|
+ :erlang.garbage_collect()
|
|
83
|
+ send(tracer, :end_collection)
|
|
84
|
+
|
|
85
|
+ receive do
|
|
86
|
+ :ready_to_end -> nil
|
|
87
|
+ end
|
|
88
|
+
|
|
89
|
+ # We need to reference these variables after we end our collection so
|
|
90
|
+ # these don't get GC'd and counted towards the memory usage of the function
|
|
91
|
+ # we're benchmarking.
|
|
92
|
+ {parent, fun}
|
|
93
|
+ end
|
|
94
|
+
|
|
95
|
+ defp get_collected_memory(tracer) do
|
|
96
|
+ ref = Process.monitor(tracer)
|
|
97
|
+ send(tracer, {:get_collected_memory, self(), ref})
|
|
98
|
+
|
|
99
|
+ receive do
|
|
100
|
+ {:DOWN, ^ref, _, _, _} -> nil
|
|
101
|
+ {^ref, collected} -> collected
|
|
102
|
+ end
|
|
103
|
+ end
|
|
104
|
+
|
|
105
|
+ defp start_tracer(pid) do
|
|
106
|
+ spawn(fn -> tracer_loop(pid, 0) end)
|
|
107
|
+ end
|
|
108
|
+
|
|
109
|
+ defp tracer_loop(pid, acc) do
|
|
110
|
+ receive do
|
|
111
|
+ :begin_collection ->
|
|
112
|
+ :erlang.trace(pid, true, [:garbage_collection, tracer: self()])
|
|
113
|
+ send(pid, :ready_to_begin)
|
|
114
|
+ tracer_loop(pid, acc)
|
|
115
|
+
|
|
116
|
+ :end_collection ->
|
|
117
|
+ :erlang.trace(pid, false, [:garbage_collection])
|
|
118
|
+ send(pid, :ready_to_end)
|
|
119
|
+ tracer_loop(pid, acc)
|
|
120
|
+
|
|
121
|
+ {:get_collected_memory, reply_to, ref} ->
|
|
122
|
+ send(reply_to, {ref, acc})
|
|
123
|
+
|
|
124
|
+ {:trace, ^pid, :gc_minor_start, info} ->
|
|
125
|
+ listen_gc_end(pid, :gc_minor_end, acc, total_memory(info))
|
|
126
|
+
|
|
127
|
+ {:trace, ^pid, :gc_major_start, info} ->
|
|
128
|
+ listen_gc_end(pid, :gc_major_end, acc, total_memory(info))
|
|
129
|
+
|
|
130
|
+ :done ->
|
|
131
|
+ exit(:normal)
|
|
132
|
+ end
|
|
133
|
+ end
|
|
134
|
+
|
|
135
|
+ defp listen_gc_end(pid, tag, acc, mem_before) do
|
|
136
|
+ receive do
|
|
137
|
+ {:trace, ^pid, ^tag, info} ->
|
|
138
|
+ mem_after = total_memory(info)
|
|
139
|
+ tracer_loop(pid, acc + mem_before - mem_after)
|
|
140
|
+ end
|
|
141
|
+ end
|
|
142
|
+
|
|
143
|
+ defp total_memory(info) do
|
|
144
|
+ # `:heap_size` seems to only contain the memory size of the youngest
|
|
145
|
+ # generation `:old_heap_size` has the old generation. There is also
|
|
146
|
+ # `:recent_size` but that seems to already be accounted for.
|
|
147
|
+ Keyword.fetch!(info, :heap_size) + Keyword.fetch!(info, :old_heap_size)
|
|
148
|
+ end
|
|
149
|
+ end
|
added
lib/benchee/benchmark/collect/native_time.ex
|
|
@@ -0,0 +1,18 @@
|
|
1
|
+ defmodule Benchee.Benchmark.Collect.NativeTime do
|
|
2
|
+ @moduledoc false
|
|
3
|
+
|
|
4
|
+ # Measure the time elapsed while executing a given function.
|
|
5
|
+ #
|
|
6
|
+ # Uses only the time unit native to the platform. Used for determining how many times a function
|
|
7
|
+ # should be repeated in `Benchee.Benchmark.Runner.determine_n_times/3` (private method though).
|
|
8
|
+
|
|
9
|
+ @behaviour Benchee.Benchmark.Collect
|
|
10
|
+
|
|
11
|
+ def collect(function) do
|
|
12
|
+ start = :erlang.monotonic_time()
|
|
13
|
+ result = function.()
|
|
14
|
+ finish = :erlang.monotonic_time()
|
|
15
|
+
|
|
16
|
+ {finish - start, result}
|
|
17
|
+ end
|
|
18
|
+ end
|
added
lib/benchee/benchmark/collect/time.ex
|
|
@@ -0,0 +1,23 @@
|
|
1
|
+ defmodule Benchee.Benchmark.Collect.Time do
|
|
2
|
+ @moduledoc false
|
|
3
|
+
|
|
4
|
+ # Measure the time elapsed while executing a given function.
|
|
5
|
+ #
|
|
6
|
+ # In contrast to `:timer.tc/1` it always returns the result in nano seconds instead of micro
|
|
7
|
+ # seconds. This helps us avoid losing precision as both Linux and MacOSX seem to be able to
|
|
8
|
+ # measure in nano seconds. `:timer.tc/n`
|
|
9
|
+ # [forfeits this precision](
|
|
10
|
+ # https://github.com/erlang/otp/blob/master/lib/stdlib/src/timer.erl#L164-L169).
|
|
11
|
+
|
|
12
|
+ @behaviour Benchee.Benchmark.Collect
|
|
13
|
+
|
|
14
|
+ def collect(function) do
|
|
15
|
+ start = :erlang.monotonic_time()
|
|
16
|
+ result = function.()
|
|
17
|
+ finish = :erlang.monotonic_time()
|
|
18
|
+
|
|
19
|
+ duration_nano_seconds = :erlang.convert_time_unit(finish - start, :native, :nanosecond)
|
|
20
|
+
|
|
21
|
+ {duration_nano_seconds, result}
|
|
22
|
+ end
|
|
23
|
+ end
|
changed
lib/benchee/benchmark/hooks.ex
|
|
@@ -2,7 +2,8 @@ defmodule Benchee.Benchmark.Hooks do
|
|
2
2
|
@moduledoc false
|
|
3
3
|
# Non benchee code should not rely on this module.
|
|
4
4
|
|
|
5
|
- alias Benchee.Benchmark.{Scenario, ScenarioContext}
|
|
5
|
+ alias Benchee.Benchmark.ScenarioContext
|
|
6
|
+ alias Benchee.Scenario
|
|
6
7
|
|
|
7
8
|
def run_before_scenario(
|
|
8
9
|
%Scenario{
|
removed
lib/benchee/benchmark/measure.ex
|
|
@@ -1,15 +0,0 @@
|
|
1
|
- defmodule Benchee.Benchmark.Measure do
|
|
2
|
- @moduledoc false
|
|
3
|
-
|
|
4
|
- # A thing that measures something about a function execution - like time or
|
|
5
|
- # memory needed.
|
|
6
|
-
|
|
7
|
- @doc """
|
|
8
|
- Takes an anonymous 0 arity function to measure and returns the measurement
|
|
9
|
- and the return value of the function in a tuple.
|
|
10
|
-
|
|
11
|
- The returned measurement may be `nil` if the measurement failed for some
|
|
12
|
- reason - it will then be ignored and not counted.
|
|
13
|
- """
|
|
14
|
- @callback measure((() -> any)) :: {non_neg_integer | nil, any}
|
|
15
|
- end
|
removed
lib/benchee/benchmark/measure/memory.ex
|
|
@@ -1,149 +0,0 @@
|
|
1
|
- defmodule Benchee.Benchmark.Measure.Memory do
|
|
2
|
- @moduledoc false
|
|
3
|
-
|
|
4
|
- # Measure memory consumption of a function.
|
|
5
|
- #
|
|
6
|
- # This is somewhat tricky and hence some resources can be recommended reading alongside
|
|
7
|
- # this code:
|
|
8
|
- # * description of the approach: https://devonestes.herokuapp.com/using-erlang-trace-3
|
|
9
|
- # * devon describing the journey that this feature put us through (includes remarks
|
|
10
|
- # on why certain parts are very important: https://www.youtube.com/watch?v=aqLujfzvUgM)
|
|
11
|
- # * erlang docs on the info data structure we use:
|
|
12
|
- # https://erlang.org/doc/man/erlang.html#gc_minor_start
|
|
13
|
- #
|
|
14
|
- # Returns `{nil, return_value}` in case the memory measurement went bad.
|
|
15
|
-
|
|
16
|
- @behaviour Benchee.Benchmark.Measure
|
|
17
|
-
|
|
18
|
- def measure(fun) do
|
|
19
|
- ref = make_ref()
|
|
20
|
- Process.flag(:trap_exit, true)
|
|
21
|
- start_runner(fun, ref)
|
|
22
|
- await_results(nil, ref)
|
|
23
|
- end
|
|
24
|
-
|
|
25
|
- defp await_results(return_value, ref) do
|
|
26
|
- receive do
|
|
27
|
- {^ref, memory_usage} ->
|
|
28
|
- return_memory({memory_usage, return_value})
|
|
29
|
-
|
|
30
|
- {^ref, :shutdown} ->
|
|
31
|
- nil
|
|
32
|
-
|
|
33
|
- # we need a really basic pattern here because sending anything other than
|
|
34
|
- # just what's returned from the function that we're benchmarking will
|
|
35
|
- # involve allocating a new term, which will skew the measurements.
|
|
36
|
- # We need to be very careful to always send the `ref` in every other
|
|
37
|
- # message to this process.
|
|
38
|
- new_result ->
|
|
39
|
- await_results(new_result, ref)
|
|
40
|
- end
|
|
41
|
- end
|
|
42
|
-
|
|
43
|
- defp start_runner(fun, ref) do
|
|
44
|
- parent = self()
|
|
45
|
-
|
|
46
|
- spawn_link(fn ->
|
|
47
|
- tracer = start_tracer(self())
|
|
48
|
-
|
|
49
|
- try do
|
|
50
|
- _ = measure_memory(fun, tracer, parent)
|
|
51
|
- word_size = :erlang.system_info(:wordsize)
|
|
52
|
- memory_used = get_collected_memory(tracer)
|
|
53
|
- send(parent, {ref, memory_used * word_size})
|
|
54
|
- catch
|
|
55
|
- kind, reason ->
|
|
56
|
- # would love to have this in a separate function, but elixir 1.7 complains
|
|
57
|
- send(tracer, :done)
|
|
58
|
- send(parent, {ref, :shutdown})
|
|
59
|
- stacktrace = System.stacktrace()
|
|
60
|
- IO.puts(Exception.format(kind, reason, stacktrace))
|
|
61
|
- exit(:normal)
|
|
62
|
- after
|
|
63
|
- send(tracer, :done)
|
|
64
|
- end
|
|
65
|
- end)
|
|
66
|
- end
|
|
67
|
-
|
|
68
|
- defp return_memory({memory_usage, return_value}) when memory_usage < 0, do: {nil, return_value}
|
|
69
|
- defp return_memory(memory_usage_info), do: memory_usage_info
|
|
70
|
-
|
|
71
|
- defp measure_memory(fun, tracer, parent) do
|
|
72
|
- :erlang.garbage_collect()
|
|
73
|
- send(tracer, :begin_collection)
|
|
74
|
-
|
|
75
|
- receive do
|
|
76
|
- :ready_to_begin -> nil
|
|
77
|
- end
|
|
78
|
-
|
|
79
|
- return_value = fun.()
|
|
80
|
- send(parent, return_value)
|
|
81
|
-
|
|
82
|
- :erlang.garbage_collect()
|
|
83
|
- send(tracer, :end_collection)
|
|
84
|
-
|
|
85
|
- receive do
|
|
86
|
- :ready_to_end -> nil
|
|
87
|
- end
|
|
88
|
-
|
|
89
|
- # We need to reference these variables after we end our collection so
|
|
90
|
- # these don't get GC'd and counted towards the memory usage of the function
|
|
91
|
- # we're benchmarking.
|
|
92
|
- {parent, fun}
|
|
93
|
- end
|
|
94
|
-
|
|
95
|
- defp get_collected_memory(tracer) do
|
|
96
|
- ref = Process.monitor(tracer)
|
|
97
|
- send(tracer, {:get_collected_memory, self(), ref})
|
|
98
|
-
|
|
99
|
- receive do
|
|
100
|
- {:DOWN, ^ref, _, _, _} -> nil
|
|
101
|
- {^ref, collected} -> collected
|
|
102
|
- end
|
|
103
|
- end
|
|
104
|
-
|
|
105
|
- defp start_tracer(pid) do
|
|
106
|
- spawn(fn -> tracer_loop(pid, 0) end)
|
|
107
|
- end
|
|
108
|
-
|
|
109
|
- defp tracer_loop(pid, acc) do
|
|
110
|
- receive do
|
|
111
|
- :begin_collection ->
|
|
112
|
- :erlang.trace(pid, true, [:garbage_collection, tracer: self()])
|
|
113
|
- send(pid, :ready_to_begin)
|
|
114
|
- tracer_loop(pid, acc)
|
|
115
|
-
|
|
116
|
- :end_collection ->
|
|
117
|
- :erlang.trace(pid, false, [:garbage_collection])
|
|
118
|
- send(pid, :ready_to_end)
|
|
119
|
- tracer_loop(pid, acc)
|
|
120
|
-
|
|
121
|
- {:get_collected_memory, reply_to, ref} ->
|
|
122
|
- send(reply_to, {ref, acc})
|
|
123
|
-
|
|
124
|
- {:trace, ^pid, :gc_minor_start, info} ->
|
|
125
|
- listen_gc_end(pid, :gc_minor_end, acc, total_memory(info))
|
|
126
|
-
|
|
127
|
- {:trace, ^pid, :gc_major_start, info} ->
|
|
128
|
- listen_gc_end(pid, :gc_major_end, acc, total_memory(info))
|
|
129
|
-
|
|
130
|
- :done ->
|
|
131
|
- exit(:normal)
|
|
132
|
- end
|
|
133
|
- end
|
|
134
|
-
|
|
135
|
- defp listen_gc_end(pid, tag, acc, mem_before) do
|
|
136
|
- receive do
|
|
137
|
- {:trace, ^pid, ^tag, info} ->
|
|
138
|
- mem_after = total_memory(info)
|
|
139
|
- tracer_loop(pid, acc + mem_before - mem_after)
|
|
140
|
- end
|
|
141
|
- end
|
|
142
|
-
|
|
143
|
- defp total_memory(info) do
|
|
144
|
- # `:heap_size` seems to only contain the memory size of the youngest
|
|
145
|
- # generation `:old_heap_size` has the old generation. There is also
|
|
146
|
- # `:recent_size` but that seems to already be accounted for.
|
|
147
|
- Keyword.fetch!(info, :heap_size) + Keyword.fetch!(info, :old_heap_size)
|
|
148
|
- end
|
|
149
|
- end
|
removed
lib/benchee/benchmark/measure/native_time.ex
|
|
@@ -1,18 +0,0 @@
|
|
1
|
- defmodule Benchee.Benchmark.Measure.NativeTime do
|
|
2
|
- @moduledoc false
|
|
3
|
-
|
|
4
|
- # Measure the time elapsed while executing a given function.
|
|
5
|
- #
|
|
6
|
- # Uses only the time unit native to the platform. Used for determining how many times a function
|
|
7
|
- # should be repeated in `Benchee.Benchmark.Runner.determine_n_times/3` (private method though).
|
|
8
|
-
|
|
9
|
- @behaviour Benchee.Benchmark.Measure
|
|
10
|
-
|
|
11
|
- def measure(function) do
|
|
12
|
- start = :erlang.monotonic_time()
|
|
13
|
- result = function.()
|
|
14
|
- finish = :erlang.monotonic_time()
|
|
15
|
-
|
|
16
|
- {finish - start, result}
|
|
17
|
- end
|
|
18
|
- end
|
removed
lib/benchee/benchmark/measure/time.ex
|
|
@@ -1,23 +0,0 @@
|
|
1
|
- defmodule Benchee.Benchmark.Measure.Time do
|
|
2
|
- @moduledoc false
|
|
3
|
-
|
|
4
|
- # Measure the time elapsed while executing a given function.
|
|
5
|
- #
|
|
6
|
- # In contrast to `:timer.tc/1` it always returns the result in nano seconds instead of micro
|
|
7
|
- # seconds. This helps us avoid losing precision as both Linux and MacOSX seem to be able to
|
|
8
|
- # measure in nano seconds. `:timer.tc/n`
|
|
9
|
- # [forfeits this precision](
|
|
10
|
- # https://github.com/erlang/otp/blob/master/lib/stdlib/src/timer.erl#L164-L169).
|
|
11
|
-
|
|
12
|
- @behaviour Benchee.Benchmark.Measure
|
|
13
|
-
|
|
14
|
- def measure(function) do
|
|
15
|
- start = :erlang.monotonic_time()
|
|
16
|
- result = function.()
|
|
17
|
- finish = :erlang.monotonic_time()
|
|
18
|
-
|
|
19
|
- duration_nano_seconds = :erlang.convert_time_unit(finish - start, :native, :nanosecond)
|
|
20
|
-
|
|
21
|
- {duration_nano_seconds, result}
|
|
22
|
- end
|
|
23
|
- end
|
changed
lib/benchee/benchmark/repeated_measurement.ex
|
|
@@ -21,7 +21,8 @@ defmodule Benchee.Benchmark.RepeatedMeasurement do
|
|
21
21
|
# with too high variance. Therefore determine an n how often it should be
|
|
22
22
|
# executed in the measurement cycle.
|
|
23
23
|
|
|
24
|
- alias Benchee.Benchmark.{Hooks, Measure, Runner, Scenario, ScenarioContext}
|
|
24
|
+ alias Benchee.Benchmark.{Collect, Hooks, Runner, ScenarioContext}
|
|
25
|
+ alias Benchee.Scenario
|
|
25
26
|
alias Benchee.Utility.RepeatN
|
|
26
27
|
|
|
27
28
|
@minimum_execution_time 10
|
|
|
@@ -33,12 +34,12 @@ defmodule Benchee.Benchmark.RepeatedMeasurement do
|
|
33
34
|
printer: printer
|
|
34
35
|
},
|
|
35
36
|
fast_warning,
|
|
36
|
- measurer \\ Measure.NativeTime
|
|
37
|
+ collector \\ Collect.NativeTime
|
|
37
38
|
) do
|
|
38
|
- run_time = measure_iteration(scenario, scenario_context, measurer)
|
|
39
|
+ run_time = measure_iteration(scenario, scenario_context, collector)
|
|
39
40
|
|
|
40
41
|
if run_time >= @minimum_execution_time do
|
|
41
|
- {num_iterations, adjust_for_iterations(run_time, num_iterations)}
|
|
42
|
+ {num_iterations, report_time(run_time, num_iterations)}
|
|
42
43
|
else
|
|
43
44
|
if fast_warning, do: printer.fast_warning()
|
|
44
45
|
|
|
|
@@ -47,21 +48,29 @@ defmodule Benchee.Benchmark.RepeatedMeasurement do
|
|
47
48
|
| num_iterations: num_iterations * @times_multiplier
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
- determine_n_times(scenario, new_context, false, measurer)
|
|
51
|
+ determine_n_times(scenario, new_context, false, collector)
|
|
51
52
|
end
|
|
52
53
|
end
|
|
53
54
|
|
|
55
|
+ # we need to convert the time here since we measure native time to see when we have enough
|
|
56
|
+ # repetitions but the first time is used in the actual samples
|
|
57
|
+ defp report_time(measurement, num_iterations) do
|
|
58
|
+ measurement
|
|
59
|
+ |> :erlang.convert_time_unit(:native, :nanosecond)
|
|
60
|
+ |> adjust_for_iterations(num_iterations)
|
|
61
|
+ end
|
|
62
|
+
|
|
54
63
|
defp adjust_for_iterations(measurement, 1), do: measurement
|
|
55
64
|
defp adjust_for_iterations(measurement, num_iterations), do: measurement / num_iterations
|
|
56
65
|
|
|
57
|
- def measure(
|
|
66
|
+ def collect(
|
|
58
67
|
scenario,
|
|
59
68
|
scenario_context = %ScenarioContext{
|
|
60
69
|
num_iterations: num_iterations
|
|
61
70
|
},
|
|
62
|
- measurer
|
|
71
|
+ collector
|
|
63
72
|
) do
|
|
64
|
- measurement = measure_iteration(scenario, scenario_context, measurer)
|
|
73
|
+ measurement = measure_iteration(scenario, scenario_context, collector)
|
|
65
74
|
|
|
66
75
|
adjust_for_iterations(measurement, num_iterations)
|
|
67
76
|
end
|
|
|
@@ -71,9 +80,9 @@ defmodule Benchee.Benchmark.RepeatedMeasurement do
|
|
71
80
|
scenario_context = %ScenarioContext{
|
|
72
81
|
num_iterations: 1
|
|
73
82
|
},
|
|
74
|
- measurer
|
|
83
|
+ collector
|
|
75
84
|
) do
|
|
76
|
- Runner.measure(scenario, scenario_context, measurer)
|
|
85
|
+ Runner.collect(scenario, scenario_context, collector)
|
|
77
86
|
end
|
|
78
87
|
|
|
79
88
|
defp measure_iteration(
|
|
|
@@ -81,7 +90,7 @@ defmodule Benchee.Benchmark.RepeatedMeasurement do
|
|
81
90
|
scenario_context = %ScenarioContext{
|
|
82
91
|
num_iterations: iterations
|
|
83
92
|
},
|
|
84
|
- measurer
|
|
93
|
+ collector
|
|
85
94
|
)
|
|
86
95
|
when iterations > 1 do
|
|
87
96
|
# When we have more than one iteration, then the repetition and calling
|
|
|
@@ -89,7 +98,7 @@ defmodule Benchee.Benchmark.RepeatedMeasurement do
|
|
89
98
|
# `build_benchmarking_function/2`
|
|
90
99
|
function = build_benchmarking_function(scenario, scenario_context)
|
|
91
100
|
|
|
92
|
- {measurement, _return_value} = measurer.measure(function)
|
|
101
|
+ {measurement, _return_value} = collector.collect(function)
|
|
93
102
|
|
|
94
103
|
measurement
|
|
95
104
|
end
|
changed
lib/benchee/benchmark/runner.ex
|
|
@@ -4,8 +4,8 @@ defmodule Benchee.Benchmark.Runner do
|
|
4
4
|
# This module actually runs our benchmark scenarios, adding information about
|
|
5
5
|
# run time and memory usage to each scenario.
|
|
6
6
|
|
|
7
|
- alias Benchee.{Benchmark, Configuration, Conversion, Statistics, Utility.Parallel}
|
|
8
|
- alias Benchmark.{Hooks, Measure, RepeatedMeasurement, Scenario, ScenarioContext}
|
|
7
|
+ alias Benchee.{Benchmark, Configuration, Conversion, Scenario, Statistics, Utility.Parallel}
|
|
8
|
+ alias Benchmark.{Collect, Hooks, RepeatedMeasurement, ScenarioContext}
|
|
9
9
|
|
|
10
10
|
@doc """
|
|
11
11
|
Executes the benchmarks defined before by first running the defined functions
|
|
|
@@ -47,7 +47,7 @@ defmodule Benchee.Benchmark.Runner do
|
|
47
47
|
defp pre_check(scenario, scenario_context) do
|
|
48
48
|
scenario_input = Hooks.run_before_scenario(scenario, scenario_context)
|
|
49
49
|
scenario_context = %ScenarioContext{scenario_context | scenario_input: scenario_input}
|
|
50
|
- _ = measure(scenario, scenario_context, Measure.Time)
|
|
50
|
+ _ = collect(scenario, scenario_context, Collect.Time)
|
|
51
51
|
_ = Hooks.run_after_scenario(scenario, scenario_context)
|
|
52
52
|
nil
|
|
53
53
|
end
|
|
|
@@ -94,7 +94,11 @@ defmodule Benchee.Benchmark.Runner do
|
|
94
94
|
run_times = Enum.flat_map(measurements, fn {run_times, _} -> run_times end)
|
|
95
95
|
memory_usages = Enum.flat_map(measurements, fn {_, memory_usages} -> memory_usages end)
|
|
96
96
|
|
|
97
|
- %Scenario{scenario | run_times: run_times, memory_usages: memory_usages}
|
|
97
|
+ %{
|
|
98
|
+ scenario
|
|
99
|
+ | run_time_data: %{scenario.run_time_data | samples: run_times},
|
|
100
|
+ memory_usage_data: %{scenario.memory_usage_data | samples: memory_usages}
|
|
101
|
+ }
|
|
98
102
|
end
|
|
99
103
|
|
|
100
104
|
defp measure_scenario(scenario, scenario_context) do
|
|
|
@@ -164,7 +168,7 @@ defmodule Benchee.Benchmark.Runner do
|
|
164
168
|
end_time: end_time
|
|
165
169
|
}
|
|
166
170
|
|
|
167
|
- do_benchmark(scenario, new_context, Measure.Memory, [])
|
|
171
|
+ do_benchmark(scenario, new_context, Collect.Memory, [])
|
|
168
172
|
end
|
|
169
173
|
|
|
170
174
|
defp measure_runtimes(scenario, context, run_time, fast_warning)
|
|
|
@@ -184,7 +188,7 @@ defmodule Benchee.Benchmark.Runner do
|
|
184
188
|
num_iterations: num_iterations
|
|
185
189
|
}
|
|
186
190
|
|
|
187
|
- do_benchmark(scenario, new_context, Measure.Time, [initial_run_time])
|
|
191
|
+ do_benchmark(scenario, new_context, Collect.Time, [initial_run_time])
|
|
188
192
|
end
|
|
189
193
|
|
|
190
194
|
defp current_time, do: :erlang.system_time(:nano_seconds)
|
|
|
@@ -200,7 +204,7 @@ defmodule Benchee.Benchmark.Runner do
|
|
200
204
|
current_time: current_time,
|
|
201
205
|
end_time: end_time
|
|
202
206
|
},
|
|
203
|
- _measurer,
|
|
207
|
+ _collector,
|
|
204
208
|
measurements
|
|
205
209
|
)
|
|
206
210
|
when current_time > end_time do
|
|
|
@@ -208,14 +212,14 @@ defmodule Benchee.Benchmark.Runner do
|
|
208
212
|
Enum.reverse(measurements)
|
|
209
213
|
end
|
|
210
214
|
|
|
211
|
- defp do_benchmark(scenario, scenario_context, measurer, measurements) do
|
|
212
|
- measurement = measure(scenario, scenario_context, measurer)
|
|
215
|
+ defp do_benchmark(scenario, scenario_context, collector, measurements) do
|
|
216
|
+ measurement = collect(scenario, scenario_context, collector)
|
|
213
217
|
updated_context = %ScenarioContext{scenario_context | current_time: current_time()}
|
|
214
218
|
|
|
215
219
|
do_benchmark(
|
|
216
220
|
scenario,
|
|
217
221
|
updated_context,
|
|
218
|
- measurer,
|
|
222
|
+ collector,
|
|
219
223
|
updated_measurements(measurement, measurements)
|
|
220
224
|
)
|
|
221
225
|
end
|
|
|
@@ -225,35 +229,35 @@ defmodule Benchee.Benchmark.Runner do
|
|
225
229
|
defp updated_measurements(measurement, measurements), do: [measurement | measurements]
|
|
226
230
|
|
|
227
231
|
@doc """
|
|
228
|
- Takes one measure with the given measurer.
|
|
232
|
+ Takes one measure with the given collector.
|
|
229
233
|
|
|
230
234
|
Correctly dispatches based on the number of iterations to perform.
|
|
231
235
|
"""
|
|
232
|
- def measure(
|
|
236
|
+ def collect(
|
|
233
237
|
scenario = %Scenario{function: function},
|
|
234
238
|
scenario_context = %ScenarioContext{
|
|
235
239
|
num_iterations: 1
|
|
236
240
|
},
|
|
237
|
- measurer
|
|
241
|
+ collector
|
|
238
242
|
) do
|
|
239
243
|
new_input = Hooks.run_before_each(scenario, scenario_context)
|
|
240
244
|
function = main_function(function, new_input)
|
|
241
245
|
|
|
242
|
- {measurement, return_value} = measurer.measure(function)
|
|
246
|
+ {measurement, return_value} = collector.collect(function)
|
|
243
247
|
|
|
244
248
|
Hooks.run_after_each(return_value, scenario, scenario_context)
|
|
245
249
|
measurement
|
|
246
250
|
end
|
|
247
251
|
|
|
248
|
- def measure(
|
|
252
|
+ def collect(
|
|
249
253
|
scenario,
|
|
250
254
|
scenario_context = %ScenarioContext{
|
|
251
255
|
num_iterations: iterations
|
|
252
256
|
},
|
|
253
|
- measurer
|
|
257
|
+ collector
|
|
254
258
|
)
|
|
255
259
|
when iterations > 1 do
|
|
256
|
- RepeatedMeasurement.measure(scenario, scenario_context, measurer)
|
|
260
|
+ RepeatedMeasurement.collect(scenario, scenario_context, collector)
|
|
257
261
|
end
|
|
258
262
|
|
|
259
263
|
def main_function(function, @no_input), do: function
|
removed
lib/benchee/benchmark/scenario.ex
|
|
@@ -1,102 +0,0 @@
|
|
1
|
- defmodule Benchee.Benchmark.Scenario do
|
|
2
|
- @moduledoc """
|
|
3
|
- A Scenario in Benchee is a particular case of a whole benchmarking suite. That
|
|
4
|
- is the combination of a particular function to benchmark (`job_name` and
|
|
5
|
- `function`) in combination with a specific input (`input_name` and `input`).
|
|
6
|
-
|
|
7
|
- It then gathers all data measured for this particular combination during
|
|
8
|
- `Benchee.Benchmark.measure/3` (`run_times` and `memory_usages`),
|
|
9
|
- which are then used later in the process by `Benchee.Statistics` to compute
|
|
10
|
- the relevant statistics (`run_time_statistics` and `memory_usage_statistics`).
|
|
11
|
-
|
|
12
|
- `name` is the name that should be used by formatters to display scenarios as
|
|
13
|
- it potentially includes the `tag` present when loading scenarios that were
|
|
14
|
- saved before. See `display_name/1`.
|
|
15
|
- """
|
|
16
|
- defstruct [
|
|
17
|
- :name,
|
|
18
|
- :job_name,
|
|
19
|
- :function,
|
|
20
|
- :input_name,
|
|
21
|
- :input,
|
|
22
|
- :run_time_statistics,
|
|
23
|
- :memory_usage_statistics,
|
|
24
|
- run_times: [],
|
|
25
|
- memory_usages: [],
|
|
26
|
- before_each: nil,
|
|
27
|
- after_each: nil,
|
|
28
|
- before_scenario: nil,
|
|
29
|
- after_scenario: nil,
|
|
30
|
- tag: nil
|
|
31
|
- ]
|
|
32
|
-
|
|
33
|
- @type t :: %__MODULE__{
|
|
34
|
- name: String.t(),
|
|
35
|
- job_name: String.t(),
|
|
36
|
- function: fun,
|
|
37
|
- input_name: String.t() | nil,
|
|
38
|
- input: any | nil,
|
|
39
|
- run_times: [float],
|
|
40
|
- run_time_statistics: Benchee.Statistics.t() | nil,
|
|
41
|
- memory_usages: [non_neg_integer],
|
|
42
|
- memory_usage_statistics: Benchee.Statistics.t() | nil,
|
|
43
|
- before_each: fun | nil,
|
|
44
|
- after_each: fun | nil,
|
|
45
|
- before_scenario: fun | nil,
|
|
46
|
- after_scenario: fun | nil,
|
|
47
|
- tag: String.t() | nil
|
|
48
|
- }
|
|
49
|
-
|
|
50
|
- @doc """
|
|
51
|
- Returns the correct name to display of the given scenario data.
|
|
52
|
-
|
|
53
|
- In the normal case this is `job_name`, however when scenarios are loaded they
|
|
54
|
- are tagged and these tags should be shown for disambiguation.
|
|
55
|
-
|
|
56
|
- ## Examples
|
|
57
|
-
|
|
58
|
- iex> alias Benchee.Benchmark.Scenario
|
|
59
|
- iex> Scenario.display_name(%Scenario{job_name: "flat_map"})
|
|
60
|
- "flat_map"
|
|
61
|
- iex> Scenario.display_name(%Scenario{job_name: "flat_map", tag: "master"})
|
|
62
|
- "flat_map (master)"
|
|
63
|
- iex> Scenario.display_name(%{job_name: "flat_map"})
|
|
64
|
- "flat_map"
|
|
65
|
- """
|
|
66
|
- @spec display_name(t) :: String.t()
|
|
67
|
- def display_name(%{job_name: job_name, tag: nil}), do: job_name
|
|
68
|
- def display_name(%{job_name: job_name, tag: tag}), do: "#{job_name} (#{tag})"
|
|
69
|
- def display_name(%{job_name: job_name}), do: job_name
|
|
70
|
-
|
|
71
|
- @doc """
|
|
72
|
- Returns `true` if data of the provided type has been fully procsessed, `false` otherwise.
|
|
73
|
-
|
|
74
|
- Current available types are `run_time` and `memory`. Reasons they might not have been processed
|
|
75
|
- yet are:
|
|
76
|
- * Suite wasn't configured to collect them at all
|
|
77
|
- * `Benchee.statistics/1` hasn't been called yet so that data was collected but statistics
|
|
78
|
- aren't present yet
|
|
79
|
-
|
|
80
|
- ## Examples
|
|
81
|
-
|
|
82
|
- iex> alias Benchee.Benchmark.Scenario
|
|
83
|
- iex> alias Benchee.Statistics
|
|
84
|
- iex> scenario = %Scenario{run_time_statistics: %Statistics{sample_size: 100}}
|
|
85
|
- iex> Scenario.data_processed?(scenario, :run_time)
|
|
86
|
- true
|
|
87
|
- iex> scenario = %Scenario{memory_usage_statistics: %Statistics{sample_size: 1}}
|
|
88
|
- iex> Scenario.data_processed?(scenario, :memory)
|
|
89
|
- true
|
|
90
|
- iex> scenario = %Scenario{memory_usage_statistics: %Statistics{sample_size: 0}}
|
|
91
|
- iex> Scenario.data_processed?(scenario, :memory)
|
|
92
|
- false
|
|
93
|
- """
|
|
94
|
- @spec data_processed?(t, :run_time | :memory) :: boolean
|
|
95
|
- def data_processed?(scenario, :run_time) do
|
|
96
|
- scenario.run_time_statistics.sample_size > 0
|
|
97
|
- end
|
|
98
|
-
|
|
99
|
- def data_processed?(scenario, :memory) do
|
|
100
|
- scenario.memory_usage_statistics.sample_size > 0
|
|
101
|
- end
|
|
102
|
- end
|
added
lib/benchee/collection_data.ex
|
|
@@ -0,0 +1,19 @@
|
|
1
|
+ defmodule Benchee.CollectionData do
|
|
2
|
+ @moduledoc """
|
|
3
|
+ The unified data structure for a given collection of data.
|
|
4
|
+
|
|
5
|
+ Consists of the recorded `samples` and the statistics computed from them.
|
|
6
|
+ """
|
|
7
|
+
|
|
8
|
+ defstruct statistics: %Benchee.Statistics{}, samples: []
|
|
9
|
+
|
|
10
|
+ @typedoc """
|
|
11
|
+ Samples and statistics.
|
|
12
|
+
|
|
13
|
+ Statistics might only come later when they are computed.
|
|
14
|
+ """
|
|
15
|
+ @type t :: %__MODULE__{
|
|
16
|
+ samples: [float | non_neg_integer],
|
|
17
|
+ statistics: Benchee.Statistics.t() | nil
|
|
18
|
+ }
|
|
19
|
+ end
|
changed
lib/benchee/configuration.ex
|
|
@@ -31,12 +31,7 @@ defmodule Benchee.Configuration do
|
|
31
31
|
load: false,
|
|
32
32
|
# formatters should end up here but known once are still picked up at
|
|
33
33
|
# the top level for now
|
|
34
|
- formatter_options: %{
|
|
35
|
- console: %{
|
|
36
|
- comparison: true,
|
|
37
|
- extended_statistics: false
|
|
38
|
- }
|
|
39
|
- },
|
|
34
|
+ formatter_options: %{},
|
|
40
35
|
unit_scaling: :best,
|
|
41
36
|
# If you/your plugin/whatever needs it your data can go here
|
|
42
37
|
assigns: %{},
|
|
|
@@ -47,37 +42,10 @@ defmodule Benchee.Configuration do
|
|
47
42
|
measure_function_call_overhead: true,
|
|
48
43
|
title: nil
|
|
49
44
|
|
|
50
|
- @type t :: %__MODULE__{
|
|
51
|
- parallel: integer,
|
|
52
|
- time: number,
|
|
53
|
- warmup: number,
|
|
54
|
- memory_time: number,
|
|
55
|
- pre_check: boolean,
|
|
56
|
- formatters: [(Suite.t() -> Suite.t())],
|
|
57
|
- print: map,
|
|
58
|
- inputs: %{Suite.key() => any} | [{Suite.key(), any}] | nil,
|
|
59
|
- save: map | false,
|
|
60
|
- load: String.t() | [String.t()] | false,
|
|
61
|
- formatter_options: map,
|
|
62
|
- unit_scaling: Scale.scaling_strategy(),
|
|
63
|
- assigns: map,
|
|
64
|
- before_each: fun | nil,
|
|
65
|
- after_each: fun | nil,
|
|
66
|
- before_scenario: fun | nil,
|
|
67
|
- after_scenario: fun | nil,
|
|
68
|
- measure_function_call_overhead: boolean,
|
|
69
|
- title: String.t() | nil
|
|
70
|
- }
|
|
45
|
+ @typedoc """
|
|
46
|
+ The configuration supplied by the user as either a map or a keyword list
|
|
71
47
|
|
|
72
|
- @type user_configuration :: map | keyword
|
|
73
|
- @time_keys [:time, :warmup, :memory_time]
|
|
74
|
-
|
|
75
|
- @doc """
|
|
76
|
- Returns the initial benchmark configuration for Benchee, composed of defaults
|
|
77
|
- and an optional custom configuration.
|
|
78
|
-
|
|
79
|
- Configuration times are given in seconds, but are converted to microseconds
|
|
80
|
- internally.
|
|
48
|
+ Possible options are:
|
|
81
49
|
|
|
82
50
|
Possible options:
|
|
83
51
|
|
|
|
@@ -157,6 +125,47 @@ defmodule Benchee.Configuration do
|
|
157
125
|
equivalent to the behaviour Benchee had pre 0.5.0)
|
|
158
126
|
* `:before_scenario`/`after_scenario`/`before_each`/`after_each` - read up on them in the hooks section in the README
|
|
159
127
|
* `:measure_function_call_overhead` - Measure how long an empty function call takes and deduct this from each measure run time. Defaults to true.
|
|
128
|
+ """
|
|
129
|
+ @type user_configuration :: map | keyword
|
|
130
|
+
|
|
131
|
+ @typedoc """
|
|
132
|
+ Generated configuration struct from the user supplied configuration options.
|
|
133
|
+
|
|
134
|
+ Filled in with a lot of defaults. Also notably every option is already converted to
|
|
135
|
+ a map or struct at this point for easier handling in benchee.
|
|
136
|
+ """
|
|
137
|
+ @type t :: %__MODULE__{
|
|
138
|
+ parallel: integer,
|
|
139
|
+ time: number,
|
|
140
|
+ warmup: number,
|
|
141
|
+ memory_time: number,
|
|
142
|
+ pre_check: boolean,
|
|
143
|
+ formatters: [(Suite.t() -> Suite.t()) | module | {module, map}],
|
|
144
|
+ print: map,
|
|
145
|
+ inputs: %{Suite.key() => any} | [{Suite.key(), any}] | nil,
|
|
146
|
+ save: map | false,
|
|
147
|
+ load: String.t() | [String.t()] | false,
|
|
148
|
+ formatter_options: map,
|
|
149
|
+ unit_scaling: Scale.scaling_strategy(),
|
|
150
|
+ assigns: map,
|
|
151
|
+ before_each: fun | nil,
|
|
152
|
+ after_each: fun | nil,
|
|
153
|
+ before_scenario: fun | nil,
|
|
154
|
+ after_scenario: fun | nil,
|
|
155
|
+ measure_function_call_overhead: boolean,
|
|
156
|
+ title: String.t() | nil
|
|
157
|
+ }
|
|
158
|
+
|
|
159
|
+ @time_keys [:time, :warmup, :memory_time]
|
|
160
|
+
|
|
161
|
+ @doc """
|
|
162
|
+ Returns the initial benchmark configuration for Benchee, composed of defaults
|
|
163
|
+ and an optional custom configuration.
|
|
164
|
+
|
|
165
|
+ Configuration times are given in seconds, but are converted to microseconds
|
|
166
|
+ internally.
|
|
167
|
+
|
|
168
|
+ For a list of all possible options see `t:user_configuration/0`
|
|
160
169
|
|
|
161
170
|
## Examples
|
|
162
171
|
|
|
|
@@ -170,12 +179,7 @@ defmodule Benchee.Configuration do
|
|
170
179
|
inputs: nil,
|
|
171
180
|
save: false,
|
|
172
181
|
load: false,
|
|
173
|
- formatters: [
|
|
174
|
- {
|
|
175
|
- Benchee.Formatters.Console,
|
|
176
|
- %{comparison: true, extended_statistics: false}
|
|
177
|
- }
|
|
178
|
- ],
|
|
182
|
+ formatters: [Benchee.Formatters.Console],
|
|
179
183
|
print: %{
|
|
180
184
|
benchmarking: true,
|
|
181
185
|
fast_warning: true,
|
|
|
@@ -203,12 +207,7 @@ defmodule Benchee.Configuration do
|
|
203
207
|
inputs: nil,
|
|
204
208
|
save: false,
|
|
205
209
|
load: false,
|
|
206
|
- formatters: [
|
|
207
|
- {
|
|
208
|
- Benchee.Formatters.Console,
|
|
209
|
- %{comparison: true, extended_statistics: false}
|
|
210
|
- }
|
|
211
|
- ],
|
|
210
|
+ formatters: [Benchee.Formatters.Console],
|
|
212
211
|
print: %{
|
|
213
212
|
benchmarking: true,
|
|
214
213
|
fast_warning: true,
|
|
|
@@ -236,12 +235,7 @@ defmodule Benchee.Configuration do
|
|
236
235
|
inputs: nil,
|
|
237
236
|
save: false,
|
|
238
237
|
load: false,
|
|
239
|
- formatters: [
|
|
240
|
- {
|
|
241
|
- Benchee.Formatters.Console,
|
|
242
|
- %{comparison: true, extended_statistics: false}
|
|
243
|
- }
|
|
244
|
- ],
|
|
238
|
+ formatters: [Benchee.Formatters.Console],
|
|
245
239
|
print: %{
|
|
246
240
|
benchmarking: true,
|
|
247
241
|
fast_warning: true,
|
|
|
@@ -265,9 +259,7 @@ defmodule Benchee.Configuration do
|
|
265
259
|
...> warmup: 0.2,
|
|
266
260
|
...> formatters: [&IO.puts/1],
|
|
267
261
|
...> print: [fast_warning: false],
|
|
268
|
- ...> console: [comparison: false],
|
|
269
262
|
...> inputs: %{"Small" => 5, "Big" => 9999},
|
|
270
|
- ...> formatter_options: [some: "option"],
|
|
271
263
|
...> unit_scaling: :smallest)
|
|
272
264
|
%Benchee.Suite{
|
|
273
265
|
configuration:
|
|
|
@@ -284,13 +276,7 @@ defmodule Benchee.Configuration do
|
|
284
276
|
fast_warning: false,
|
|
285
277
|
configuration: true
|
|
286
278
|
},
|
|
287
|
- formatter_options: %{
|
|
288
|
- console: %{
|
|
289
|
- comparison: false,
|
|
290
|
- extended_statistics: false
|
|
291
|
- },
|
|
292
|
- some: "option"
|
|
293
|
- },
|
|
279
|
+ formatter_options: %{},
|
|
294
280
|
percentiles: [50, 99],
|
|
295
281
|
unit_scaling: :smallest,
|
|
296
282
|
assigns: %{},
|
|
|
@@ -347,6 +333,14 @@ defmodule Benchee.Configuration do
|
|
347
333
|
|
|
348
334
|
defp formatter_configuration_from_options(config, module, legacy_option_key) do
|
|
349
335
|
if Map.has_key?(config.formatter_options, legacy_option_key) do
|
|
336
|
+ IO.puts("""
|
|
337
|
+
|
|
338
|
+ Using :formatter_options to configure formatters is now deprecated.
|
|
339
|
+ Please configure them in `:formatters` - see the documentation for
|
|
340
|
+ Benchee.Configuration.init/1 for details.
|
|
341
|
+
|
|
342
|
+ """)
|
|
343
|
+
|
|
350
344
|
{module, config.formatter_options[legacy_option_key]}
|
|
351
345
|
else
|
|
352
346
|
module
|
|
|
@@ -405,7 +399,9 @@ defmodule Benchee.Configuration do
|
|
405
399
|
|
|
406
400
|
Measuring memory consumption is only available on OTP 19 or greater.
|
|
407
401
|
If you would like to benchmark memory consumption, please upgrade to a
|
|
408
|
- newer verion of OTP.
|
|
402
|
+ newer version of OTP.
|
|
403
|
+ Benchee now also only supports elixir ~> 1.6 which doesn't support OTP 18.
|
|
404
|
+ Please upgrade :)
|
|
409
405
|
|
|
410
406
|
""")
|
|
411
407
|
end
|
changed
lib/benchee/conversion.ex
|
|
@@ -5,7 +5,7 @@ defmodule Benchee.Conversion do
|
|
5
5
|
Can be used by plugins to use benchee unit scaling logic.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
- alias Benchee.Benchmark.Scenario
|
|
8
|
+ alias Benchee.Scenario
|
|
9
9
|
alias Benchee.Conversion.{Count, Duration, Memory}
|
|
10
10
|
|
|
11
11
|
@doc """
|
|
|
@@ -19,9 +19,9 @@ defmodule Benchee.Conversion do
|
|
19
19
|
## Examples
|
|
20
20
|
|
|
21
21
|
iex> statistics = %Benchee.Statistics{average: 1_000_000.0, ips: 1000.0}
|
|
22
|
- iex> scenario = %Benchee.Benchmark.Scenario{
|
|
23
|
- ...> run_time_statistics: statistics,
|
|
24
|
- ...> memory_usage_statistics: statistics
|
|
22
|
+ iex> scenario = %Benchee.Scenario{
|
|
23
|
+ ...> run_time_data: %Benchee.CollectionData{statistics: statistics},
|
|
24
|
+ ...> memory_usage_data: %Benchee.CollectionData{statistics: statistics}
|
|
25
25
|
...> }
|
|
26
26
|
iex> Benchee.Conversion.units([scenario], :best)
|
|
27
27
|
%{
|
|
|
@@ -48,16 +48,16 @@ defmodule Benchee.Conversion do
|
|
48
48
|
def units(scenarios, scaling_strategy) do
|
|
49
49
|
run_time_measurements =
|
|
50
50
|
scenarios
|
|
51
|
- |> Enum.flat_map(fn scenario -> Map.to_list(scenario.run_time_statistics) end)
|
|
51
|
+ |> Enum.flat_map(fn scenario -> Map.to_list(scenario.run_time_data.statistics) end)
|
|
52
52
|
|> Enum.group_by(fn {stat_name, _} -> stat_name end, fn {_, value} -> value end)
|
|
53
53
|
|
|
54
54
|
memory_measurements =
|
|
55
55
|
scenarios
|
|
56
56
|
|> Enum.flat_map(fn
|
|
57
|
- %Scenario{memory_usage_statistics: nil} ->
|
|
57
|
+ %Scenario{memory_usage_data: %{statistics: nil}} ->
|
|
58
58
|
[]
|
|
59
59
|
|
|
60
|
- %Scenario{memory_usage_statistics: memory_usage_statistics} ->
|
|
60
|
+ %Scenario{memory_usage_data: %{statistics: memory_usage_statistics}} ->
|
|
61
61
|
Map.to_list(memory_usage_statistics)
|
|
62
62
|
end)
|
|
63
63
|
|> Enum.group_by(fn {stat_name, _} -> stat_name end, fn {_, value} -> value end)
|
changed
lib/benchee/conversion/count.ex
|
|
@@ -1,6 +1,8 @@
|
|
1
1
|
defmodule Benchee.Conversion.Count do
|
|
2
2
|
@moduledoc """
|
|
3
3
|
Unit scaling for counts, such that 1000000 can be converted to 1 Million.
|
|
4
|
+
|
|
5
|
+ Only benchee plugins should use this code.
|
|
4
6
|
"""
|
|
5
7
|
|
|
6
8
|
alias Benchee.Conversion.{Format, Scale, Unit}
|
changed
lib/benchee/conversion/deviation_percent.ex
|
|
@@ -2,6 +2,8 @@ defmodule Benchee.Conversion.DeviationPercent do
|
|
2
2
|
@moduledoc """
|
|
3
3
|
Helps with formattiong for the standard deviation ratio converting it into the
|
|
4
4
|
more common percent form.
|
|
5
|
+
|
|
6
|
+ Only benchee plugins should use this code.
|
|
5
7
|
"""
|
|
6
8
|
|
|
7
9
|
alias Benchee.Conversion.Format
|
changed
lib/benchee/conversion/duration.ex
|
|
@@ -1,6 +1,8 @@
|
|
1
1
|
defmodule Benchee.Conversion.Duration do
|
|
2
2
|
@moduledoc """
|
|
3
3
|
Unit scaling for duration converting from microseconds to minutes and others.
|
|
4
|
+
|
|
5
|
+ Only benchee plugins should use this code.
|
|
4
6
|
"""
|
|
5
7
|
|
|
6
8
|
alias Benchee.Conversion.{Format, Scale, Unit}
|
changed
lib/benchee/conversion/format.ex
|
|
@@ -3,7 +3,7 @@ defmodule Benchee.Conversion.Format do
|
|
3
3
|
Functions for formatting values and their unit labels. Different domains
|
|
4
4
|
handle this task differently, for example durations and counts.
|
|
5
5
|
|
|
6
|
- See `Benchee.Conversion.Count` and `Benchee.Conversion.Duration` for examples
|
|
6
|
+ See `Benchee.Conversion.Count` and `Benchee.Conversion.Duration` for examples.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
alias Benchee.Conversion.Unit
|
changed
lib/benchee/conversion/memory.ex
|
|
@@ -1,6 +1,8 @@
|
|
1
1
|
defmodule Benchee.Conversion.Memory do
|
|
2
2
|
@moduledoc """
|
|
3
3
|
Unit scaling for memory converting from bytes to kilobytes and others.
|
|
4
|
+
|
|
5
|
+ Only benchee plugins should use this code.
|
|
4
6
|
"""
|
|
5
7
|
|
|
6
8
|
alias Benchee.Conversion.{Format, Scale, Unit}
|
changed
lib/benchee/conversion/unit.ex
|
|
@@ -1,7 +1,10 @@
|
|
1
1
|
defmodule Benchee.Conversion.Unit do
|
|
2
2
|
@moduledoc """
|
|
3
|
- A representation of the different units used in `Benchee.Conversion.Duration`
|
|
4
|
- and `Benchee.Conversion.Count`. A unit is characterized by:
|
|
3
|
+ A representation of the different units used in `Benchee.Conversion.Format`
|
|
4
|
+ and `Benchee.Conversion.Scale` as well as the modules implementing these
|
|
5
|
+ behaviours.
|
|
6
|
+
|
|
7
|
+ A unit is characterized by:
|
|
5
8
|
|
|
6
9
|
* name - an atom representation of the unit for easy access (`:microseconds`,
|
|
7
10
|
`thousand`)
|
changed
lib/benchee/formatter.ex
|
|
@@ -1,7 +1,6 @@
|
|
1
1
|
defmodule Benchee.Formatter do
|
|
2
2
|
@moduledoc """
|
|
3
|
- Defines a behaviour for formatters in Benchee, and also defines functions to
|
|
4
|
- handle invoking that defined behavior.
|
|
3
|
+ Defines a behaviour for formatters in benchee, and functions to work with these.
|
|
5
4
|
|
|
6
5
|
When implementing a benchee formatter as a behaviour please adopt this
|
|
7
6
|
behaviour, as it helps with uniformity and also allows at least the `.format`
|
|
|
@@ -15,6 +14,9 @@ defmodule Benchee.Formatter do
|
|
15
14
|
alias Benchee.{Suite, Utility.Parallel}
|
|
16
15
|
alias Benchee.Utility.DeepConvert
|
|
17
16
|
|
|
17
|
+ @typedoc """
|
|
18
|
+ Options given to formatters, entirely defined by formatter authors.
|
|
19
|
+ """
|
|
18
20
|
@type options :: any
|
|
19
21
|
|
|
20
22
|
@doc """
|
|
|
@@ -37,7 +39,7 @@ defmodule Benchee.Formatter do
|
|
37
39
|
@doc """
|
|
38
40
|
Format and output all configured formatters and formatting functions.
|
|
39
41
|
|
|
40
|
- Expects a suite that already has been run through all previous functions so has the aggregated
|
|
42
|
+ Expects a suite that already has been run through all previous steps so has the aggregated
|
|
41
43
|
statistics etc. that the formatters rely on.
|
|
42
44
|
|
|
43
45
|
Works by invoking the `format/2` and `write/2` functions defined in this module. The `format/2`
|
|
|
@@ -46,8 +48,7 @@ defmodule Benchee.Formatter do
|
|
46
48
|
|
|
47
49
|
Also handles pure functions that will then be called with the suite.
|
|
48
50
|
|
|
49
|
- You can't rely on the formatters being called in pre determined order. Right now first those
|
|
50
|
- that are parallelizable (benchee formatter modules) are called, then normal functions.
|
|
51
|
+ You can't rely on the formatters being called in pre determined order.
|
|
51
52
|
"""
|
|
52
53
|
@spec output(Suite.t()) :: Suite.t()
|
|
53
54
|
def output(suite = %{configuration: %{formatters: formatters}}) do
|
|
|
@@ -73,7 +74,20 @@ defmodule Benchee.Formatter do
|
|
73
74
|
{formatter, @default_opts}
|
|
74
75
|
end
|
|
75
76
|
|
|
76
|
- defp normalize_module_configuration(formatter), do: formatter
|
|
77
|
+ defp normalize_module_configuration(formatter) when is_function(formatter, 1), do: formatter
|
|
78
|
+
|
|
79
|
+ defp normalize_module_configuration(formatter) do
|
|
80
|
+ IO.puts("""
|
|
81
|
+
|
|
82
|
+ All formaters that are not modules implementing the Benchee.Formatter
|
|
83
|
+ behaviour or functions with an arity of 1 are deprecated.
|
|
84
|
+ Please see the documentation for Benchee.Configuration.init/1 for
|
|
85
|
+ current usage instructions.
|
|
86
|
+
|
|
87
|
+ """)
|
|
88
|
+
|
|
89
|
+ formatter
|
|
90
|
+ end
|
|
77
91
|
|
|
78
92
|
defp is_formatter_module?({formatter, _options}) when is_atom(formatter) do
|
|
79
93
|
module_attributes = formatter.module_info(:attributes)
|
|
|
@@ -93,7 +107,7 @@ defmodule Benchee.Formatter do
|
|
93
107
|
to the documentation of the formatters you use.
|
|
94
108
|
"""
|
|
95
109
|
@spec output(Suite.t(), module, options) :: Suite.t()
|
|
96
|
- def output(suite, formatter, options) do
|
|
110
|
+ def output(suite, formatter, options \\ %{}) do
|
|
97
111
|
:ok =
|
|
98
112
|
suite
|
|
99
113
|
|> formatter.format(options)
|
changed
lib/benchee/formatters/console.ex
|
|
@@ -1,7 +1,24 @@
|
|
1
1
|
defmodule Benchee.Formatters.Console do
|
|
2
2
|
@moduledoc """
|
|
3
|
- Formatter to transform the statistics output into a structure suitable for
|
|
4
|
- output through `IO.write` on the console.
|
|
3
|
+ Formatter to print out the results of benchmarking suite to the console.
|
|
4
|
+
|
|
5
|
+ Example:
|
|
6
|
+
|
|
7
|
+ Name ips average deviation median 99th %
|
|
8
|
+ flat_map 2.40 K 417.00 μs ±9.40% 411.45 μs 715.21 μs
|
|
9
|
+ map.flatten 1.24 K 806.89 μs ±16.62% 768.02 μs 1170.67 μs
|
|
10
|
+
|
|
11
|
+ Comparison:
|
|
12
|
+ flat_map 2.40 K
|
|
13
|
+ map.flatten 1.24 K - 1.93x slower
|
|
14
|
+
|
|
15
|
+ Memory usage statistics:
|
|
16
|
+
|
|
17
|
+ Name Memory usage
|
|
18
|
+ flat_map 624.97 KB
|
|
19
|
+ map.flatten 781.25 KB - 1.25x memory usage
|
|
20
|
+
|
|
21
|
+ **All measurements for memory usage were the same**
|
|
5
22
|
"""
|
|
6
23
|
|
|
7
24
|
@behaviour Benchee.Formatter
|
|
|
@@ -9,8 +26,6 @@ defmodule Benchee.Formatters.Console do
|
|
9
26
|
alias Benchee.Suite
|
|
10
27
|
alias Benchee.Formatters.Console.{Memory, RunTime}
|
|
11
28
|
|
|
12
|
- def format(suite), do: format(suite, %{})
|
|
13
|
-
|
|
14
29
|
@doc """
|
|
15
30
|
Formats the benchmark statistics to a report suitable for output on the CLI.
|
|
16
31
|
|
|
|
@@ -22,27 +37,29 @@ defmodule Benchee.Formatters.Console do
|
|
22
37
|
|
|
23
38
|
```
|
|
24
39
|
iex> scenarios = [
|
|
25
|
- ...> %Benchee.Benchmark.Scenario{
|
|
26
|
- ...> name: "My Job", input_name: "My input", run_time_statistics: %Benchee.Statistics{
|
|
27
|
- ...> average: 200.0,
|
|
28
|
- ...> ips: 5000.0,
|
|
29
|
- ...> std_dev_ratio: 0.1,
|
|
30
|
- ...> median: 190.0,
|
|
31
|
- ...> percentiles: %{99 => 300.1},
|
|
32
|
- ...> sample_size: 200
|
|
33
|
- ...> },
|
|
34
|
- ...> memory_usage_statistics: %Benchee.Statistics{}
|
|
40
|
+ ...> %Benchee.Scenario{
|
|
41
|
+ ...> name: "My Job", input_name: "My input", run_time_data: %Benchee.CollectionData{
|
|
42
|
+ ...> statistics: %Benchee.Statistics{
|
|
43
|
+ ...> average: 200.0,
|
|
44
|
+ ...> ips: 5000.0,
|
|
45
|
+ ...> std_dev_ratio: 0.1,
|
|
46
|
+ ...> median: 190.0,
|
|
47
|
+ ...> percentiles: %{99 => 300.1},
|
|
48
|
+ ...> sample_size: 200
|
|
49
|
+ ...> }
|
|
50
|
+ ...> }
|
|
35
51
|
...> },
|
|
36
|
- ...> %Benchee.Benchmark.Scenario{
|
|
37
|
- ...> name: "Job 2", input_name: "My input", run_time_statistics: %Benchee.Statistics{
|
|
38
|
- ...> average: 400.0,
|
|
39
|
- ...> ips: 2500.0,
|
|
40
|
- ...> std_dev_ratio: 0.2,
|
|
41
|
- ...> median: 390.0,
|
|
42
|
- ...> percentiles: %{99 => 500.1},
|
|
43
|
- ...> sample_size: 200
|
|
44
|
- ...> },
|
|
45
|
- ...> memory_usage_statistics: %Benchee.Statistics{}
|
|
52
|
+ ...> %Benchee.Scenario{
|
|
53
|
+ ...> name: "Job 2", input_name: "My input", run_time_data: %Benchee.CollectionData{
|
|
54
|
+ ...> statistics: %Benchee.Statistics{
|
|
55
|
+ ...> average: 400.0,
|
|
56
|
+ ...> ips: 2500.0,
|
|
57
|
+ ...> std_dev_ratio: 0.2,
|
|
58
|
+ ...> median: 390.0,
|
|
59
|
+ ...> percentiles: %{99 => 500.1},
|
|
60
|
+ ...> sample_size: 200
|
|
61
|
+ ...> }
|
|
62
|
+ ...> }
|
|
46
63
|
...> }
|
|
47
64
|
...> ]
|
|
48
65
|
iex> suite = %Benchee.Suite{
|
|
|
@@ -61,7 +78,7 @@ defmodule Benchee.Formatters.Console do
|
|
61
78
|
"""
|
|
62
79
|
@impl true
|
|
63
80
|
@spec format(Suite.t(), map) :: [any]
|
|
64
|
- def format(%Suite{scenarios: scenarios, configuration: config}, options) do
|
|
81
|
+ def format(%Suite{scenarios: scenarios, configuration: config}, options \\ %{}) do
|
|
65
82
|
if Map.has_key?(options, :unit_scaling), do: warn_unit_scaling()
|
|
66
83
|
|
|
67
84
|
config =
|
|
|
@@ -90,14 +107,12 @@ defmodule Benchee.Formatters.Console do
|
|
90
107
|
end
|
|
91
108
|
end
|
|
92
109
|
|
|
93
|
- def write(suite), do: write(suite, %{})
|
|
94
|
-
|
|
95
110
|
@doc """
|
|
96
111
|
Takes the output of `format/1` and writes that to the console.
|
|
97
112
|
"""
|
|
98
113
|
@impl true
|
|
99
114
|
@spec write(any, map) :: :ok | {:error, String.t()}
|
|
100
|
- def write(output, _options) do
|
|
115
|
+ def write(output, _options \\ %{}) do
|
|
101
116
|
IO.write(output)
|
|
102
117
|
rescue
|
|
103
118
|
_ -> {:error, "Unknown Error"}
|
changed
lib/benchee/formatters/console/helpers.ex
|
|
@@ -1,8 +1,8 @@
|
|
1
1
|
defmodule Benchee.Formatters.Console.Helpers do
|
|
2
|
- @moduledoc """
|
|
3
|
- These are common functions shared between the formatting of the run time and
|
|
4
|
- memory usage statistics.
|
|
5
|
- """
|
|
2
|
+ @moduledoc false
|
|
3
|
+
|
|
4
|
+ # These are common functions shared between the formatting of the run time and
|
|
5
|
+ # memory usage statistics.
|
|
6
6
|
|
|
7
7
|
alias Benchee.Conversion.{Count, DeviationPercent, Format, Scale, Unit}
|
|
8
8
|
alias Benchee.Statistics
|
|
|
@@ -49,4 +49,51 @@ defmodule Benchee.Formatters.Console.Helpers do
|
|
49
49
|
|
|
50
50
|
@spec descriptor(String.t()) :: String.t()
|
|
51
51
|
def descriptor(header_str), do: "\n#{header_str}: \n"
|
|
52
|
+
|
|
53
|
+ def format_comparison(
|
|
54
|
+ name,
|
|
55
|
+ statistics,
|
|
56
|
+ display_value,
|
|
57
|
+ comparison_name,
|
|
58
|
+ display_unit,
|
|
59
|
+ label_width,
|
|
60
|
+ column_width
|
|
61
|
+ ) do
|
|
62
|
+ "~*s~*s ~ts"
|
|
63
|
+ |> :io_lib.format([
|
|
64
|
+ -label_width,
|
|
65
|
+ name,
|
|
66
|
+ column_width,
|
|
67
|
+ display_value,
|
|
68
|
+ comparison_display(statistics, comparison_name, display_unit)
|
|
69
|
+ ])
|
|
70
|
+ |> to_string
|
|
71
|
+ end
|
|
72
|
+
|
|
73
|
+ defp comparison_display(%Statistics{relative_more: nil, absolute_difference: nil}, _, _), do: ""
|
|
74
|
+
|
|
75
|
+ defp comparison_display(statistics, comparison_name, unit) do
|
|
76
|
+ "- #{comparison_text(statistics, comparison_name)} #{
|
|
77
|
+ absolute_difference_text(statistics, unit)
|
|
78
|
+ }\n"
|
|
79
|
+ end
|
|
80
|
+
|
|
81
|
+ defp comparison_text(%Statistics{relative_more: :infinity}, name), do: "∞ x #{name}"
|
|
82
|
+ defp comparison_text(%Statistics{relative_more: nil}, _), do: "N/A"
|
|
83
|
+
|
|
84
|
+ defp comparison_text(statistics, comparison_name) do
|
|
85
|
+ "~.2fx ~s"
|
|
86
|
+ |> :io_lib.format([statistics.relative_more, comparison_name])
|
|
87
|
+ |> to_string
|
|
88
|
+ end
|
|
89
|
+
|
|
90
|
+ defp absolute_difference_text(statistics, unit) do
|
|
91
|
+ formatted_value = Format.format({Scale.scale(statistics.absolute_difference, unit), unit})
|
|
92
|
+
|
|
93
|
+ if statistics.absolute_difference >= 0 do
|
|
94
|
+ "+#{formatted_value}"
|
|
95
|
+ else
|
|
96
|
+ formatted_value
|
|
97
|
+ end
|
|
98
|
+ end
|
|
52
99
|
end
|
changed
lib/benchee/formatters/console/memory.ex
|
|
@@ -1,18 +1,18 @@
|
|
1
1
|
defmodule Benchee.Formatters.Console.Memory do
|
|
2
|
- @moduledoc """
|
|
3
|
- This deals with just the formatting of the run time results. They are similar
|
|
4
|
- to the way the memory results are formatted, but different enough to where the
|
|
5
|
- abstractions start to break down pretty significantly, so I wanted to extract
|
|
6
|
- these two things into separate modules to avoid confusion.
|
|
7
|
- """
|
|
2
|
+ @moduledoc false
|
|
3
|
+
|
|
4
|
+ # This deals with just the formatting of the run time results. They are similar
|
|
5
|
+ # to the way the memory results are formatted, but different enough to where the
|
|
6
|
+ # abstractions start to break down pretty significantly, so I wanted to extract
|
|
7
|
+ # these two things into separate modules to avoid confusion.
|
|
8
8
|
|
|
9
9
|
alias Benchee.{
|
|
10
|
- Benchmark.Scenario,
|
|
11
10
|
Conversion,
|
|
12
11
|
Conversion.Count,
|
|
13
12
|
Conversion.Memory,
|
|
14
13
|
Conversion.Unit,
|
|
15
14
|
Formatters.Console.Helpers,
|
|
15
|
+ Scenario,
|
|
16
16
|
Statistics
|
|
17
17
|
}
|
|
18
18
|
|
|
|
@@ -45,7 +45,7 @@ defmodule Benchee.Formatters.Console.Memory do
|
|
45
45
|
|
|
46
46
|
defp memory_measurements_present?(scenarios) do
|
|
47
47
|
Enum.any?(scenarios, fn scenario ->
|
|
48
|
- scenario.memory_usage_statistics.sample_size > 0
|
|
48
|
+ scenario.memory_usage_data.statistics.sample_size > 0
|
|
49
49
|
end)
|
|
50
50
|
end
|
|
51
51
|
|
|
|
@@ -66,7 +66,7 @@ defmodule Benchee.Formatters.Console.Memory do
|
|
66
66
|
|
|
67
67
|
defp all_have_deviation_of_0?(scenarios) do
|
|
68
68
|
Enum.all?(scenarios, fn scenario ->
|
|
69
|
- scenario.memory_usage_statistics.std_dev == 0.0
|
|
69
|
+ scenario.memory_usage_data.statistics.std_dev == 0.0
|
|
70
70
|
end)
|
|
71
71
|
end
|
|
72
72
|
|
|
|
@@ -106,7 +106,7 @@ defmodule Benchee.Formatters.Console.Memory do
|
|
106
106
|
defp scenario_reports([scenario | other_scenarios], units, label_width, true) do
|
|
107
107
|
[
|
|
108
108
|
reference_report(scenario, units, label_width),
|
|
109
|
- comparisons(scenario, units, label_width, other_scenarios),
|
|
109
|
+ comparisons(other_scenarios, units, label_width),
|
|
110
110
|
"\n**All measurements for memory usage were the same**\n"
|
|
111
111
|
]
|
|
112
112
|
end
|
|
|
@@ -123,7 +123,7 @@ defmodule Benchee.Formatters.Console.Memory do
|
|
123
123
|
defp format_scenario(scenario, units, label_width, hide_statistics)
|
|
124
124
|
|
|
125
125
|
defp format_scenario(
|
|
126
|
- scenario = %Scenario{memory_usage_statistics: %{sample_size: 0}},
|
|
126
|
+ scenario = %Scenario{memory_usage_data: %{statistics: %{sample_size: 0}}},
|
|
127
127
|
_,
|
|
128
128
|
label_width,
|
|
129
129
|
_
|
|
|
@@ -149,11 +149,13 @@ defmodule Benchee.Formatters.Console.Memory do
|
|
149
149
|
defp format_scenario(scenario, %{memory: memory_unit}, label_width, false) do
|
|
150
150
|
%Scenario{
|
|
151
151
|
name: name,
|
|
152
|
- memory_usage_statistics: %Statistics{
|
|
153
|
- average: average,
|
|
154
|
- std_dev_ratio: std_dev_ratio,
|
|
155
|
- median: median,
|
|
156
|
- percentiles: %{99 => percentile_99}
|
|
152
|
+ memory_usage_data: %{
|
|
153
|
+ statistics: %Statistics{
|
|
154
|
+ average: average,
|
|
155
|
+ std_dev_ratio: std_dev_ratio,
|
|
156
|
+ median: median,
|
|
157
|
+ percentiles: %{99 => percentile_99}
|
|
158
|
+ }
|
|
157
159
|
}
|
|
158
160
|
} = scenario
|
|
159
161
|
|
|
|
@@ -176,8 +178,10 @@ defmodule Benchee.Formatters.Console.Memory do
|
|
176
178
|
defp format_scenario(scenario, %{memory: memory_unit}, label_width, true) do
|
|
177
179
|
%Scenario{
|
|
178
180
|
name: name,
|
|
179
|
- memory_usage_statistics: %Statistics{
|
|
180
|
- average: average
|
|
181
|
+ memory_usage_data: %{
|
|
182
|
+ statistics: %Statistics{
|
|
183
|
+ average: average
|
|
184
|
+ }
|
|
181
185
|
}
|
|
182
186
|
} = scenario
|
|
183
187
|
|
|
|
@@ -205,15 +209,13 @@ defmodule Benchee.Formatters.Console.Memory do
|
|
205
209
|
[
|
|
206
210
|
Helpers.descriptor("Comparison"),
|
|
207
211
|
reference_report(scenario, units, label_width)
|
|
208
|
- | comparisons(scenario, units, label_width, other_scenarios)
|
|
212
|
+ | comparisons(other_scenarios, units, label_width)
|
|
209
213
|
]
|
|
210
214
|
end
|
|
211
215
|
|
|
212
216
|
defp reference_report(scenario, %{memory: memory_unit}, label_width) do
|
|
213
|
- %Scenario{
|
|
214
|
- name: name,
|
|
215
|
- memory_usage_statistics: %Statistics{median: median}
|
|
216
|
- } = scenario
|
|
217
|
+ %Scenario{name: name, memory_usage_data: %{statistics: %Statistics{median: median}}} =
|
|
218
|
+ scenario
|
|
217
219
|
|
|
218
220
|
"~*s~*s\n"
|
|
219
221
|
|> :io_lib.format([
|
|
|
@@ -225,43 +227,25 @@ defmodule Benchee.Formatters.Console.Memory do
|
|
225
227
|
|> to_string
|
|
226
228
|
end
|
|
227
229
|
|
|
228
|
- @spec comparisons(Scenario.t(), unit_per_statistic, integer, [Scenario.t()]) :: [String.t()]
|
|
229
|
- defp comparisons(scenario, units, label_width, scenarios_to_compare) do
|
|
230
|
- %Scenario{memory_usage_statistics: reference_stats} = scenario
|
|
230
|
+ @spec comparisons([Scenario.t()], unit_per_statistic, integer) :: [String.t()]
|
|
231
|
+ defp comparisons(scenarios_to_compare, units, label_width) do
|
|
232
|
+ Enum.map(
|
|
233
|
+ scenarios_to_compare,
|
|
234
|
+ fn scenario ->
|
|
235
|
+ statistics = scenario.memory_usage_data.statistics
|
|
236
|
+ memory_format = memory_output(statistics.average, units.memory)
|
|
231
237
|
|
|
232
|
- Enum.map(scenarios_to_compare, fn scenario = %Scenario{memory_usage_statistics: job_stats} ->
|
|
233
|
- slower = calculate_slower_value(job_stats.median, reference_stats.median)
|
|
234
|
-
|
|
235
|
- format_comparison(scenario, units, label_width, slower)
|
|
236
|
- end)
|
|
237
|
- end
|
|
238
|
-
|
|
239
|
- defp calculate_slower_value(job_median, reference_median)
|
|
240
|
- when job_median == 0 or is_nil(job_median) or reference_median == 0 or
|
|
241
|
- is_nil(reference_median) do
|
|
242
|
- @na
|
|
243
|
- end
|
|
244
|
-
|
|
245
|
- defp calculate_slower_value(job_median, reference_median) do
|
|
246
|
- job_median / reference_median
|
|
247
|
- end
|
|
248
|
-
|
|
249
|
- defp format_comparison(scenario, %{memory: memory_unit}, label_width, @na) do
|
|
250
|
- %Scenario{name: name, memory_usage_statistics: %Statistics{median: median}} = scenario
|
|
251
|
- median_format = memory_output(median, memory_unit)
|
|
252
|
-
|
|
253
|
- "~*s~*s\n"
|
|
254
|
- |> :io_lib.format([-label_width, name, @median_width, median_format])
|
|
255
|
- |> to_string
|
|
256
|
- end
|
|
257
|
-
|
|
258
|
- defp format_comparison(scenario, %{memory: memory_unit}, label_width, slower) do
|
|
259
|
- %Scenario{name: name, memory_usage_statistics: %Statistics{median: median}} = scenario
|
|
260
|
- median_format = memory_output(median, memory_unit)
|
|
261
|
-
|
|
262
|
- "~*s~*s - ~.2fx memory usage\n"
|
|
263
|
- |> :io_lib.format([-label_width, name, @median_width, median_format, slower])
|
|
264
|
- |> to_string
|
|
238
|
+ Helpers.format_comparison(
|
|
239
|
+ scenario.name,
|
|
240
|
+ statistics,
|
|
241
|
+ memory_format,
|
|
242
|
+ "memory usage",
|
|
243
|
+ units.memory,
|
|
244
|
+ label_width,
|
|
245
|
+ @median_width
|
|
246
|
+ )
|
|
247
|
+ end
|
|
248
|
+ )
|
|
265
249
|
end
|
|
266
250
|
|
|
267
251
|
defp memory_output(nil, _unit), do: "N/A"
|
|
|
@@ -307,11 +291,13 @@ defmodule Benchee.Formatters.Console.Memory do
|
|
307
291
|
defp format_scenario_extended(scenario, %{memory: memory_unit}, label_width) do
|
|
308
292
|
%Scenario{
|
|
309
293
|
name: name,
|
|
310
|
- memory_usage_statistics: %Statistics{
|
|
311
|
- minimum: minimum,
|
|
312
|
- maximum: maximum,
|
|
313
|
- sample_size: sample_size,
|
|
314
|
- mode: mode
|
|
294
|
+ memory_usage_data: %{
|
|
295
|
+ statistics: %Statistics{
|
|
296
|
+ minimum: minimum,
|
|
297
|
+ maximum: maximum,
|
|
298
|
+ sample_size: sample_size,
|
|
299
|
+ mode: mode
|
|
300
|
+ }
|
|
315
301
|
}
|
|
316
302
|
} = scenario
|
changed
lib/benchee/formatters/console/run_time.ex
|
|
@@ -1,18 +1,18 @@
|
|
1
1
|
defmodule Benchee.Formatters.Console.RunTime do
|
|
2
|
- @moduledoc """
|
|
3
|
- This deals with just the formatting of the run time results. They are similar
|
|
4
|
- to the way the memory results are formatted, but different enough to where the
|
|
5
|
- abstractions start to break down pretty significantly, so I wanted to extract
|
|
6
|
- these two things into separate modules to avoid confusion.
|
|
7
|
- """
|
|
2
|
+ @moduledoc false
|
|
3
|
+
|
|
4
|
+ # This deals with just the formatting of the run time results. They are similar
|
|
5
|
+ # to the way the memory results are formatted, but different enough to where the
|
|
6
|
+ # abstractions start to break down pretty significantly, so I wanted to extract
|
|
7
|
+ # these two things into separate modules to avoid confusion.
|
|
8
8
|
|
|
9
9
|
alias Benchee.{
|
|
10
|
- Benchmark.Scenario,
|
|
11
10
|
Conversion,
|
|
12
11
|
Conversion.Count,
|
|
13
12
|
Conversion.Duration,
|
|
14
13
|
Conversion.Unit,
|
|
15
14
|
Formatters.Console.Helpers,
|
|
15
|
+ Scenario,
|
|
16
16
|
Statistics
|
|
17
17
|
}
|
|
18
18
|
|
|
|
@@ -36,21 +36,25 @@ defmodule Benchee.Formatters.Console.RunTime do
|
|
36
36
|
```
|
|
37
37
|
iex> memory_statistics = %Benchee.Statistics{average: 100.0}
|
|
38
38
|
iex> scenarios = [
|
|
39
|
- ...> %Benchee.Benchmark.Scenario{
|
|
39
|
+ ...> %Benchee.Scenario{
|
|
40
40
|
...> name: "My Job",
|
|
41
|
- ...> run_time_statistics: %Benchee.Statistics{
|
|
42
|
- ...> average: 200.0, ips: 5000.0,std_dev_ratio: 0.1, median: 190.0, percentiles: %{99 => 300.1},
|
|
43
|
- ...> minimum: 100.1, maximum: 200.2, sample_size: 10_101, mode: 333.2
|
|
41
|
+ ...> run_time_data: %Benchee.CollectionData{
|
|
42
|
+ ...> statistics: %Benchee.Statistics{
|
|
43
|
+ ...> average: 200.0, ips: 5000.0,std_dev_ratio: 0.1, median: 190.0, percentiles: %{99 => 300.1},
|
|
44
|
+ ...> minimum: 100.1, maximum: 200.2, sample_size: 10_101, mode: 333.2
|
|
45
|
+ ...> },
|
|
44
46
|
...> },
|
|
45
|
- ...> memory_usage_statistics: memory_statistics
|
|
47
|
+ ...> memory_usage_data: %Benchee.CollectionData{statistics: memory_statistics}
|
|
46
48
|
...> },
|
|
47
|
- ...> %Benchee.Benchmark.Scenario{
|
|
49
|
+ ...> %Benchee.Scenario{
|
|
48
50
|
...> name: "Job 2",
|
|
49
|
- ...> run_time_statistics: %Benchee.Statistics{
|
|
50
|
- ...> average: 400.0, ips: 2500.0, std_dev_ratio: 0.2, median: 390.0, percentiles: %{99 => 500.1},
|
|
51
|
- ...> minimum: 200.2, maximum: 400.4, sample_size: 20_202, mode: [612.3, 554.1]
|
|
51
|
+ ...> run_time_data: %Benchee.CollectionData{
|
|
52
|
+ ...> statistics: %Benchee.Statistics{
|
|
53
|
+ ...> average: 400.0, ips: 2500.0, std_dev_ratio: 0.2, median: 390.0, percentiles: %{99 => 500.1},
|
|
54
|
+ ...> minimum: 200.2, maximum: 400.4, sample_size: 20_202, mode: [612.3, 554.1]
|
|
55
|
+ ...> }
|
|
52
56
|
...> },
|
|
53
|
- ...> memory_usage_statistics: memory_statistics
|
|
57
|
+ ...> memory_usage_data: %Benchee.CollectionData{statistics: memory_statistics}
|
|
54
58
|
...> }
|
|
55
59
|
...> ]
|
|
56
60
|
iex> configuration = %{comparison: false, unit_scaling: :best, extended_statistics: true}
|
|
|
@@ -77,7 +81,7 @@ defmodule Benchee.Formatters.Console.RunTime do
|
|
77
81
|
|
|
78
82
|
defp run_time_measurements_present?(scenarios) do
|
|
79
83
|
Enum.any?(scenarios, fn scenario ->
|
|
80
|
- scenario.run_time_statistics.sample_size > 0
|
|
84
|
+ scenario.run_time_data.statistics.sample_size > 0
|
|
81
85
|
end)
|
|
82
86
|
end
|
|
83
87
|
|
|
|
@@ -120,11 +124,13 @@ defmodule Benchee.Formatters.Console.RunTime do
|
|
120
124
|
defp format_scenario_extended(scenario, %{run_time: run_time_unit}, label_width) do
|
|
121
125
|
%Scenario{
|
|
122
126
|
name: name,
|
|
123
|
- run_time_statistics: %Statistics{
|
|
124
|
- minimum: minimum,
|
|
125
|
- maximum: maximum,
|
|
126
|
- sample_size: sample_size,
|
|
127
|
- mode: mode
|
|
127
|
+ run_time_data: %{
|
|
128
|
+ statistics: %Statistics{
|
|
129
|
+ minimum: minimum,
|
|
130
|
+ maximum: maximum,
|
|
131
|
+ sample_size: sample_size,
|
|
132
|
+ mode: mode
|
|
133
|
+ }
|
|
128
134
|
}
|
|
129
135
|
} = scenario
|
|
130
136
|
|
|
|
@@ -193,12 +199,14 @@ defmodule Benchee.Formatters.Console.RunTime do
|
|
193
199
|
defp format_scenario(scenario, %{run_time: run_time_unit, ips: ips_unit}, label_width) do
|
|
194
200
|
%Scenario{
|
|
195
201
|
name: name,
|
|
196
|
- run_time_statistics: %Statistics{
|
|
197
|
- average: average,
|
|
198
|
- ips: ips,
|
|
199
|
- std_dev_ratio: std_dev_ratio,
|
|
200
|
- median: median,
|
|
201
|
- percentiles: %{99 => percentile_99}
|
|
202
|
+ run_time_data: %{
|
|
203
|
+ statistics: %Statistics{
|
|
204
|
+ average: average,
|
|
205
|
+ ips: ips,
|
|
206
|
+ std_dev_ratio: std_dev_ratio,
|
|
207
|
+ median: median,
|
|
208
|
+ percentiles: %{99 => percentile_99}
|
|
209
|
+ }
|
|
202
210
|
}
|
|
203
211
|
} = scenario
|
|
204
212
|
|
|
|
@@ -231,38 +239,37 @@ defmodule Benchee.Formatters.Console.RunTime do
|
|
231
239
|
[
|
|
232
240
|
Helpers.descriptor("Comparison"),
|
|
233
241
|
reference_report(scenario, units, label_width)
|
|
234
|
- | comparisons(scenario, units, label_width, other_scenarios)
|
|
242
|
+ | comparisons(other_scenarios, units, label_width)
|
|
235
243
|
]
|
|
236
244
|
end
|
|
237
245
|
|
|
238
246
|
defp reference_report(scenario, %{ips: ips_unit}, label_width) do
|
|
239
|
- %Scenario{
|
|
240
|
- name: name,
|
|
241
|
- run_time_statistics: %Statistics{ips: ips}
|
|
242
|
- } = scenario
|
|
247
|
+ %Scenario{name: name, run_time_data: %{statistics: %Statistics{ips: ips}}} = scenario
|
|
243
248
|
|
|
244
249
|
"~*s~*s\n"
|
|
245
250
|
|> :io_lib.format([-label_width, name, @ips_width, Helpers.count_output(ips, ips_unit)])
|
|
246
251
|
|> to_string
|
|
247
252
|
end
|
|
248
253
|
|
|
249
|
- @spec comparisons(Scenario.t(), unit_per_statistic, integer, [Scenario.t()]) :: [String.t()]
|
|
250
|
- defp comparisons(scenario, units, label_width, scenarios_to_compare) do
|
|
251
|
- %Scenario{run_time_statistics: reference_stats} = scenario
|
|
254
|
+ @spec comparisons([Scenario.t()], unit_per_statistic, integer) :: [String.t()]
|
|
255
|
+ defp comparisons(scenarios_to_compare, units, label_width) do
|
|
256
|
+ Enum.map(
|
|
257
|
+ scenarios_to_compare,
|
|
258
|
+ fn scenario ->
|
|
259
|
+ statistics = scenario.run_time_data.statistics
|
|
260
|
+ ips_format = Helpers.count_output(statistics.ips, units.ips)
|
|
252
261
|
|
|
253
|
- Enum.map(scenarios_to_compare, fn scenario = %Scenario{run_time_statistics: job_stats} ->
|
|
254
|
- slower = reference_stats.ips / job_stats.ips
|
|
255
|
- format_comparison(scenario, units, label_width, slower)
|
|
256
|
- end)
|
|
257
|
- end
|
|
258
|
-
|
|
259
|
- defp format_comparison(scenario, %{ips: ips_unit}, label_width, slower) do
|
|
260
|
- %Scenario{name: name, run_time_statistics: %Statistics{ips: ips}} = scenario
|
|
261
|
- ips_format = Helpers.count_output(ips, ips_unit)
|
|
262
|
-
|
|
263
|
- "~*s~*s - ~.2fx slower\n"
|
|
264
|
- |> :io_lib.format([-label_width, name, @ips_width, ips_format, slower])
|
|
265
|
- |> to_string
|
|
262
|
+ Helpers.format_comparison(
|
|
263
|
+ scenario.name,
|
|
264
|
+ statistics,
|
|
265
|
+ ips_format,
|
|
266
|
+ "slower",
|
|
267
|
+ units.run_time,
|
|
268
|
+ label_width,
|
|
269
|
+ @ips_width
|
|
270
|
+ )
|
|
271
|
+ end
|
|
272
|
+ )
|
|
266
273
|
end
|
|
267
274
|
|
|
268
275
|
defp duration_output(duration, unit) do
|
changed
lib/benchee/formatters/tagged_save.ex
|
|
@@ -2,18 +2,21 @@ defmodule Benchee.Formatters.TaggedSave do
|
|
2
2
|
@moduledoc """
|
|
3
3
|
Store the whole suite in the Erlang `ExternalTermFormat` while tagging the
|
|
4
4
|
scenarios of the current run with a specified tag - can be used for storing
|
|
5
|
- and loading the results of previous runs.
|
|
5
|
+ and later loading the results of previous runs with `Benchee.ScenarioLoader`.
|
|
6
6
|
|
|
7
|
- Automatically invoked and appended when specifiying the `save` option in the
|
|
8
|
- configuration.
|
|
7
|
+ Automatically configured as the last formatter to run when specifiying the
|
|
8
|
+ `save` option in the configuration - see `Benchee.Configuration`.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
@behaviour Benchee.Formatter
|
|
12
12
|
|
|
13
|
- alias Benchee.Benchmark.Scenario
|
|
13
|
+ alias Benchee.Scenario
|
|
14
14
|
alias Benchee.Suite
|
|
15
15
|
alias Benchee.Utility.FileCreation
|
|
16
16
|
|
|
17
|
+ @doc """
|
|
18
|
+ Tags all scenario with the desired tag and returns it in term_to_binary along with save path.
|
|
19
|
+ """
|
|
17
20
|
@impl true
|
|
18
21
|
@spec format(Suite.t(), map) :: {binary, String.t()}
|
|
19
22
|
def format(suite = %Suite{scenarios: scenarios}, formatter_config) do
|
|
|
@@ -71,6 +74,9 @@ defmodule Benchee.Formatters.TaggedSave do
|
|
71
74
|
%Scenario{scenario | name: Scenario.display_name(scenario)}
|
|
72
75
|
end
|
|
73
76
|
|
|
77
|
+ @doc """
|
|
78
|
+ Writes the binary returned by `format/2` to the indicated location, telling you where that is.
|
|
79
|
+ """
|
|
74
80
|
@spec write({binary, String.t()}, map) :: :ok
|
|
75
81
|
@impl true
|
|
76
82
|
def write({term_binary, filename}, _) do
|
changed
lib/benchee/output/benchmark_printer.ex
|
|
@@ -64,7 +64,6 @@ defmodule Benchee.Output.BenchmarkPrinter do
|
|
64
64
|
parallel: #{parallel}
|
|
65
65
|
inputs: #{inputs_out(inputs)}
|
|
66
66
|
Estimated total run time: #{Duration.format(total_time)}
|
|
67
|
-
|
|
68
67
|
""")
|
|
69
68
|
end
|
added
lib/benchee/relative_statistics.ex
|
|
@@ -0,0 +1,98 @@
|
|
1
|
+ defmodule Benchee.RelativeStatistics do
|
|
2
|
+ @moduledoc """
|
|
3
|
+ Statistics that are relative from one scenario to another.
|
|
4
|
+
|
|
5
|
+ Such as how much slower/faster something is or what the absolute difference is in the measured
|
|
6
|
+ values.
|
|
7
|
+ Is its own step because it has to be executed after scenarios have been loaded via
|
|
8
|
+ `Benchee.ScenarioLoader` to include them in the calculation, while `Benchee.Statistics`
|
|
9
|
+ has to happen before they are loaded to avoid recalculating their statistics.
|
|
10
|
+ """
|
|
11
|
+
|
|
12
|
+ alias Benchee.{Scenario, Statistics, Suite}
|
|
13
|
+
|
|
14
|
+ @doc """
|
|
15
|
+ Calculate the statistics of scenarios relative to each other and sorts scenarios.
|
|
16
|
+
|
|
17
|
+ Such as `relative_more`, `relative_less` and `absolute_difference`,
|
|
18
|
+ see `t:Benchee.Statistics.t/0` for more.
|
|
19
|
+
|
|
20
|
+ The sorting of scenarios is important so that they always have the same order in
|
|
21
|
+ all formatters. Scenarios are sorted first by run time average, then by memory average.
|
|
22
|
+ """
|
|
23
|
+ @spec relative_statistics(Suite.t()) :: Suite.t()
|
|
24
|
+ def relative_statistics(suite) do
|
|
25
|
+ scenarios =
|
|
26
|
+ suite.scenarios
|
|
27
|
+ |> sort()
|
|
28
|
+ |> calculate_relative_statistics(suite.configuration.inputs)
|
|
29
|
+
|
|
30
|
+ %Suite{suite | scenarios: scenarios}
|
|
31
|
+ end
|
|
32
|
+
|
|
33
|
+ defp calculate_relative_statistics([], _inputs), do: []
|
|
34
|
+
|
|
35
|
+ defp calculate_relative_statistics(scenarios, inputs) do
|
|
36
|
+ scenarios
|
|
37
|
+ |> scenarios_by_input(inputs)
|
|
38
|
+ |> Enum.flat_map(fn scenarios_with_same_input ->
|
|
39
|
+ {reference, others} = split_reference_scenario(scenarios_with_same_input)
|
|
40
|
+ others_with_relative = statistics_relative_to(others, reference)
|
|
41
|
+ [reference | others_with_relative]
|
|
42
|
+ end)
|
|
43
|
+ end
|
|
44
|
+
|
|
45
|
+ @spec sort([Scenario.t()]) :: [Scenario.t()]
|
|
46
|
+ defp sort(scenarios) do
|
|
47
|
+ Enum.sort_by(scenarios, fn scenario ->
|
|
48
|
+ {scenario.run_time_data.statistics.average, scenario.memory_usage_data.statistics.average}
|
|
49
|
+ end)
|
|
50
|
+ end
|
|
51
|
+
|
|
52
|
+ defp scenarios_by_input(scenarios, nil), do: [scenarios]
|
|
53
|
+
|
|
54
|
+ # we can't just group_by `input_name` because that'd lose the order of inputs which might
|
|
55
|
+ # be important
|
|
56
|
+ defp scenarios_by_input(scenarios, inputs) do
|
|
57
|
+ Enum.map(inputs, fn {input_name, _} ->
|
|
58
|
+ Enum.filter(scenarios, fn scenario -> scenario.input_name == input_name end)
|
|
59
|
+ end)
|
|
60
|
+ end
|
|
61
|
+
|
|
62
|
+ # right now we take the first scenario as we sorted them and it is the fastest,
|
|
63
|
+ # whenever we implement #179 though this becomesd more involved
|
|
64
|
+ defp split_reference_scenario(scenarios) do
|
|
65
|
+ [reference | others] = scenarios
|
|
66
|
+ {reference, others}
|
|
67
|
+ end
|
|
68
|
+
|
|
69
|
+ defp statistics_relative_to(scenarios, reference) do
|
|
70
|
+ Enum.map(scenarios, fn scenario ->
|
|
71
|
+ scenario
|
|
72
|
+ |> update_in([Access.key!(:run_time_data), Access.key!(:statistics)], fn statistics ->
|
|
73
|
+ add_relative_statistics(statistics, reference.run_time_data.statistics)
|
|
74
|
+ end)
|
|
75
|
+ |> update_in([Access.key!(:memory_usage_data), Access.key!(:statistics)], fn statistics ->
|
|
76
|
+ add_relative_statistics(statistics, reference.memory_usage_data.statistics)
|
|
77
|
+ end)
|
|
78
|
+ end)
|
|
79
|
+ end
|
|
80
|
+
|
|
81
|
+ # we might not run time/memory --> we shouldn't crash then ;)
|
|
82
|
+ defp add_relative_statistics(statistics = %{average: nil}, _reference), do: statistics
|
|
83
|
+ defp add_relative_statistics(statistics, %{average: nil}), do: statistics
|
|
84
|
+
|
|
85
|
+ defp add_relative_statistics(statistics, reference_statistics) do
|
|
86
|
+ %Statistics{
|
|
87
|
+ statistics
|
|
88
|
+ | relative_more: zero_safe_division(statistics.average, reference_statistics.average),
|
|
89
|
+ relative_less: zero_safe_division(reference_statistics.average, statistics.average),
|
|
90
|
+ absolute_difference: statistics.average - reference_statistics.average
|
|
91
|
+ }
|
|
92
|
+ end
|
|
93
|
+
|
|
94
|
+ defp zero_safe_division(0.0, 0.0), do: 1.0
|
|
95
|
+ defp zero_safe_division(_, 0), do: :infinity
|
|
96
|
+ defp zero_safe_division(_, 0.0), do: :infinity
|
|
97
|
+ defp zero_safe_division(a, b), do: a / b
|
|
98
|
+ end
|
added
lib/benchee/scenario.ex
|
|
@@ -0,0 +1,110 @@
|
|
1
|
+ defmodule Benchee.Scenario do
|
|
2
|
+ @moduledoc """
|
|
3
|
+ Core data structure representing one particular case (combination of function and input).
|
|
4
|
+
|
|
5
|
+ Represents the combination of a particular function to benchmark (also called "job" defined
|
|
6
|
+ by `job_name` and `function`) in combination with a specific input (`input_name` and `input`).
|
|
7
|
+ When no input is given, the combined value is representative of "no input".
|
|
8
|
+
|
|
9
|
+ A scenario then further gathers all data collected for this particular combination during
|
|
10
|
+ `Benchee.Benchmark.collect/3`, which are then used later in the process by `Benchee.Statistics`
|
|
11
|
+ to compute the relevant statistics which are then also added to the scenario.
|
|
12
|
+ It is the home of the aggregated knowledge regarding this particular case/scenario.
|
|
13
|
+
|
|
14
|
+ `name` is the name that should be used by formatters to display scenarios as
|
|
15
|
+ it potentially includes the `tag` present when loading scenarios that were
|
|
16
|
+ saved before. See `display_name/1`.
|
|
17
|
+ """
|
|
18
|
+
|
|
19
|
+ alias Benchee.CollectionData
|
|
20
|
+
|
|
21
|
+ defstruct [
|
|
22
|
+ :name,
|
|
23
|
+ :job_name,
|
|
24
|
+ :function,
|
|
25
|
+ :input_name,
|
|
26
|
+ :input,
|
|
27
|
+ run_time_data: %CollectionData{},
|
|
28
|
+ memory_usage_data: %CollectionData{},
|
|
29
|
+ before_each: nil,
|
|
30
|
+ after_each: nil,
|
|
31
|
+ before_scenario: nil,
|
|
32
|
+ after_scenario: nil,
|
|
33
|
+ tag: nil
|
|
34
|
+ ]
|
|
35
|
+
|
|
36
|
+ @typedoc """
|
|
37
|
+ All the data collected for a scenario (combination of function and input)
|
|
38
|
+
|
|
39
|
+ Among all the data required to execute the scenario (function, input, all the hooks aka
|
|
40
|
+ after_*/before_*), data needed to display (name, job_name, input_name, tag) and of course
|
|
41
|
+ run_time_data and memory_data with all the samples and computed statistics.
|
|
42
|
+ """
|
|
43
|
+ @type t :: %__MODULE__{
|
|
44
|
+ name: String.t(),
|
|
45
|
+ job_name: String.t(),
|
|
46
|
+ function: fun,
|
|
47
|
+ input_name: String.t() | nil,
|
|
48
|
+ input: any | nil,
|
|
49
|
+ run_time_data: CollectionData.t(),
|
|
50
|
+ memory_usage_data: CollectionData.t(),
|
|
51
|
+ before_each: fun | nil,
|
|
52
|
+ after_each: fun | nil,
|
|
53
|
+ before_scenario: fun | nil,
|
|
54
|
+ after_scenario: fun | nil,
|
|
55
|
+ tag: String.t() | nil
|
|
56
|
+ }
|
|
57
|
+
|
|
58
|
+ @doc """
|
|
59
|
+ Returns the correct name to display of the given scenario data.
|
|
60
|
+
|
|
61
|
+ In the normal case this is `job_name`, however when scenarios are loaded they
|
|
62
|
+ are tagged and these tags should be shown for disambiguation.
|
|
63
|
+
|
|
64
|
+ ## Examples
|
|
65
|
+
|
|
66
|
+ iex> alias Benchee.Scenario
|
|
67
|
+ iex> Scenario.display_name(%Scenario{job_name: "flat_map"})
|
|
68
|
+ "flat_map"
|
|
69
|
+ iex> Scenario.display_name(%Scenario{job_name: "flat_map", tag: "master"})
|
|
70
|
+ "flat_map (master)"
|
|
71
|
+ iex> Scenario.display_name(%{job_name: "flat_map"})
|
|
72
|
+ "flat_map"
|
|
73
|
+ """
|
|
74
|
+ @spec display_name(t) :: String.t()
|
|
75
|
+ def display_name(%{job_name: job_name, tag: nil}), do: job_name
|
|
76
|
+ def display_name(%{job_name: job_name, tag: tag}), do: "#{job_name} (#{tag})"
|
|
77
|
+ def display_name(%{job_name: job_name}), do: job_name
|
|
78
|
+
|
|
79
|
+ @doc """
|
|
80
|
+ Returns `true` if data of the provided type has been fully procsessed, `false` otherwise.
|
|
81
|
+
|
|
82
|
+ Current available types are `run_time` and `memory`. Reasons they might not have been processed
|
|
83
|
+ yet are:
|
|
84
|
+ * Suite wasn't configured to collect them at all
|
|
85
|
+ * `Benchee.statistics/1` hasn't been called yet so that data was collected but statistics
|
|
86
|
+ aren't present yet
|
|
87
|
+
|
|
88
|
+ ## Examples
|
|
89
|
+
|
|
90
|
+ iex> alias Benchee.Scenario
|
|
91
|
+ iex> alias Benchee.Statistics
|
|
92
|
+ iex> scenario = %Scenario{run_time_data: %Benchee.CollectionData{statistics: %Statistics{sample_size: 100}}}
|
|
93
|
+ iex> Scenario.data_processed?(scenario, :run_time)
|
|
94
|
+ true
|
|
95
|
+ iex> scenario = %Scenario{memory_usage_data: %Benchee.CollectionData{statistics: %Statistics{sample_size: 1}}}
|
|
96
|
+ iex> Scenario.data_processed?(scenario, :memory)
|
|
97
|
+ true
|
|
98
|
+ iex> scenario = %Scenario{memory_usage_data: %Benchee.CollectionData{statistics: %Statistics{sample_size: 0}}}
|
|
99
|
+ iex> Scenario.data_processed?(scenario, :memory)
|
|
100
|
+ false
|
|
101
|
+ """
|
|
102
|
+ @spec data_processed?(t, :run_time | :memory) :: boolean
|
|
103
|
+ def data_processed?(scenario, :run_time) do
|
|
104
|
+ scenario.run_time_data.statistics.sample_size > 0
|
|
105
|
+ end
|
|
106
|
+
|
|
107
|
+ def data_processed?(scenario, :memory) do
|
|
108
|
+ scenario.memory_usage_data.statistics.sample_size > 0
|
|
109
|
+ end
|
|
110
|
+ end
|
changed
lib/benchee/scenario_loader.ex
|
|
@@ -5,15 +5,15 @@ defmodule Benchee.ScenarioLoader do
|
|
5
5
|
Usually this is done right before the formatters run (that's when it happens
|
|
6
6
|
in `Benchee.run/2`) as all measurements and statistics should be there.
|
|
7
7
|
However, if you want to recompute statistics or others you can load them at
|
|
8
|
- any time. Just be aware that if you load them before `Benchee.measure/1` then
|
|
8
|
+ any time. Just be aware that if you load them before `Benchee.collect/1` then
|
|
9
9
|
they'll be rerun and measurements overridden.
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
12
|
alias Benchee.Suite
|
|
13
13
|
|
|
14
14
|
@doc """
|
|
15
|
- Load the file(s) specified as `load_path` and add the scenarios to the list of
|
|
16
|
- the current scenarios.
|
|
15
|
+ Load the file(s) specified as `load_path` and add the scenarios to the list of the
|
|
16
|
+ current scenarios in the suite.
|
|
17
17
|
"""
|
|
18
18
|
def load(suite = %{configuration: %{load: load_path}, scenarios: scenarios}) do
|
|
19
19
|
loaded = load_scenarios(load_path)
|
changed
lib/benchee/statistics.ex
|
|
@@ -1,10 +1,12 @@
|
|
1
1
|
defmodule Benchee.Statistics do
|
|
2
2
|
@moduledoc """
|
|
3
|
- Statistics related functionality that is meant to take the raw benchmark run
|
|
4
|
- times and then compute statistics like the average and the standard deviation.
|
|
3
|
+ Statistics related functionality that is meant to take the raw benchmark data
|
|
4
|
+ and then compute statistics like the average and the standard deviation etc.
|
|
5
|
+
|
|
6
|
+ See `statistics/1` for a breakdown of the included statistics.
|
|
5
7
|
"""
|
|
6
8
|
|
|
7
|
- alias Benchee.{Benchmark.Scenario, Conversion.Duration, Suite, Utility.Parallel}
|
|
9
|
+ alias Benchee.{Conversion.Duration, Suite, Utility.Parallel}
|
|
8
10
|
|
|
9
11
|
alias Benchee.Statistics.Mode
|
|
10
12
|
alias Benchee.Statistics.Percentile
|
|
|
@@ -21,30 +23,21 @@ defmodule Benchee.Statistics do
|
|
21
23
|
:mode,
|
|
22
24
|
:minimum,
|
|
23
25
|
:maximum,
|
|
26
|
+ :relative_more,
|
|
27
|
+ :relative_less,
|
|
28
|
+ :absolute_difference,
|
|
24
29
|
sample_size: 0
|
|
25
30
|
]
|
|
26
31
|
|
|
32
|
+ @typedoc """
|
|
33
|
+ Careful with the mode, might be multiple values, one value or nothing.😱
|
|
34
|
+ """
|
|
27
35
|
@type mode :: [number] | number | nil
|
|
28
36
|
|
|
29
|
- @type t :: %__MODULE__{
|
|
30
|
- average: float,
|
|
31
|
- ips: float | nil,
|
|
32
|
- std_dev: float,
|
|
33
|
- std_dev_ratio: float,
|
|
34
|
- std_dev_ips: float | nil,
|
|
35
|
- median: number,
|
|
36
|
- percentiles: %{number => float},
|
|
37
|
- mode: mode,
|
|
38
|
- minimum: number,
|
|
39
|
- maximum: number,
|
|
40
|
- sample_size: integer
|
|
41
|
- }
|
|
37
|
+ @typedoc """
|
|
38
|
+ All the statistics `statistics/1` computes from the samples.
|
|
42
39
|
|
|
43
|
- @type samples :: [number]
|
|
44
|
-
|
|
45
|
- @doc """
|
|
46
|
- Takes a job suite with job run times, returns a map representing the
|
|
47
|
- statistics of the job suite as follows:
|
|
40
|
+ Overview of all the statistics benchee currently provides:
|
|
48
41
|
|
|
49
42
|
* average - average run time of the job in μs (the lower the better)
|
|
50
43
|
* ips - iterations per second, how often can the given function be
|
|
|
@@ -66,22 +59,59 @@ defmodule Benchee.Statistics do
|
|
66
59
|
* mode - the run time(s) that occur the most. Often one value, but
|
|
67
60
|
can be multiple values if they occur the same amount of times. If no value
|
|
68
61
|
occurs at least twice, this value will be nil.
|
|
69
|
- * minimum - the smallest (fastest) run time measured for the job
|
|
70
|
- * maximum - the biggest (slowest) run time measured for the job
|
|
62
|
+ * minimum - the smallest sample measured for the scenario
|
|
63
|
+ * maximum - the biggest sample measured for the scenario
|
|
64
|
+ * relative_more - relative to the reference (usually the fastest scenario) how much more
|
|
65
|
+ was the average of this scenario. E.g. for reference at 100, this scenario 200 then it
|
|
66
|
+ is 2.0.
|
|
67
|
+ * relative_less - relative to the reference (usually the fastest scenario) how much less
|
|
68
|
+ was the average of this scenario. E.g. for reference at 100, this scenario 200 then it
|
|
69
|
+ is 0.5.
|
|
70
|
+ * absolute_difference - relative to the reference (usually the fastest scenario) what is
|
|
71
|
+ the difference of the averages of the scenarios. e.g. for reference at 100, this
|
|
72
|
+ scenario 200 then it is 100.
|
|
71
73
|
* sample_size - the number of run time measurements taken
|
|
74
|
+ """
|
|
75
|
+ @type t :: %__MODULE__{
|
|
76
|
+ average: float,
|
|
77
|
+ ips: float | nil,
|
|
78
|
+ std_dev: float,
|
|
79
|
+ std_dev_ratio: float,
|
|
80
|
+ std_dev_ips: float | nil,
|
|
81
|
+ median: number,
|
|
82
|
+ percentiles: %{number => float},
|
|
83
|
+ mode: mode,
|
|
84
|
+ minimum: number,
|
|
85
|
+ maximum: number,
|
|
86
|
+ relative_more: float | nil | :infinity,
|
|
87
|
+ relative_less: float | nil | :infinity,
|
|
88
|
+ absolute_difference: float | nil,
|
|
89
|
+ sample_size: integer
|
|
90
|
+ }
|
|
72
91
|
|
|
73
|
- ## Parameters
|
|
92
|
+ @typedoc """
|
|
93
|
+ The samples a `Benchee.Collect` collected to compute statistics from.
|
|
94
|
+ """
|
|
95
|
+ @type samples :: [number]
|
|
74
96
|
|
|
75
|
- * `suite` - the job suite represented as a map after running the measurements,
|
|
76
|
- required to have the run_times available under the `run_times` key
|
|
97
|
+ @doc """
|
|
98
|
+ Takes a suite with scenarios and their data samples, adds the statistics to the
|
|
99
|
+ scenarios. For an overview of what the statistics mean see `t:t/0`.
|
|
100
|
+
|
|
101
|
+ Note that this will also sort the scenarios fastest to slowest to ensure a consistent order
|
|
102
|
+ of scenarios in all used formatters.
|
|
77
103
|
|
|
78
104
|
## Examples
|
|
79
105
|
|
|
80
106
|
iex> scenarios = [
|
|
81
|
- ...> %Benchee.Benchmark.Scenario{
|
|
107
|
+ ...> %Benchee.Scenario{
|
|
82
108
|
...> job_name: "My Job",
|
|
83
|
- ...> run_times: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
84
|
- ...> memory_usages: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
109
|
+ ...> run_time_data: %Benchee.CollectionData{
|
|
110
|
+ ...> samples: [200, 400, 400, 400, 500, 500, 500, 700, 900]
|
|
111
|
+ ...> },
|
|
112
|
+ ...> memory_usage_data: %Benchee.CollectionData{
|
|
113
|
+ ...> samples: [200, 400, 400, 400, 500, 500, 500, 700, 900]
|
|
114
|
+ ...> },
|
|
85
115
|
...> input_name: "Input",
|
|
86
116
|
...> input: "Input"
|
|
87
117
|
...> }
|
|
|
@@ -90,37 +120,41 @@ defmodule Benchee.Statistics do
|
|
90
120
|
iex> Benchee.Statistics.statistics(suite)
|
|
91
121
|
%Benchee.Suite{
|
|
92
122
|
scenarios: [
|
|
93
|
- %Benchee.Benchmark.Scenario{
|
|
123
|
+ %Benchee.Scenario{
|
|
94
124
|
job_name: "My Job",
|
|
95
|
- run_times: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
96
|
- memory_usages: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
97
125
|
input_name: "Input",
|
|
98
126
|
input: "Input",
|
|
99
|
- run_time_statistics: %Benchee.Statistics{
|
|
100
|
- average: 500.0,
|
|
101
|
- ips: 2000_000.0,
|
|
102
|
- std_dev: 200.0,
|
|
103
|
- std_dev_ratio: 0.4,
|
|
104
|
- std_dev_ips: 800_000.0,
|
|
105
|
- median: 500.0,
|
|
106
|
- percentiles: %{50 => 500.0, 99 => 900.0},
|
|
107
|
- mode: [500, 400],
|
|
108
|
- minimum: 200,
|
|
109
|
- maximum: 900,
|
|
110
|
- sample_size: 9
|
|
127
|
+ run_time_data: %Benchee.CollectionData{
|
|
128
|
+ samples: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
129
|
+ statistics: %Benchee.Statistics{
|
|
130
|
+ average: 500.0,
|
|
131
|
+ ips: 2000_000.0,
|
|
132
|
+ std_dev: 200.0,
|
|
133
|
+ std_dev_ratio: 0.4,
|
|
134
|
+ std_dev_ips: 800_000.0,
|
|
135
|
+ median: 500.0,
|
|
136
|
+ percentiles: %{50 => 500.0, 99 => 900.0},
|
|
137
|
+ mode: [500, 400],
|
|
138
|
+ minimum: 200,
|
|
139
|
+ maximum: 900,
|
|
140
|
+ sample_size: 9
|
|
141
|
+ }
|
|
111
142
|
},
|
|
112
|
- memory_usage_statistics: %Benchee.Statistics{
|
|
113
|
- average: 500.0,
|
|
114
|
- ips: nil,
|
|
115
|
- std_dev: 200.0,
|
|
116
|
- std_dev_ratio: 0.4,
|
|
117
|
- std_dev_ips: nil,
|
|
118
|
- median: 500.0,
|
|
119
|
- percentiles: %{50 => 500.0, 99 => 900.0},
|
|
120
|
- mode: [500, 400],
|
|
121
|
- minimum: 200,
|
|
122
|
- maximum: 900,
|
|
123
|
- sample_size: 9
|
|
143
|
+ memory_usage_data: %Benchee.CollectionData{
|
|
144
|
+ samples: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
145
|
+ statistics: %Benchee.Statistics{
|
|
146
|
+ average: 500.0,
|
|
147
|
+ ips: nil,
|
|
148
|
+ std_dev: 200.0,
|
|
149
|
+ std_dev_ratio: 0.4,
|
|
150
|
+ std_dev_ips: nil,
|
|
151
|
+ median: 500.0,
|
|
152
|
+ percentiles: %{50 => 500.0, 99 => 900.0},
|
|
153
|
+ mode: [500, 400],
|
|
154
|
+ minimum: 200,
|
|
155
|
+ maximum: 900,
|
|
156
|
+ sample_size: 9
|
|
157
|
+ }
|
|
124
158
|
}
|
|
125
159
|
}
|
|
126
160
|
],
|
|
|
@@ -132,92 +166,42 @@ defmodule Benchee.Statistics do
|
|
132
166
|
def statistics(suite = %Suite{scenarios: scenarios}) do
|
|
133
167
|
config = suite.configuration
|
|
134
168
|
|
|
135
|
- percentiles = Enum.uniq([50 | config.percentiles])
|
|
169
|
+ scenarios_with_statistics = calculate_per_scenario_statistics(scenarios, config)
|
|
136
170
|
|
|
137
|
- scenarios_with_statistics =
|
|
138
|
- Parallel.map(scenarios, fn scenario ->
|
|
139
|
- run_time_stats = scenario.run_times |> job_statistics(percentiles) |> add_ips
|
|
140
|
- memory_stats = job_statistics(scenario.memory_usages, percentiles)
|
|
141
|
-
|
|
142
|
- %Scenario{
|
|
143
|
- scenario
|
|
144
|
- | run_time_statistics: run_time_stats,
|
|
145
|
- memory_usage_statistics: memory_stats
|
|
146
|
- }
|
|
147
|
- end)
|
|
148
|
-
|
|
149
|
- %Suite{suite | scenarios: sort(scenarios_with_statistics)}
|
|
171
|
+ %Suite{suite | scenarios: scenarios_with_statistics}
|
|
150
172
|
end
|
|
151
173
|
|
|
152
|
- @doc """
|
|
153
|
- Calculates statistical data based on a series of run times for a job
|
|
154
|
- in microseconds.
|
|
174
|
+ defp calculate_per_scenario_statistics(scenarios, config) do
|
|
175
|
+ percentiles = Enum.uniq([50 | config.percentiles])
|
|
155
176
|
|
|
156
|
- ## Examples
|
|
177
|
+ Parallel.map(scenarios, fn scenario ->
|
|
178
|
+ run_time_stats = scenario.run_time_data.samples |> job_statistics(percentiles) |> add_ips
|
|
179
|
+ memory_stats = job_statistics(scenario.memory_usage_data.samples, percentiles)
|
|
157
180
|
|
|
158
|
- iex> run_times = [200, 400, 400, 400, 500, 500, 500, 700, 900]
|
|
159
|
- iex> Benchee.Statistics.job_statistics(run_times, [50, 99])
|
|
160
|
- %Benchee.Statistics{
|
|
161
|
- average: 500.0,
|
|
162
|
- ips: nil,
|
|
163
|
- std_dev: 200.0,
|
|
164
|
- std_dev_ratio: 0.4,
|
|
165
|
- std_dev_ips: nil,
|
|
166
|
- median: 500.0,
|
|
167
|
- percentiles: %{50 => 500.0, 99 => 900.0},
|
|
168
|
- mode: [500, 400],
|
|
169
|
- minimum: 200,
|
|
170
|
- maximum: 900,
|
|
171
|
- sample_size: 9
|
|
181
|
+ %{
|
|
182
|
+ scenario
|
|
183
|
+ | run_time_data: %{scenario.run_time_data | statistics: run_time_stats},
|
|
184
|
+ memory_usage_data: %{scenario.memory_usage_data | statistics: memory_stats}
|
|
172
185
|
}
|
|
186
|
+ end)
|
|
187
|
+ end
|
|
173
188
|
|
|
174
|
- iex> Benchee.Statistics.job_statistics([100], [50, 99])
|
|
175
|
- %Benchee.Statistics{
|
|
176
|
- average: 100.0,
|
|
177
|
- ips: nil,
|
|
178
|
- std_dev: 0,
|
|
179
|
- std_dev_ratio: 0.0,
|
|
180
|
- std_dev_ips: nil,
|
|
181
|
- median: 100.0,
|
|
182
|
- percentiles: %{50 => 100.0, 99 => 100.0},
|
|
183
|
- mode: nil,
|
|
184
|
- minimum: 100,
|
|
185
|
- maximum: 100,
|
|
186
|
- sample_size: 1
|
|
187
|
- }
|
|
188
|
-
|
|
189
|
- iex> Benchee.Statistics.job_statistics([], [])
|
|
190
|
- %Benchee.Statistics{
|
|
191
|
- average: nil,
|
|
192
|
- ips: nil,
|
|
193
|
- std_dev: nil,
|
|
194
|
- std_dev_ratio: nil,
|
|
195
|
- std_dev_ips: nil,
|
|
196
|
- median: nil,
|
|
197
|
- percentiles: nil,
|
|
198
|
- mode: nil,
|
|
199
|
- minimum: nil,
|
|
200
|
- maximum: nil,
|
|
201
|
- sample_size: 0
|
|
202
|
- }
|
|
203
|
-
|
|
204
|
- """
|
|
205
189
|
@spec job_statistics(samples, list) :: __MODULE__.t()
|
|
206
|
- def job_statistics([], _) do
|
|
190
|
+ defp job_statistics([], _) do
|
|
207
191
|
%__MODULE__{sample_size: 0}
|
|
208
192
|
end
|
|
209
193
|
|
|
210
|
- def job_statistics(measurements, percentiles) do
|
|
211
|
- total = Enum.sum(measurements)
|
|
212
|
- num_iterations = length(measurements)
|
|
194
|
+ defp job_statistics(samples, percentiles) do
|
|
195
|
+ total = Enum.sum(samples)
|
|
196
|
+ num_iterations = length(samples)
|
|
213
197
|
average = total / num_iterations
|
|
214
|
- deviation = standard_deviation(measurements, average, num_iterations)
|
|
198
|
+ deviation = standard_deviation(samples, average, num_iterations)
|
|
215
199
|
standard_dev_ratio = if average == 0, do: 0, else: deviation / average
|
|
216
|
- percentiles = Percentile.percentiles(measurements, percentiles)
|
|
200
|
+ percentiles = Percentile.percentiles(samples, percentiles)
|
|
217
201
|
median = Map.fetch!(percentiles, 50)
|
|
218
|
- mode = Mode.mode(measurements)
|
|
219
|
- minimum = Enum.min(measurements)
|
|
220
|
- maximum = Enum.max(measurements)
|
|
202
|
+ mode = Mode.mode(samples)
|
|
203
|
+ minimum = Enum.min(samples)
|
|
204
|
+ maximum = Enum.max(samples)
|
|
221
205
|
|
|
222
206
|
%__MODULE__{
|
|
223
207
|
average: average,
|
|
|
@@ -257,80 +241,4 @@ defmodule Benchee.Statistics do
|
|
257
241
|
std_dev_ips: standard_dev_ips
|
|
258
242
|
}
|
|
259
243
|
end
|
|
260
|
-
|
|
261
|
- @doc """
|
|
262
|
- Calculate additional percentiles and add them to the `run_time_statistics`.
|
|
263
|
- Should only be used after `statistics/1`, to calculate extra values that
|
|
264
|
- may be needed for reporting.
|
|
265
|
-
|
|
266
|
- ## Examples
|
|
267
|
-
|
|
268
|
- iex> scenarios = [
|
|
269
|
- ...> %Benchee.Benchmark.Scenario{
|
|
270
|
- ...> job_name: "My Job",
|
|
271
|
- ...> run_times: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
272
|
- ...> memory_usages: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
273
|
- ...> input_name: "Input",
|
|
274
|
- ...> input: "Input"
|
|
275
|
- ...> }
|
|
276
|
- ...> ]
|
|
277
|
- iex> %Benchee.Suite{scenarios: scenarios}
|
|
278
|
- ...> |> Benchee.Statistics.statistics
|
|
279
|
- ...> |> Benchee.Statistics.add_percentiles([25, 75])
|
|
280
|
- %Benchee.Suite{
|
|
281
|
- scenarios: [
|
|
282
|
- %Benchee.Benchmark.Scenario{
|
|
283
|
- job_name: "My Job",
|
|
284
|
- run_times: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
285
|
- memory_usages: [200, 400, 400, 400, 500, 500, 500, 700, 900],
|
|
286
|
- input_name: "Input",
|
|
287
|
- input: "Input",
|
|
288
|
- run_time_statistics: %Benchee.Statistics{
|
|
289
|
- average: 500.0,
|
|
290
|
- ips: 2_000_000.0,
|
|
291
|
- std_dev: 200.0,
|
|
292
|
- std_dev_ratio: 0.4,
|
|
293
|
- std_dev_ips: 800_000.0,
|
|
294
|
- median: 500.0,
|
|
295
|
- percentiles: %{25 => 400.0, 50 => 500.0, 75 => 600.0, 99 => 900.0},
|
|
296
|
- mode: [500, 400],
|
|
297
|
- minimum: 200,
|
|
298
|
- maximum: 900,
|
|
299
|
- sample_size: 9
|
|
300
|
- },
|
|
301
|
- memory_usage_statistics: %Benchee.Statistics{
|
|
302
|
- average: 500.0,
|
|
303
|
- ips: nil,
|
|
304
|
- std_dev: 200.0,
|
|
305
|
- std_dev_ratio: 0.4,
|
|
306
|
- std_dev_ips: nil,
|
|
307
|
- median: 500.0,
|
|
308
|
- percentiles: %{50 => 500.0, 99 => 900.0},
|
|
309
|
- mode: [500, 400],
|
|
310
|
- minimum: 200,
|
|
311
|
- maximum: 900,
|
|
312
|
- sample_size: 9
|
|
313
|
- }
|
|
314
|
- }
|
|
315
|
- ]
|
|
316
|
- }
|
|
317
|
- """
|
|
318
|
- def add_percentiles(suite = %Suite{scenarios: scenarios}, percentile_ranks) do
|
|
319
|
- new_scenarios =
|
|
320
|
- Parallel.map(scenarios, fn scenario ->
|
|
321
|
- update_in(scenario.run_time_statistics.percentiles, fn existing ->
|
|
322
|
- new = Percentile.percentiles(scenario.run_times, percentile_ranks)
|
|
323
|
- Map.merge(existing, new)
|
|
324
|
- end)
|
|
325
|
- end)
|
|
326
|
-
|
|
327
|
- %Suite{suite | scenarios: new_scenarios}
|
|
328
|
- end
|
|
329
|
-
|
|
330
|
- @spec sort([Scenario.t()]) :: [Scenario.t()]
|
|
331
|
- defp sort(scenarios) do
|
|
332
|
- Enum.sort_by(scenarios, fn scenario ->
|
|
333
|
- {scenario.run_time_statistics.average, scenario.memory_usage_statistics.average}
|
|
334
|
- end)
|
|
335
|
- end
|
|
336
244
|
end
|
changed
lib/benchee/suite.ex
|
|
@@ -4,8 +4,8 @@ defmodule Benchee.Suite do
|
|
4
4
|
|
|
5
5
|
Different layers of the benchmarking rely on different data being present
|
|
6
6
|
here. For instance for `Benchee.Statistics.statistics/1` to work the
|
|
7
|
- `run_times` key needs to be filled with the results from
|
|
8
|
- `Benchee.Benchmark.measure/1`.
|
|
7
|
+ `run_time_data` key of each scenario needs to be filled with the samples
|
|
8
|
+ collected by `Benchee.Benchmark.collect/1`.
|
|
9
9
|
|
|
10
10
|
Formatters can then use the data to display all of the results and the
|
|
11
11
|
configuration.
|
|
|
@@ -16,12 +16,19 @@ defmodule Benchee.Suite do
|
|
16
16
|
scenarios: []
|
|
17
17
|
]
|
|
18
18
|
|
|
19
|
- @type key :: atom | String.t()
|
|
20
|
- @type optional_map :: map | nil
|
|
19
|
+ @typedoc """
|
|
20
|
+ Valid key for either input or benchmarking job names.
|
|
21
|
+ """
|
|
22
|
+ @type key :: String.t() | atom
|
|
23
|
+
|
|
24
|
+ @typedoc """
|
|
25
|
+ The main suite consisting of the configuration data, information about the system and most
|
|
26
|
+ importantly a list of `t:Benchee.Scenario.t/0`.
|
|
27
|
+ """
|
|
21
28
|
@type t :: %__MODULE__{
|
|
22
29
|
configuration: Benchee.Configuration.t() | nil,
|
|
23
|
- system: optional_map,
|
|
24
|
- scenarios: [] | [Benchee.Benchmark.Scenario.t()]
|
|
30
|
+ system: map | nil,
|
|
31
|
+ scenarios: [] | [Benchee.Scenario.t()]
|
|
25
32
|
}
|
|
26
33
|
end
|
changed
lib/benchee/system.ex
|
|
@@ -1,6 +1,10 @@
|
|
1
1
|
defmodule Benchee.System do
|
|
2
2
|
@moduledoc """
|
|
3
3
|
Provides information about the system the benchmarks are run on.
|
|
4
|
+
|
|
5
|
+ Includes information such as elixir/erlang version, OS, CPU and memory.
|
|
6
|
+
|
|
7
|
+ So far supports/should work for Linux, MacOS, FreeBSD and Windows.
|
|
4
8
|
"""
|
|
5
9
|
|
|
6
10
|
alias Benchee.Conversion.Memory
|
|
|
@@ -25,15 +29,9 @@ defmodule Benchee.System do
|
|
25
29
|
%Suite{suite | system: system_info}
|
|
26
30
|
end
|
|
27
31
|
|
|
28
|
- @doc """
|
|
29
|
- Returns current Elixir version in use.
|
|
30
|
- """
|
|
31
|
- def elixir, do: System.version()
|
|
32
|
+ defp elixir, do: System.version()
|
|
32
33
|
|
|
33
|
- @doc """
|
|
34
|
- Returns the current erlang/otp version in use.
|
|
35
|
- """
|
|
36
|
- def erlang do
|
|
34
|
+ defp erlang do
|
|
37
35
|
otp_release = :erlang.system_info(:otp_release)
|
|
38
36
|
file = Path.join([:code.root_dir(), "releases", otp_release, "OTP_VERSION"])
|
|
39
37
|
|
|
|
@@ -46,17 +44,11 @@ defmodule Benchee.System do
|
|
46
44
|
end
|
|
47
45
|
end
|
|
48
46
|
|
|
49
|
- @doc """
|
|
50
|
- Returns the number of cores available for the currently running VM.
|
|
51
|
- """
|
|
52
|
- def num_cores do
|
|
47
|
+ defp num_cores do
|
|
53
48
|
System.schedulers_online()
|
|
54
49
|
end
|
|
55
50
|
|
|
56
|
- @doc """
|
|
57
|
- Returns an atom representing the platform the VM is running on.
|
|
58
|
- """
|
|
59
|
- def os do
|
|
51
|
+ defp os do
|
|
60
52
|
{_, name} = :os.type()
|
|
61
53
|
os(name)
|
|
62
54
|
end
|
|
|
@@ -66,11 +58,7 @@ defmodule Benchee.System do
|
|
66
58
|
defp os(:freebsd), do: :FreeBSD
|
|
67
59
|
defp os(_), do: :Linux
|
|
68
60
|
|
|
69
|
- @doc """
|
|
70
|
- Returns a string with detailed information about the CPU the benchmarks are
|
|
71
|
- being performed on.
|
|
72
|
- """
|
|
73
|
- def cpu_speed, do: cpu_speed(os())
|
|
61
|
+ defp cpu_speed, do: cpu_speed(os())
|
|
74
62
|
|
|
75
63
|
defp cpu_speed(:Windows) do
|
|
76
64
|
parse_cpu_for(:Windows, system_cmd("WMIC", ["CPU", "GET", "NAME"]))
|
|
|
@@ -90,6 +78,7 @@ defmodule Benchee.System do
|
|
90
78
|
|
|
91
79
|
@linux_cpuinfo_regex ~r/model name.*:([\w \(\)\-\@\.]*)/i
|
|
92
80
|
|
|
81
|
+ @doc false
|
|
93
82
|
def parse_cpu_for(_, "N/A"), do: "N/A"
|
|
94
83
|
|
|
95
84
|
def parse_cpu_for(:Windows, raw_output) do
|
|
|
@@ -110,11 +99,7 @@ defmodule Benchee.System do
|
|
110
99
|
end
|
|
111
100
|
end
|
|
112
101
|
|
|
113
|
- @doc """
|
|
114
|
- Returns an integer with the total number of available memory on the machine
|
|
115
|
- running the benchmarks.
|
|
116
|
- """
|
|
117
|
- def available_memory, do: available_memory(os())
|
|
102
|
+ defp available_memory, do: available_memory(os())
|
|
118
103
|
|
|
119
104
|
defp available_memory(:Windows) do
|
|
120
105
|
parse_memory_for(
|
|
|
@@ -171,6 +156,7 @@ defmodule Benchee.System do
|
|
171
156
|
Memory.format(memory_in_bytes)
|
|
172
157
|
end
|
|
173
158
|
|
|
159
|
+ @doc false
|
|
174
160
|
def system_cmd(cmd, args, system_func \\ &System.cmd/2) do
|
|
175
161
|
{output, exit_code} = system_func.(cmd, args)
|
changed
lib/benchee/utility/file_creation.ex
|
|
@@ -1,15 +1,15 @@
|
|
1
1
|
defmodule Benchee.Utility.FileCreation do
|
|
2
2
|
@moduledoc """
|
|
3
|
- Methods to create files used in plugins.
|
|
3
|
+ Methods to easily handle file creation used in plugins.
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
alias Benchee.Benchmark
|
|
7
7
|
|
|
8
8
|
@doc """
|
|
9
|
- Open a file for write for all key/value pairs, interleaves the file name and
|
|
10
|
- calls function with file, content and filename.
|
|
9
|
+ Open a file for write for all key/value pairs, interleaves the file name with
|
|
10
|
+ the key and calls the given function with file, content and filename.
|
|
11
11
|
|
|
12
|
- Uses `Benchee.Utility.FileCreation.interlave/2` to get the base filename and
|
|
12
|
+ Uses `interleave/2` to get the base filename and
|
|
13
13
|
the given keys together to one nice file name, then creates these files and
|
|
14
14
|
calls the function with the file and the content from the given map so that
|
|
15
15
|
data can be written to the file.
|
|
|
@@ -22,7 +22,8 @@ defmodule Benchee.Utility.FileCreation do
|
|
22
22
|
the corresponding file
|
|
23
23
|
* filename - the base file name as desired by the user
|
|
24
24
|
* function - a function that is then called for every file with the associated
|
|
25
|
- file content from the map
|
|
25
|
+ file content from the map, defaults to just writing the file content via
|
|
26
|
+ `IO.write/2` and printing where it put the file.
|
|
26
27
|
|
|
27
28
|
## Examples
|
changed
lib/benchee/utility/parallel.ex
|
|
@@ -3,6 +3,9 @@ defmodule Benchee.Utility.Parallel do
|
|
3
3
|
|
|
4
4
|
@doc """
|
|
5
5
|
A utility function for mapping over an enumerable collection in parallel.
|
|
6
|
+
|
|
7
|
+ Take note that this spawns a process for every element in the collection
|
|
8
|
+ which is only advisable if the function does some heavy lifting.
|
|
6
9
|
"""
|
|
7
10
|
@spec map(Enum.t(), fun) :: list
|
|
8
11
|
def map(collection, func) do
|
changed
lib/benchee/utility/repeat_n.ex
|
|
@@ -2,7 +2,7 @@ defmodule Benchee.Utility.RepeatN do
|
|
2
2
|
@moduledoc false
|
|
3
3
|
|
|
4
4
|
@doc """
|
|
5
|
- Calls the given function n times.
|
|
5
|
+ Calls the given function n times.
|
|
6
6
|
"""
|
|
7
7
|
def repeat_n(_function, 0) do
|
|
8
8
|
# noop
|
changed
mix.exs
|
|
@@ -1,13 +1,13 @@
|
|
1
1
|
defmodule Benchee.Mixfile do
|
|
2
2
|
use Mix.Project
|
|
3
3
|
|
|
4
|
- @version "0.14.0"
|
|
4
|
+ @version "0.99.0"
|
|
5
5
|
|
|
6
6
|
def project do
|
|
7
7
|
[
|
|
8
8
|
app: :benchee,
|
|
9
9
|
version: @version,
|
|
10
|
- elixir: "~> 1.4",
|
|
10
|
+ elixir: "~> 1.6",
|
|
11
11
|
elixirc_paths: elixirc_paths(Mix.env()),
|
|
12
12
|
consolidate_protocols: true,
|
|
13
13
|
build_embedded: Mix.env() == :prod,
|
|
|
@@ -44,18 +44,18 @@ defmodule Benchee.Mixfile do
|
|
44
44
|
defp elixirc_paths(_), do: ["lib"]
|
|
45
45
|
|
|
46
46
|
def application do
|
|
47
|
- [applications: [:logger, :deep_merge]]
|
|
47
|
+ []
|
|
48
48
|
end
|
|
49
49
|
|
|
50
50
|
defp deps do
|
|
51
51
|
[
|
|
52
|
- {:deep_merge, "~> 0.1"},
|
|
52
|
+ {:deep_merge, "~> 1.0"},
|
|
53
53
|
{:ex_guard, "~> 1.3", only: :dev},
|
|
54
54
|
{:credo, "~> 1.0.0", only: :dev},
|
|
55
55
|
{:ex_doc, "~> 0.19.0", only: :dev},
|
|
56
56
|
{:earmark, "~> 1.0", only: :dev},
|
|
57
57
|
{:excoveralls, "~> 0.7", only: :test},
|
|
58
|
- {:inch_ex, "~> 0.5", only: :docs},
|
|
58
|
+ {:inch_ex, "~> 2.0", only: :docs},
|
|
59
59
|
{:dialyxir, "~> 1.0.0-rc.4", only: :dev, runtime: false}
|
|
60
60
|
]
|
|
61
61
|
end
|