Skip to content

Commit

Permalink
std bench improvement proposal (#873)
Browse files Browse the repository at this point in the history
- removed usage of a new external command `math significant-digits`
- rounded precision to 4th digit including (max relative error of 0.05%)
- in `--pretty` I replaced abs `stddev` with relative `coefficient of
variation` (CV)

@amtoine, would you please take a look into my new variant once again? 
Maybe this time it will be good enough? 

<img width="804" alt="image"
src="https://github.com/nushell/nu_scripts/assets/4896754/49e62609-4301-4d5c-b1f5-d2c3355cb33e">
  • Loading branch information
maxim-uvarov committed Jun 8, 2024
1 parent d95857c commit 398cc82
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 31 deletions.
54 changes: 25 additions & 29 deletions stdlib-candidate/std-rfc/bench.nu
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
# A proposal for improving the original `std bench` command by @amtoine
# https://github.com/nushell/nushell/blob/31f3d2f6642b585f0d88192724723bf0ce330ecf/crates/nu-std/std/mod.nu#L134

use ./math

# convert an integer amount of nanoseconds to a real duration
def "from ns" [
--sign-digits: int = 4 # a number of first non-zero digits to keep (default 4; set 0 to disable rounding)
] {
if $sign_digits == 0 {} else {
math significant-digits $sign_digits
# round an integer amount of nanoseconds to 4th digit and convert to a real duration
def "from ns" [] {
if $in == 0 {0} else { #math log errors on 0
math round -p (3 - ($in | math log 10 | math floor)) # rounds to 4th digit including, with maximum realtive err 0.05%
| math round # second math round as a fix for `> 123456 | math round -p -5` = 99999.99999999999
}
| $"($in)ns"
| into duration
}

Expand Down Expand Up @@ -42,26 +38,26 @@ def "from ns" [
#
# get a pretty benchmark report
# > bench {1 + 2} --pretty
# 922ns +/- 2µs 40ns
# 716ns ± 76.67%
#
# measure the performance of simple addition with 1ms delay and output each timing
# > bench {sleep 1ms; 1 + 2} --rounds 2 --list-timings | table -e
# ╭───────┬─────────────────────
# │ mean │ 1ms 272µs
# │ min │ 1ms 259µs
# │ max │ 1ms 285µs
# │ std │ 13µs 370ns
# │ │ ╭─────────────────╮ │
# │ times │ │ 1ms 285µs 791ns │ │
# │ │ │ 1ms 259µs 42ns │ │
# │ │ ╰─────────────────╯ │
# ╰───────┴─────────────────────
# > bench {sleep 1ms; 1 + 2} --rounds 2 --timings | table -e
# ╭───────┬───────────────╮
# │ mean │ 1ms 272µs │
# │ min │ 1ms 259µs │
# │ max │ 1ms 285µs │
# │ std │ 13µs 370ns │
# │ │ ╭───────────╮ │
# │ times │ │ 1ms 285µs │ │
# │ │ │ 1ms 259µs │ │
# │ │ ╰───────────╯ │
# ╰───────┴───────────────╯
export def main [
code: closure # the piece of `nushell` code to measure the performance of
--rounds (-n): int = 50 # the number of benchmark rounds (hopefully the more rounds the less variance)
--verbose (-v) # be more verbose (namely prints the progress)
--pretty # shows the results in human-readable format: "<mean> +/- <stddev>"
--list-timings # list all rounds' timings in a `times` field
--timings (-t) # list all rounds' timings in a `times` field
] {
let times = seq 1 $rounds
| each {|i|
Expand All @@ -72,16 +68,16 @@ export def main [
if $verbose { print $"($rounds) / ($rounds)" }

{
mean: ($times | math avg | from ns --sign-digits 4)
min: ($times | math min | from ns --sign-digits 4)
max: ($times | math max | from ns --sign-digits 4)
std: ($times | math stddev | from ns --sign-digits 4)
mean: ($times | math avg | from ns)
min: ($times | math min | from ns)
max: ($times | math max | from ns)
std: ($times | math stddev | from ns)
}
| if $pretty {
$"($in.mean) +/- ($in.std)"
$"($in.mean) ± (($in.std / $in.mean) * 100 | math round -p 2)%"
} else {
if $list_timings {
merge { times: ($times | each { from ns --sign-digits 0 }) }
if $timings {
merge { times: ($times | each { from ns }) }
} else {}
}
}
4 changes: 2 additions & 2 deletions stdlib-candidate/tests/bench.nu
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ use std assert
use ../std-rfc bench

export def "test bench-timings" [] {
let $test = bench {1 + 2} --rounds 3 --list-timings | get times | length
let $test = bench {1 + 2} --rounds 3 --timings | get times | length
assert equal $test 3
}

export def "test bench-pretty" [] {
let $test = (bench {1 + 2} --rounds 3 --pretty) =~ '\d.* \+/- \d'
let $test = (bench {1 + 2} --rounds 3 --pretty) =~ '\d.* ± \d'
assert equal $test true
}

0 comments on commit 398cc82

Please sign in to comment.