run CI

.github/workflows/PerformanceCheck.yml

@@ -22,4 +22,9 @@ jobs:
  version: ${{ matrix.julia-version }}
  - name: Build project
  uses: julia-actions/julia-buildpkg@latest
- - run: julia --threads=auto --project=./benchmark/CI-scripts ./benchmark/CI-scripts/runtests.jl ${{ github.event.pull_request.base.ref }}
+ - name: Benchmark
+ continue-on-error: true
+ env:
+ GITHUB_TOKEN: ${{ secrets.BENCHMARK_KEY }}
+ PR_NUMBER: ${{ github.event.number }}
+ run: julia --threads=auto --project=./benchmark/CI-scripts ./benchmark/CI-scripts/runtests.jl ${{ github.event.pull_request.base.ref }}

benchmark/CI-scripts/Project.toml

@@ -1,6 +1,9 @@
 [deps]
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26"
+GitHubActions = "6b79fd1a-b13a-48ab-b6b0-aaee1fee41df"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
 Primes = "27ebfcd6-29c5-5fa9-bf4b-fb8fc14df3ae"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

benchmark/CI-scripts/run_benchmarks.jl

@@ -398,7 +398,7 @@ function dump_results(file, key)
  for problem in suite
  problem_name = problem.problem_name
  result = problem.result
- println(out, "$problem_name, $result")
+ println(out, "$problem_name: $result")
  end
  end
 end

benchmark/CI-scripts/runtests.jl

@@ -3,7 +3,7 @@ using Pkg
 Pkg.activate(@__DIR__)
 Pkg.instantiate()
 
-using ArgParse, GitHubActions, Random, Logging
+using ArgParse, GitHubActions, GitHub, Random, Logging
 using Test, TestSetExtensions, InteractiveUtils
 using Base.Threads
 
@@ -34,61 +34,161 @@ end
 
 # Compare results
 function compare()
- results_stable = nothing
+ results_master = nothing
  results_nightly = nothing
  try
- results_stable_file = open(dir_master * "/results", "r")
- results_stable = readlines(results_stable_file)
- close(results_stable_file)
+ results_master_file = open(dir_master * "/results", "r")
+ @info "Reading $results_master_file"
+ results_master = readlines(results_master_file)
+ close(results_master_file)
  catch e
- @warn "Error when reading the file with results, Groebner.jl master" commit
+ @warn "Error when reading the file with results"
  end
  try
  results_nightly_file = open(dir_nightly * "/results", "r")
+ @info "Reading $results_nightly_file"
  results_nightly = readlines(results_nightly_file)
  close(results_nightly_file)
  catch e
- @warn "Error when reading the file with results, Groebner.jl nightly"
+ @warn "Error when reading the file with results"
  end
- @assert !(results_stable === nothing) && !(results_nightly === nothing)
- @assert length(results_stable) == length(results_nightly)
- @assert !isempty(results_stable)
- @assert results_stable[1] == "stable" && results_nightly[1] == "nightly"
- results_stable = results_stable[2:end]
+ @assert !(results_master === nothing) && !(results_nightly === nothing)
+ @assert length(results_master) == length(results_nightly)
+ @assert !isempty(results_master)
+ @assert results_master[1] == "master" && results_nightly[1] == "nightly"
+ results_master = results_master[2:end]
  results_nightly = results_nightly[2:end]
- println("Groebner.jl, stable v nightly:")
- for (stable, nightly) in zip(results_stable, results_nightly)
- problem_name..., time_stable = split(stable, ",")
- problem_name_..., time_nightly = split(nightly, ",")
- @assert problem_name == problem_name_
- time_stable = parse(Float64, time_stable)
- time_nightly = parse(Float64, time_nightly)
- print(join(problem_name, ","), ":\t$time_stable v $time_nightly s =>\t")
- delta = time_nightly - time_stable
- if abs(delta) / time_stable > MAX_ACCEPTABLE_RELATIVE_DEVIATION
- if abs(delta) > IGNORE_SMALL_ABSOLUTE_DEVIATION
- if delta > 0
- printstyled("Regression\n", color=:red)
- else
- printstyled("Improvement\n", color=:green)
- end
- @test delta / time_stable < MAX_ACCEPTABLE_RELATIVE_DEVIATION
+ table = Matrix{Any}(undef, length(results_master), 4)
+ fail = false
+ for (i, (master, nightly)) in enumerate(zip(results_master, results_nightly))
+ problem_name_master, times_master = split(master, ":")
+ problem_name_nightly, times_nightly = split(nightly, ",")
+ @assert problem_name_master == problem_name_nightly
+ times_master = map(
+ x -> parse(Float64, String(strip(x, ['[', ']', ' ', '\t']))),
+ split(times_master, ",")
+ )
+ times_nightly = map(
+ x -> parse(Float64, String(strip(x, ['[', ']', ' ', '\t']))),
+ split(times_nightly, ",")
+ )
+ label_master = "$(mean(times_master)) ± $(std(time_master))"
+ label_nightly = "$(mean(times_nightly)) ± $(std(times_nightly))"
+ indicator =
+ if (1 + MAX_ACCEPTABLE_RELATIVE_DEVIATION) * mean(times_master) <
+ mean(times_nightly)
+ fail = true
+ 2, "**slower**❌"
+ elseif mean(times_master) >
+ (1 + MAX_ACCEPTABLE_RELATIVE_DEVIATION) * mean(times_nightly)
+ 1, "**faster**✅"
  else
- printstyled("Insignificant\n")
- @test true
+ 0, "insignificant"
  end
- else
- printstyled("Insignificant\n")
- @test true
- end
+ table[i, 1] = problem_name
+ table[i, 2] = label_master
+ table[i, 3] = label_nightly
+ table[i, 4] = indicator[2]
+ # delta = time_nightly - time_master
+ # if abs(delta) / time_master > MAX_ACCEPTABLE_RELATIVE_DEVIATION
+ # if abs(delta) > IGNORE_SMALL_ABSOLUTE_DEVIATION
+ # if delta > 0
+ # printstyled("Regression\n", color=:red)
+ # else
+ # printstyled("Improvement\n", color=:green)
+ # end
+ # @test delta / time_master < MAX_ACCEPTABLE_RELATIVE_DEVIATION
+ # else
+ # printstyled("Insignificant\n")
+ # @test true
+ # end
+ # else
+ # printstyled("Insignificant\n")
+ # @test true
+ # end
+ end
+ fail, table
+end
+
+# Adapted from https://github.com/MakieOrg/Makie.jl/blob/v0.21.0/metrics/ttfp/run-benchmark.jl.
+# License is MIT.
+function github_context()
+ owner = "sumiya11"
+ return (
+ owner=owner,
+ repo=GitHub.Repo("$(owner)/Groebner.jl"),
+ auth=GitHub.authenticate(ENV["GITHUB_TOKEN"])
+ )
+end
+
+function best_unit(m)
+ if m < 1e3
+ return 1, "ns"
+ elseif m < 1e6
+ return 1e3, "μs"
+ elseif m < 1e9
+ return 1e6, "ms"
+ else
+ return 1e9, "s"
+ end
+end
+
+function make_or_edit_comment(ctx, pr, benchmarks)
+ prev_comments, _ = GitHub.comments(ctx.repo, pr; auth=ctx.auth)
+ idx = findfirst(c -> c.user.login == "MakieBot", prev_comments)
+ if isnothing(idx)
+ comment = update_comment(COMMENT_TEMPLATE, package_name, benchmarks)
+ println(comment)
+ GitHub.create_comment(ctx.repo, pr; auth=ctx.auth, params=Dict("body" => comment))
+ else
+ old_comment = prev_comments[idx].body
+ comment = update_comment(old_comment, package_name, benchmarks)
+ println(comment)
+ GitHub.edit_comment(
+ ctx.repo,
+ prev_comments[idx],
+ :pr;
+ auth=ctx.auth,
+ params=Dict("body" => comment)
+ )
+ end
+end
+
+function post(fail, table)
+ header = """
+ ## Running times benchmark
+
+ Note, that these numbers may fluctuate on the CI servers, so take them with a grain of salt.
+ 
+ """
+ io = IOBuffer()
+ println(io, header)
+ if fail
+ println(io, "Potential regressions detected❌")
+ else
+ println(io, "No regressions detected✅")
+ end
+ pretty_table(io, table)
+ comment_str = String(take!(io))
+ println(comment_str)
+
+ # Post to github
+ ctx = github_context()
+ pr_to_comment = get(ENV, "PR_NUMBER", nothing)
+ if !isnothing(pr_to_comment)
+ pr = GitHub.pull_request(ctx.repo, pr_to_comment)
+ make_or_edit_comment(ctx, pr, table)
+ else
+ @info "Not commenting, no PR found"
+ println(update_comment(COMMENT_TEMPLATE, table))
  end
 end
 
 function main()
  runbench()
- @testset "Benchmark results" begin
-  compare()
- end
+ fail, table = compare()
+ @test !fail
+ post(fail, table)
  versioninfo(verbose=true)
 end