RFC: added performance tests for sparse getindex + a README file

test/perf/Makefile

@@ -1,9 +1,9 @@
 JULIAHOME = $(abspath ../..)
 include ../../Make.inc
 
-all: micro kernel cat shootout blas lapack sort spell
+all: micro kernel cat shootout blas lapack sort spell sparse
 
-micro kernel cat shootout blas lapack sort spell:
+micro kernel cat shootout blas lapack sort spell sparse:
  @$(MAKE) $(QUIET_MAKE) -C shootout
 ifneq ($(OS),WINNT)
  @$(call spawn,$(JULIA_EXECUTABLE)) $@/perf.jl | perl -nle '@_=split/,/; printf "%-18s %8.3f %8.3f %8.3f %8.3f\n", $$_[1], $$_[2], $$_[3], $$_[4], $$_[5]'
@@ -21,6 +21,7 @@ codespeed:
 # @$(call spawn,$(JULIA_EXECUTABLE)) lapack/perf.jl codespeed
 # @$(call spawn,$(JULIA_EXECUTABLE)) sort/perf.jl codespeed
  @$(call spawn,$(JULIA_EXECUTABLE)) spell/perf.jl codespeed
+ @$(call spawn,$(JULIA_EXECUTABLE)) sparse/perf.jl codespeed
  @$(call spawn,$(JULIA_EXECUTABLE)) report.jl
 
 
@@ -29,4 +30,4 @@ clean:
  $(MAKE) -C micro $@
  $(MAKE) -C shootout $@
 
-.PHONY: micro kernel cat shootout blas lapack sort spell clean
+.PHONY: micro kernel cat shootout blas lapack sort spell sparse clean

test/perf/README.md

@@ -0,0 +1,73 @@
+Julia performance monitoring
+============================
+
+This directory contains tests and related utilities to monitor Julia's
+performance over time. The results are presented on
+[https://speed.julialang.org/](https://speed.julialang.org/).
+
+Running the performance tests
+-----------------------------
+
+In `test/perf` run `make`. It will run the `perf.jl` script in all
+the sub-directories and display the test name with the minimum,
+maximum, mean and standard deviation of the wall-time of five repeated
+test runs in micro seconds.
+
+Calling `make codespeed` is for generating the results displayed on 
+[https://speed.julialang.org/](https://speed.julialang.org/), probably
+not what you want.
+
+There is also a `perfcomp.jl` script but it may not be working with
+the rest at the moment.
+
+Adding tests
+------------
+First decide whether the new tests should go into one of the existing
+suites:
+- `micro`: A set of micro-benchmarks commonly used to compare
+ programming languages; these results are shown on
+ [https://julialang.org/](https://julialang.org/).
+- `blas`, `lapack`: Performance tests for linear algebra tasks from
+ low-level operations such as matrix multiplies to higher-level
+ operations like eigenvalue problems.
+- `cat`: Performance tests for concatenation of vectors and matrices.
+- `kernel`: Performance tests used to track real-world code examples
+ that previously ran slowly.
+- `shootout` Tracks the performance of tests taken from the 
+ [Debian shootout](https://shootout.alioth.debian.org/) performance
+ tests.
+- `sort`: Performance tests of sorting algorithms.
+- `spell` Performance tests of 
+ [Peter Norvig's spelling corrector](https://norvig.com/spell-correct.html).
+- `sparse`: Performance tests of sparse matrix operations.
+
+Otherwise add a subdirectory containing the file `perf.jl` and
+update the `Makefile` as well.
+
+In `perf.jl`, `include("../perfutil.jl")` and then run the
+performance test functions with the `@timeit` macro. For example:
+```julia
+@timeit(spelltest(tests1), "spell", "Peter Norvig's spell corrector")
+```
+with arguments: test function call, name of the test, description,
+and, optionally, a group (only used for codespeed). `@timeit` will do
+a warm-up and then 5 timings. Alternatively `@timeit1` does one
+warm-up and one test run.
+
+If possible aim for the tests to take about 10-100 microseconds.
+
+Using the framework for your own tests
+--------------------------------------
+
+Just include `perfutil.jl`, use `@timeit` on the functions to be
+benchmarked. Alternatively have a look at the
+[Benchmark package](https://github.com/johnmyleswhite/Benchmark.jl).
+
+
+Package dependencies
+--------------------
+- HTTPClient
+- JSON
+- DataStructures
+- SortingAlgorithms
+

test/perf/sparse/perf.jl

@@ -0,0 +1,201 @@
+## Sparse matrix performance
+include("../perfutil.jl")
+
+## create some sparse matrices (need to be square):
+seed = 1
+srand(seed)
+
+small = 10^3
+med = 10^4
+large = 10^5
+huge = 10^6
+# # 1 entry per line
+# ss = {}
+# push!(ss, sprand(small, small, 1e-3))
+# #push!(ss, sprand(med, med, 1e-4))
+# push!(ss, sprand(large, large, 1e-5))
+
+# 10 entries per line
+ts = {}
+push!(ts, sprand(small, small, 1e-2))
+#push!(ts, sprand(med, med, 1e-3))
+push!(ts, sprand(large, large, 1e-4))
+
+# 100 entries per line
+us = {}
+push!(us, sprand(small, small, 1e-1))
+#push!(us, sprand(med, med, 1e-2))
+push!(us, sprand(large, large, 1e-3))
+#push!(us, sprand(huge, huge, 1e-4))
+
+# # 1000 entries per line
+# vs = {}
+# push!(vs, sprand(small, small, 1.0))
+# #push!(vs, sprand(med, med, 1e-1))
+# push!(vs, sprand(large, large, 1e-2))
+# #push!(vs, sprand(huge, huge, 1e-3))
+
+## using uint32 (works up to 10^9)
+uus = {}
+for u in us
+ push!(uus, SparseMatrixCSC(u.m, u.n, uint32(u.colptr), uint32(u.rowval), u.nzval))
+end 
+
+## getindex
+rep = 20
+reps = rep^2
+function integer_indexing(A)
+ # index with two random integers
+ nI, nJ = size(A)
+ rI = 1:nI
+ rJ = 1:nJ
+ tmp = zero(eltype(A))
+ for i in rand(rI, reps)
+ for j in rand(rJ, rep)
+ tmp += A[i,j]
+ end 
+ end
+ tmp
+end
+
+function row_indexing(A, rowinds) 
+ # index rows with rowinds and columns with a random integer
+ nI, nJ = size(A)
+ rI = 1:nI
+ rJ = 1:nJ
+ tmp = zero(eltype(A))
+ for j in rand(rJ, reps)
+ tmp += sum(A[rowinds,j])
+ end
+ tmp
+end
+
+function col_indexing(A, colinds) 
+ # index rows with a random integer and columns with colinds
+ nI, nJ = size(A)
+ rI = 1:nI
+ rJ = 1:nJ
+ tmp = zero(eltype(A))
+ for i in rand(rI, div(reps,10) )
+ tmp += sum(A[i,colinds])
+ end
+ tmp
+end
+
+function row_col_indexing(A, rowinds, colinds)
+ # index rows with rowinds and columns with colinds
+ # we need:
+ (maximum(rowinds)+rep < size(A,1) && maximum(colinds)+rep < size(A, 2)) || error("bad rowinds or colinds")
+ nI, nJ = size(A)
+ rI = 1:nI
+ rJ = 1:nJ
+ for i in 1:10
+ for j in 1:10
+ tmp2 = A[rowinds.+i, colinds.+j]
+ end
+ end
+end
+
+function one_arg_indexing(A, lininds)
+ # This is for 1d-indexing and indexing with one array of logicals.
+ # Both return a nx1 sparse matrix.
+ tmp = zero(eltype(A))
+ if isa(eltype(A), Bool)
+ tmp = sum(A[lininds])
+ else
+ for i in 1:rep
+ tmp += sum(A[lininds])
+ end
+ end
+ tmp
+end
+
+# test performance with matrices in us, uus and ts for
+# - integer indexing
+# - range indexing
+# - ordered array indexing
+# - disordered array indexing
+# - 1d indexing with integers and booleans
+
+# setup:
+# - indices
+intinds = nothing
+logicalinds = nothing # needs to be generated for a specific matrix size.
+rangeinds = 121:237
+orderedinds = [rangeinds]
+disorderedinds = orderedinds[randperm(length(orderedinds))]
+
+inds = [(intinds, "integers"), (logicalinds, "logical array"), (rangeinds, "a range"),
+ (orderedinds, "a ordered array"), (disorderedinds, "a disordered array")]
+
+# - matrix sizes
+sizes = [(1, "small", "Small sparse matrix"), (2, "medium", "Medium sparse matrix")]
+
+# - matrix types
+mattyp = [(ts, "10 entries/column"), (us, "100 entries/column")]
+# change to following line, after regression is fixed: https://github.com/JuliaLang/julia/pull/7162#issuecomment-45400517
+#mattyp = [(ts, "10 entries/column"), (us, "100 entries/column"), (uus, "100 entries/column uint32")]
+
+# - functions
+funs = [(integer_indexing, 1, "indexing"), (one_arg_indexing, 1, "1d indexing"), 
+ (row_indexing, 2, "indexing rows"), (col_indexing, 2, "indexing rows"), 
+ (row_col_indexing, 3, "indexing rows & columns")]
+
+# performance tests:
+counters = [1,1] # for small and medium matrix
+# integer indexing
+for (sz,s1,s2) in sizes # size of the matrix
+ for (mt, ms) in mattyp # type of the matrix
+ m = mt[sz]
+ c = counters[sz]
+ @timeit integer_indexing(m) "sparse_getindex_$s1$c" "$s2 with $ms, indexing with integers"
+ counters[sz] += 1
+ end
+end
+
+# range & array indexing
+for (sz,s1,s2) in sizes # size of the matrix
+ for (mt, ms) in mattyp # type of the matrix
+ m = mt[sz]
+ for (fun, nargs, funstr) in funs[3:5] # indexing test function
+ for (ind,indstr) in inds[3:5] # type of indices
+ c = counters[sz]
+ if indstr=="logical array"
+ # make a logical array of the right size
+ ind = sprandbool(size(m,1)..., 1e-5)
+ end
+ if nargs==2
+ @timeit fun(m, ind) "sparse_getindex_$s1$c" "Sparse matrix with $ms, $funstr with $indstr"
+ elseif nargs==3
+ @timeit fun(m, ind, ind) "sparse_getindex_$s1$c" "Sparse matrix with $ms, $funstr with $indstr"
+ else
+ error("Something is amiss here.")
+ end
+ counters[sz] += 1
+ end
+ end
+ end
+end
+
+# linear indexing
+for (sz,s1,s2) in sizes # size of the matrix
+ for (mt, ms) in mattyp # type of the matrix
+ m = mt[sz]
+ for (ind,indstr) in inds[2:5] # type of indices
+ if indstr=="logical array"
+ if sz==2
+ continue # logical indexing with medium size sparse matrix takes too long
+ end
+ # make a logical array of the right size
+ ind = sprandbool(size(m)..., 1e-5)
+ c = counters[sz]
+ @timeit1 one_arg_indexing(m, ind) "sparse_getindex_$s1$c" "$s2 with $ms, linear indexing with $indstr"
+ counters[sz] += 1
+ else
+ c = counters[sz]
+ @timeit one_arg_indexing(m, ind) "sparse_getindex_$s1$c" "$s2 with $ms, linear indexing with $indstr"
+ counters[sz] += 1
+ end
+ end
+ end
+end