Refactor and consolidate the performance tests

Run perf tests by running make in test/perf
Factor out timing code into test/perf/perfutil.jl
Micro benchmarks are now in test/perf/micro/
perf2 benchmarks are now in test/perf/kernel
shootout benchmarks now run (not all yet) as part of the perf tests (JuliaLang#660)
cat benchmarks now run as part of the perf tests

test/Makefile

@@ -2,7 +2,7 @@ JULIAHOME = $(abspath ..)
 include ../Make.inc
 
 TESTS = all core keywordargs numbers strings unicode collections hashing \
- remote iostring arrayops linalg blas fft dsp sparse bitarray random  \
+ remote iostring arrayops linalg blas fft dsp sparse bitarray random \
  math functional bigint sorting statistics spawn parallel arpack file \
  git pkg pkg2 resolve resolve2 suitesparse complex version pollfd mpfr \
  broadcast socket floatapprox priorityqueue readdlm regex
@@ -11,20 +11,13 @@ $(TESTS) ::
  $(QUIET_JULIA) $(call spawn,$(JULIA_EXECUTABLE)) ./runtests.jl $@
 
 perf:
-ifneq ($(MAKECMDGOALS),perf)
- $(QUIET_JULIA) $(call spawn,$(JULIA_EXECUTABLE)) perf/[email protected]
-else
- @$(call spawn,$(JULIA_EXECUTABLE)) perf/[email protected] | perl -nle '@_=split/,/; printf "%-14s %7.3f\n", $$_[1], $$_[2]'
-endif
-
-benchmark:
- @$(MAKE) -C perf $@
+ @$(MAKE) -C perf all
 
 clean:
  @$(MAKE) -C perf $@
  -rm -f libccalltest.${SHLIB_EXT} ccalltest
 
-.PHONY: $(TESTS) perf benchmark clean
+.PHONY: $(TESTS) perf clean
 
 libccalltest.$(SHLIB_EXT): ccalltest.c
  $(CC) $(CFLAGS) $(DEBUGFLAGS) -O3 $< -fPIC -shared -o $@ $(LDFLAGS) -DCC=$(CC)

test/perf/Makefile

@@ -1,94 +1,13 @@
-JULIAHOME = ../..
-include $(JULIAHOME)/Make.inc
-include $(JULIAHOME)/deps/Versions.make
+JULIAHOME = $(abspath ../..)
+include ../../Make.inc
 
-ifeq ($(OS), WINNT)
-MATHEMATICABIN = MathKernel
-else ifeq ($(OS), Darwin)
-MATHEMATICABIN = MathKernel
-else
-MATHEMATICABIN = math
-endif
+all: micro kernel cat shootout
 
-default: benchmarks.html
-
-export OMP_NUM_THREADS=1
-export GOTO_NUM_THREADS=1
-export OPENBLAS_NUM_THREADS=1
-
-perf.h: $(JULIAHOME)/deps/Versions.make
- echo '#include "$(JULIAHOME)/deps/openblas-$(OPENBLAS_VER)/cblas.h"' > $@
- echo '#include "$(JULIAHOME)/deps/random/dsfmt-$(DSFMT_VER)/dSFMT.c"' >> $@
-
-bin/perf%: perf.c perf.h
- $(CC) -std=c99 -O$* $< -o $@ $(JULIAHOME)/deps/openblas-$(OPENBLAS_VER)/libopenblas.a -lpthread
-
-bin/fperf%: perf.f90
- $(FC) -static-libgfortran -O$* -fexternal-blas $< -o $@ $(JULIAHOME)/deps/openblas-$(OPENBLAS_VER)/libopenblas.a -lpthread
-
-benchmarks/c.csv: \
- benchmarks/c0.csv \
- benchmarks/c1.csv \
- benchmarks/c2.csv \
- benchmarks/c3.csv
- cat $^ > $@
-
-benchmarks/fortran.csv: \
- benchmarks/fortran0.csv \
- benchmarks/fortran1.csv \
- benchmarks/fortran2.csv \
- benchmarks/fortran3.csv
- cat $^ > $@
-
-benchmarks/c%.csv: bin/perf%
- for t in 1 2 3 4 5; do $<; done >$@
-
-benchmarks/fortran%.csv: bin/fperf%
- for t in 1 2 3 4 5; do $<; done >$@
-
-benchmarks/go.csv: perf.go
- for t in 1 2 3 4 5; do go run $<; done >$@
-
-benchmarks/julia.csv: perf.jl
- for t in 1 2 3 4 5; do ../../julia $<; done >$@
-
-benchmarks/python.csv: perf.py
- for t in 1 2 3 4 5; do python $<; done >$@
-
-benchmarks/matlab.csv: perf.m
- for t in 1 2 3 4 5; do matlab -nosplash -nodesktop -nojvm -r 'perf;exit' 2>/dev/null | grep '^matlab,'; done >$@
-
-benchmarks/octave.csv: perf.m
- for t in 1 2 3 4 5; do octave -q --eval perf 2>/dev/null; done >$@
-
-benchmarks/r.csv: perf.R
- for t in 1 2 3 4 5; do cat $< | R --vanilla --slave 2>/dev/null; done >$@
-
-benchmarks/javascript.csv: perf.js
- for t in 1 2 3 4 5; do node $<; done >$@
-
-benchmarks/mathematica.csv: perf.nb
- for t in 1 2 3 4 5; do $(MATHEMATICABIN) -noprompt -run "<<$<; Exit[]"; done >$@
-
-BENCHMARKS = \
- benchmarks/c.csv \
- benchmarks/fortran.csv \
- benchmarks/go.csv \
- benchmarks/julia.csv \
- benchmarks/python.csv \
- benchmarks/matlab.csv \
- benchmarks/octave.csv \
- benchmarks/r.csv \
- benchmarks/javascript.csv \
- benchmarks/mathematica.csv
-
-benchmarks.csv: bin/collect.pl $(BENCHMARKS)
- $(QUIET_PERL) $^ >$@
-
-benchmarks.html: bin/table.pl benchmarks.csv
- $(QUIET_PERL) $^ >$@
+micro kernel cat shootout:
+ @$(call spawn,$(JULIA_EXECUTABLE)) $@/perf.jl | perl -nle '@_=split/,/; printf "%-18s %8.3f\n", $$_[1], $$_[2]'
 
 clean:
- @rm -rf perf.h bin/perf* bin/fperf* benchmarks/*.csv benchmarks.csv *.mod
+ rm -f *~
+ $(MAKE) -C micro $@
 
-.PHONY: all perf clean
+.PHONY: micro kernel cat shootout clean

test/perf_cat.jl → test/perf/cat/perf.jl

@@ -1,3 +1,5 @@
+include("../perfutil.jl")
+
 function cat2d_perf(n, iter)
  a = rand(n,n)
  b = rand(n,n)
@@ -10,7 +12,7 @@ function cat2d_perf2(n, iter)
  a = rand(n,n)
  b = rand(n,n)
  for i=1:iter
- c = Array(Float64, 2n, 2n)
+ c = Array(Float64,2n,2n)
  c[1:n,1:n] = a
  c[1:n,n+1:end] = b
  c[n+1:end,1:n] = b
@@ -19,11 +21,11 @@ function cat2d_perf2(n, iter)
  end
 end
 
-@timeit cat2d_perf(5, 20000) "small cat"
-@timeit cat2d_perf2(5, 20000) "small cat 2"
+@timeit cat2d_perf(5,20000) "small_cat"
+@timeit cat2d_perf2(5,20000) "small_cat_2"
 
-@timeit cat2d_perf(500, 2) "large cat"
-@timeit cat2d_perf2(500, 2) "large cat 2"
+@timeit cat2d_perf(500,2) "large_cat"
+@timeit cat2d_perf2(500,2) "large_cat_2"
 
 function hcat_perf(n, iter)
  a = rand(n,n)
@@ -45,11 +47,11 @@ function hcat_perf2(n, iter)
  end
 end
 
-@timeit hcat_perf(5, 20000) "small hcat"
-@timeit hcat_perf2(5, 20000) "small hcat 2"
+@timeit hcat_perf(5,20000) "small_hcat"
+@timeit hcat_perf2(5,20000) "small_hcat_2"
 
-@timeit hcat_perf(500, 2) "large hcat"
-@timeit hcat_perf2(500, 2) "large hcat 2"
+@timeit hcat_perf(500,2) "large_hcat"
+@timeit hcat_perf2(500,2) "large_hcat_2"
 
 function vcat_perf(n, iter)
  a = rand(n,n)
@@ -71,11 +73,11 @@ function vcat_perf2(n, iter)
  end
 end
 
-@timeit vcat_perf(5, 20000) "small vcat"
-@timeit vcat_perf2(5, 20000) "small vcat 2"
+@timeit vcat_perf(5,20000) "small_vcat"
+@timeit vcat_perf2(5,20000) "small_vcat_2"
 
-@timeit vcat_perf(500, 2) "large vcat"
-@timeit vcat_perf2(500, 2) "large vcat 2"
+@timeit vcat_perf(500,2) "large_vcat"
+@timeit vcat_perf2(500,2) "large_vcat_2"
 
 function catnd_perf(n, iter)
  a = rand(1,n,n,1)
@@ -97,8 +99,8 @@ function catnd_perf2(n, iter)
  end
 end
 
-@timeit catnd_perf(5, 20000) "small catnd"
-@timeit catnd_perf2(5, 20000) "small catnd 2"
+@timeit catnd_perf(5,20000) "small_catnd"
+@timeit catnd_perf2(5,20000) "small_catnd_2"
 
-@timeit catnd_perf(500, 2) "large catnd"
-@timeit catnd_perf2(500, 2) "large catnd 2"
+@timeit catnd_perf(500,2) "large_catnd"
+@timeit catnd_perf2(500,2) "large_catnd_2"