benchmarks

include/mxx/samplesort.hpp

@@ -47,8 +47,11 @@
 #endif
 
 
-#define SS_ENABLE_TIMER 0
-#if SS_ENABLE_TIMER
+#ifndef MXX_SAMPLESORT_TIMER
+#define MXX_SAMPLESORT_TIMER 0
+#endif
+
+#if MXX_SAMPLESORT_TIMER
 #include "timer.hpp"
 #define SS_TIMER_START(comm) mxx::section_timer timer(std::cerr, comm, 0);
 #define SS_TIMER_END_SECTION(str) timer.end_section(str);

include/mxx/stream.hpp

@@ -100,6 +100,14 @@ inline sync_ostream sync_cerr(const mxx::comm& comm, int root = 0) {
  return comm.rank() == root ? sync_ostream(comm, root, std::cerr) : sync_ostream(comm, root);
 }
 
+template <typename base_stream>
+inline sync_ostream sync_os(const mxx::comm& comm, base_stream& bs, int root = 0) {
+ return comm.rank() == root ? sync_ostream(comm, root, bs) : sync_ostream(comm, root);
+};
+
+// TODO: a sync ofstream
+// TODO: a sync cout/cerr which writes [rank] before every line/msg
+
 } // namespace mxx
 
 

src/CMakeLists.txt

@@ -1,10 +1,20 @@
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 2.8)
 
 # project settings
 project(mxx-bm)
 
 add_executable(mxx-bm-vote-off vote_off.cpp)
 target_link_libraries(mxx-bm-vote-off ${MPI_LIBRARIES})
 
-add_executable(mxx-benchmark benchmark_bw.cpp)
-target_link_libraries(mxx-benchmark ${MPI_LIBRARIES})
+# benchmark p2p bandwidth
+add_executable(mxx-benchmark-p2p-bw benchmark_p2p_bw.cpp)
+target_link_libraries(mxx-benchmark-p2p-bw ${MPI_LIBRARIES})
+
+# benchmark all2all
+add_executable(mxx-benchmark-a2a benchmark_a2a.cpp)
+target_link_libraries(mxx-benchmark-a2a ${MPI_LIBRARIES})
+
+# benchmark parallel sorting
+add_executable(mxx-sort-benchmark benchmark_sort.cpp)
+target_compile_definitions(mxx-sort-benchmark PUBLIC -DMXX_SAMPLESORT_TIMER=1)
+target_link_libraries(mxx-sort-benchmark ${MPI_LIBRARIES})

src/benchmark_a2a.cpp

@@ -0,0 +1,89 @@
+#include <string>
+#include <sstream>
+#include <fstream>
+
+#include <mxx/env.hpp>
+#include <mxx/comm.hpp>
+#include <mxx/benchmark.hpp>
+#include <mxx/utils.hpp>
+
+#include <ext/cxx-prettyprint/prettyprint.hpp>
+
+std::string exec_name;
+
+void print_usage() {
+ std::cerr << "Usage: " << exec_name << " -m <msg_size> <output-file>" << std::endl;
+ std::cerr << "where" << std::endl;
+ std::cerr << " <output-file> (optional) Filename for the benchmark results (default: 'all2all_benchmark.csv')." << std::endl;
+ std::cerr << " -m <max_mem> (optional) Maximum message space per node (in GB) (default: 32)." << std::endl;
+}
+
+int main(int argc, char* argv[]) {
+ mxx::env e(argc, argv);
+ mxx::comm comm;
+
+ // print out node and rank distribution
+ mxx::print_node_distribution(comm);
+
+ // create shared-mem MPI+MPI hybrid communicator
+ mxx::hybrid_comm hc(comm);
+
+ // assert same number processors per node
+ int proc_per_node = hc.local.size();
+ if (!mxx::all_same(proc_per_node, comm)) {
+ std::cerr << "Error: this benchmark assumes the same number of processors per node" << std::endl;
+ MPI_Abort(comm, -1);
+ }
+
+ // assert we have an even number of nodes
+ int num_nodes = hc.num_nodes();
+ if (num_nodes > 1 && num_nodes % 2 != 0) {
+ std::cerr << "Error: this benchmark assumes an even number of nodes" << std::endl;
+ MPI_Abort(comm, -1);
+ }
+
+ // default args
+ size_t mem_per_node_gb = 32; // setting the max experiment at 32 GB per node
+ std::string filename = "all2all_benchmark.csv";
+
+ // parse input arguments
+ exec_name = argv[0];
+ argv++; argc--;
+ if (argc >= 2) {
+ std::string x(argv[0]);
+ if (x == "-m") {
+ mem_per_node_gb = atoi(argv[1]);
+ argv+=2; argc-=2;
+ }
+ if (x != "-m" || mem_per_node_gb > 1024 || mem_per_node_gb == 0) {
+ print_usage();
+ MPI_Abort(comm, -1);
+ }
+ }
+ if (argc > 0) {
+ filename = argv[0];
+ argv++; argc--;
+ }
+ if (argc > 0) {
+ print_usage();
+ MPI_Abort(comm, -1);
+ }
+
+ MXX_ASSERT(mxx::all_same(mem_per_node_gb, comm));
+
+ // benchmark all:
+ std::ofstream of;
+ if (hc.global.rank() == 0) {
+ of.open(filename);
+ of << "p,nnodes,q,m,n,min,avg,max" << std::endl;
+ }
+
+ // 32 GB/node max?
+ size_t mempernode = mem_per_node_gb << 30;
+
+ mxx::forall_p2_nnodes_and_ppn(hc, [&](const mxx::hybrid_comm& hc){
+ bm_all2all(hc, of, mempernode);
+ });
+
+ return 0;
+}

src/benchmark_p2p_bw.cpp

@@ -0,0 +1,82 @@
+#include <string>
+#include <sstream>
+#include <fstream>
+
+#include <mxx/env.hpp>
+#include <mxx/comm.hpp>
+#include <mxx/benchmark.hpp>
+#include <mxx/utils.hpp>
+
+#include <ext/cxx-prettyprint/prettyprint.hpp>
+
+std::string exec_name;
+
+void print_usage() {
+ std::cerr << "Usage: " << exec_name << " -m <msg_size> <output-file>" << std::endl;
+ std::cerr << "where" << std::endl;
+ std::cerr << " <output-file> (optional) Filename for the pairwise bandwidth matrix (default: 'p2p_bw.csv')." << std::endl;
+ std::cerr << " -m <msg_size> (optional) Message size for each process pair in kilo bytes. (default: 131072 (128 MB))." << std::endl;
+}
+
+int main(int argc, char* argv[]) {
+ mxx::env e(argc, argv);
+ mxx::comm comm;
+
+ // print out node and rank distribution
+ mxx::print_node_distribution(comm);
+
+ // create shared-mem MPI+MPI hybrid communicator
+ mxx::hybrid_comm hc(comm);
+
+ // assert same number processors per node
+ int proc_per_node = hc.local.size();
+ if (!mxx::all_same(proc_per_node, comm)) {
+ std::cerr << "Error: this benchmark assumes the same number of processors per node" << std::endl;
+ MPI_Abort(comm, -1);
+ }
+
+ // assert we have an even number of nodes
+ int num_nodes = hc.num_nodes();
+ if (num_nodes > 1 && num_nodes % 2 != 0) {
+ std::cerr << "Error: this benchmark assumes an even number of nodes" << std::endl;
+ MPI_Abort(comm, -1);
+ }
+
+ // default args
+ size_t msg_size_kB = 128*1024; // 128 MiB per process default
+ std::string filename = "p2p_bw.csv";
+
+ // parse input arguments
+ exec_name = argv[0];
+ argv++; argc--;
+ if (argc >= 2) {
+ std::string x(argv[0]);
+ if (x == "-m") {
+ msg_size_kB = atoi(argv[1]);
+ argv+=2; argc-=2;
+ }
+ if (x != "-m" || msg_size_kB > 4*1024*1024 || msg_size_kB == 0) {
+ print_usage();
+ MPI_Abort(comm, -1);
+ }
+ }
+ if (argc > 0) {
+ filename = argv[0];
+ argv++; argc--;
+ }
+ if (argc > 0) {
+ print_usage();
+ MPI_Abort(comm, -1);
+ }
+
+ MXX_ASSERT(mxx::all_same(msg_size_kB, comm));
+
+ // perform pairwise bandwidth benchmarking
+ std::vector<double> bw_row = mxx::pairwise_bw_matrix(hc, msg_size_kB*1024);
+
+ // print out benchmarking results and save as file
+ mxx::print_bw_matrix_stats(hc, bw_row);
+ mxx::save_matrix_pernode(hc, filename, bw_row);
+
+ return 0;
+}

src/benchmark_sort.cpp

@@ -0,0 +1,54 @@
+#include <string>
+#include <sstream>
+#include <fstream>
+
+#include <mxx/env.hpp>
+#include <mxx/comm.hpp>
+#include <mxx/benchmark.hpp>
+#include <mxx/utils.hpp>
+
+#include <ext/cxx-prettyprint/prettyprint.hpp>
+
+std::string exec_name;
+
+// TODO fix usage
+void print_usage() {
+ std::cerr << "Usage: " << exec_name << " <n> <out-node-filename>" << std::endl;
+ std::cerr << "where" << std::endl;
+ std::cerr << " <n> Number of nodes to vote off." << std::endl;
+ std::cerr << " <out-node-filename> Filename for the new nodefile, output by this program." << std::endl;
+}
+
+int main(int argc, char* argv[]) {
+ mxx::env e(argc, argv);
+ mxx::comm comm;
+
+ // print out node and rank distribution
+ mxx::print_node_distribution(comm);
+
+ // create shared-mem MPI+MPI hybrid communicator
+ mxx::hybrid_comm hc(comm);
+
+ // create output file for benchmark
+ std::ofstream of;
+ if (hc.global.rank() == 0) {
+ of.open("bm_samplesort.csv");
+ of << "p,nnodes,q,m,n,min,avg,max" << std::endl;
+ }
+
+ size_t mempernode = 16ull << 30;
+
+ // input in growing sizes of 2
+ typedef std::tuple<size_t, size_t> T;
+ for (size_t npn = 1024; npn <= mempernode/sizeof(T)/2; npn <<= 1) {
+ // generate input
+ std::vector<T> a(npn);
+ srand(comm.rank()* 13 + 5);
+ std::generate(a.begin(), a.end(), [](){ return std::make_pair<size_t, size_t>(rand(), rand()); });
+
+ }
+
+ // TODO: sorting benchmark
+
+ return 0;
+}

src/pbs_run.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+#PBS -q swarm
+#PBS -l nodes=16:ppn=28
+#PBS -l walltime=2:00:00
+
+# set up env
+module load gcc/4.9.0
+module load mvapich2/2.2
+#module load openmpi
+
+
+# Change to directory from which qsub command was issued
+cd $PBS_O_WORKDIR
+
+# Old num nodes and PPN
+PPN=$PBS_NUM_PPN
+NUM_NODES=$PBS_NUM_NODES
+NP=$(expr $NUM_NODES \\* $PPN)
+
+echo "Running with np = $NP, ppn = $PPN, nnodes = $NUM_NODES"
+
+mpirun -np $NP -ppn $PPN ./bin/mxx-benchmark-p2p-bw p2p_bw_${NUM_NODES}nodes.csv
+mpirun -np $NP -ppn $PPN ./bin/mxx-benchmark-a2a bm_all2all_${NUM_NODES}nodes.csv

src/vote_off.cpp

@@ -48,7 +48,7 @@ int main(int argc, char* argv[]) {
 
  bool benchmark_char_align = false;
 
- std::vector<double> bw_row = mxx::pairwise_bw_matrix(hc);
+ std::vector<double> bw_row = mxx::pairwise_bw_matrix(hc, 32*1024*1024);
  mxx::print_bw_matrix_stats(hc, bw_row);
  bool part = mxx::vote_off(hc, n_vote_off, bw_row);
  if (hc.global.rank() == 0)

test/CMakeLists.txt

@@ -26,9 +26,6 @@ target_link_libraries(mxx-test-sort mxx-gtest-main)
 add_executable(mxx-test-distribution test_distribution.cpp)
 target_link_libraries(mxx-test-distribution mxx-gtest-main)
 
-add_executable(mxx-benchmarks benchmarks.cpp)
-target_link_libraries(mxx-benchmarks mxx-gtest-main)
-
 # Combination of all parallel tests:
 add_executable(mxx-test-all test_collective.cpp test_reductions.cpp test_send.cpp test_sort.cpp test_distribution.cpp)
 target_link_libraries(mxx-test-all mxx-gtest-main)