mxx::min_element and mxx::max_element

[tcpan]

vector version, even with small vectors of doubles, produced invalid read error under valgrind.

tested on compbio with nprocs = 4 and 128, can reproduce error. with nprocs = 2, did not observe error.

[1,4]<stderr>:==54670== Invalid read of size 8
[1,4]<stderr>:==54670==    at 0x4BE9C7: std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<doubl
e, int> const&, std::pair<double, int> const&)#1}::operator()(std::pair<double, int> const&, std::pair<double, int> const&) const (in /home/tpan7/build/kmerind-gcc-rel/bin/testFASTQ_load)
[1,4]<stderr>:==54670==    by 0x4E1393: void mxx::custom_op<std::pair<double, int>, true>::custom_function<std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::all
ocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}>(std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<doubl
e, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}, void*, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>
(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}, int*) (reduction.hpp:269)
[1,4]<stderr>:==54670==    by 0x4E465F: _ZNSt5_BindIFPFvZN3mxx11max_elementIdEESt6vectorISt4pairIT_iESaIS5_EERKS2_IS4_SaIS4_EERKNS0_4commEEUlRKS3_IdiESH_E_PvSJ_PiESI_St12_PlaceholderILi1EESN_ILi2EESN_ILi3EEEE6__callIvJOSJ_SU_OSK_EJLm0ELm1
ELm2ELm3EEEES4_OSt5tupleIJDpT0_EESt12_Index_tupleIJXspT1_EEE (functional:1074)
[1,4]<stderr>:==54670==    by 0x4E4090: _ZNSt5_BindIFPFvZN3mxx11max_elementIdEESt6vectorISt4pairIT_iESaIS5_EERKS2_IS4_SaIS4_EERKNS0_4commEEUlRKS3_IdiESH_E_PvSJ_PiESI_St12_PlaceholderILi1EESN_ILi2EESN_ILi3EEEEclIJSJ_SJ_SK_EvEET0_DpOT_ (fun
ctional:1133)
[1,4]<stderr>:==54670==    by 0x4E39EB: std::_Function_handler<void (void*, void*, int*), std::_Bind<void (*({lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}, std::_Placeholder<1>, void (*)(std::vector<std::pair<do
uble, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}, void*, void*,
 int*)<2>, void (*)(std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std:
:pair<double, int> const&)#1}, void*, void*, int*)<3>))(std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda
(std::pair<double, int> const&, std::pair<double, int> const&)#1}, void*, void*, int*)> >::_M_invoke(std::_Any_data const&, void*&&, std::_Any_data const&, int*&&) (functional:1871)
[1,4]<stderr>:==54670==    by 0x4C3DD5: std::function<void (void*, void*, int*)>::operator()(void*, void*, int*) const (functional:2271)
[1,4]<stderr>:==54670==    by 0x4E12FC: mxx::custom_op<std::pair<double, int>, true>::mpi_user_function(void*, void*, int*, ompi_datatype_t**) (reduction.hpp:278)
[1,4]<stderr>:==54670==    by 0xB15B1B5: ompi_coll_tuned_allreduce_intra_recursivedoubling (in /usr/local/modules/openmpi/1.10.2/lib/openmpi/mca_coll_tuned.so)
[1,4]<stderr>:==54670==    by 0x4E8840A: PMPI_Allreduce (in /usr/local/modules/openmpi/1.10.2/lib/libmpi.so.12.0.2)
[1,4]<stderr>:==54670==    by 0x4DB81D: void mxx::allreduce<std::pair<double, int>, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&
, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}>(std::pair<double, int> const*, unsigned long, std::pair<double, int>*, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int
> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}, mxx::comm const&) (reduction.hpp:362)
[1,4]<stderr>:==54670==    by 0x4D440B: std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::allreduce<std::pair<double, int>, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx
::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}>(std::pair<double, int> const*, unsigned long, std::vector<std::pair<do
uble, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}, mxx::comm con
st&) (reduction.hpp:373)
[1,4]<stderr>:==54670==    by 0x4C9A3D: std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::allreduce<std::pair<double, int>, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx
::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}>(std::vector<std::pair<double, int>, std::allocator<std::pair<double, i
nt> > > const&, std::vector<std::pair<double, int>[1,4]<stderr>:, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> co
nst&, std::pair<double, int> const&)#1}, mxx::comm const&) (reduction.hpp:384)
[1,4]<stderr>:==54670==  Address 0x935eea8 is 40 bytes inside a block of size 44 alloc'd
[1,4]<stderr>:==54670==    at 0x4A06A2E: malloc (vg_replace_malloc.c:270)
[1,4]<stderr>:==54670==    by 0xB15AD36: ompi_coll_tuned_allreduce_intra_recursivedoubling (in /usr/local/modules/openmpi/1.10.2/lib/openmpi/mca_coll_tuned.so)
[1,4]<stderr>:==54670==    by 0x4E8840A: PMPI_Allreduce (in /usr/local/modules/openmpi/1.10.2/lib/libmpi.so.12.0.2)
[1,4]<stderr>:==54670==    by 0x4DB81D: void mxx::allreduce<std::pair<double, int>, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&
, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}>(std::pair<double, int> const*, unsigned long, std::pair<double, int>*, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int
> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}, mxx::comm const&) (reduction.hpp:362)
[1,4]<stderr>:==54670==    by 0x4D440B: std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::allreduce<std::pair<double, int>, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx
::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}>(std::pair<double, int> const*, unsigned long, std::vector<std::pair<do
uble, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}, mxx::comm con
st&) (reduction.hpp:373)
[1,4]<stderr>:==54670==    by 0x4C9A3D: std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::allreduce<std::pair<double, int>, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx
::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pair<double, int> const&)#1}>(std::vector<std::pair<double, int>, std::allocator<std::pair<double, i
nt> > > const&, std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&)::{lambda(std::pair<double, int> const&, std::pai
r<double, int> const&)#1}, mxx::comm const&) (reduction.hpp:384)
[1,4]<stderr>:==54670==    by 0x4BEB75: std::vector<std::pair<double, int>, std::allocator<std::pair<double, int> > > mxx::max_element<double>(std::vector<double, std::allocator<double> > const&, mxx::comm const&) (reduction.hpp:561)
[1,4]<stderr>:==54670==    by 0x4AD145: unsigned long read_file<bliss::io::FASTQParser, false>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, mxx::comm const&) (BenchmarkFileLoader.cpp:146)
[1,4]<stderr>:==54670==    by 0x4A48C1: void testIndex<bliss::io::FASTQParser, false>(mxx::comm const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_tra
its<char>, std::allocator<char> >) (BenchmarkFileLoader.cpp:367)
[1,4]<stderr>:==54670==    by 0x49E08F: main (BenchmarkFileLoader.cpp:483)
[1,4]<stderr>:==54670== 

[patflick]

Turns out this is an issue inside OpenMPI: open-mpi/ompi#1462
We can't really do much, and it appears as if all current and older versions of OpenMPI are affected.

As a work-around, we might have to use a different custom data structure for which the extend is equal to the true_extend. This won't solve the underlying problem for custom reductions with lambda functions though, just avoid the problem within mxx::max_element()

[patflick]

@tcpan this should now be fixed