Merge pull request #384 from pachterlab/devel

merging from devel

.gitignore

@@ -20,4 +20,14 @@ release/
 .vscode/
 ext/htslib/src/htslib-stamp
 ext/htslib/tmp/
+ext/bifrost/tmp/
+ext/bifrost/build/
+ext/bifrost/src/bifrost-stamp/
 .snakemake
+*.swp
+*.swo
+
+ext/zlib-ng/src/
+ext/zlib-ng/build/
+ext/zlib-ng/tmp/
+src/.Rhistory

CMakeLists.txt

@@ -1,16 +1,21 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 3.0.0)
 
 project(kallisto)
 
 include(GNUInstallDirs)
 
 
 option(USE_HDF5 "Compile with HDF5 support" OFF) #OFF by default
+option(USE_BAM "Compile with HTSLIB support" OFF) # OFF by default
 
 if(USE_HDF5)
  add_compile_definitions("USE_HDF5=ON")
 endif(USE_HDF5)
 
+if(NOT USE_BAM)
+ add_compile_definitions("NO_HTSLIB=ON")
+endif()
+
 set(EXT_PROJECTS_DIR ${PROJECT_SOURCE_DIR}/ext)
 set(CMAKE_CXX_FLAGS_PROFILE "-g")
 
@@ -28,12 +33,20 @@ if(${CMAKE_VERSION} VERSION_LESS 3.1)
  # remove this block once CMake >=3.1 has fixated in the ecosystem
  add_compile_options(-std=c++11)
 else()
+ include(CheckCXXCompilerFlag)
+ check_cxx_compiler_flag(-std=c++17 COMPILER_SUPPORTS_CXX17)
+ if(COMPILER_SUPPORTS_CXX17)
+ set(CMAKE_CXX_STANDARD 17)
+ else()
  set(CMAKE_CXX_STANDARD 11)
+ endif()
  set(CMAKE_CXX_STANDARD_REQUIRED ON)
  set(CMAKE_CXX_EXTENSIONS OFF)
 endif()
 
 #add_compile_options(-Wall -Wno-unused-function)
+add_compile_options(-Wno-subobject-linkage) # Suppress bifrost warning
+set(PROJECT_BIFROST_CMAKE_CXX_FLAGS "-Wno-subobject-linkage -Wno-return-type") # Suppress bifrost warning
 
 if(LINK MATCHES static)
  message("static build")
@@ -43,17 +56,46 @@ ENDIF(LINK MATCHES static)
 
 
 include(ExternalProject)
+if (USE_BAM)
 ExternalProject_Add(htslib
  PREFIX ${PROJECT_SOURCE_DIR}/ext/htslib
  SOURCE_DIR ${PROJECT_SOURCE_DIR}/ext/htslib
  BUILD_IN_SOURCE 1
- CONFIGURE_COMMAND autoheader && autoconf && ${PROJECT_SOURCE_DIR}/ext/htslib/configure 
+ CONFIGURE_COMMAND autoreconf -i && autoheader &&  autoconf && ${PROJECT_SOURCE_DIR}/ext/htslib/configure
  --prefix=${PREFIX} --disable-bz2 --disable-lzma --disable-libcurl
  BUILD_COMMAND make lib-static
  INSTALL_COMMAND ""
 )
+endif(USE_BAM)
+
+ExternalProject_Add(bifrost
+ PREFIX ${PROJECT_SOURCE_DIR}/ext/bifrost
+ SOURCE_DIR ${PROJECT_SOURCE_DIR}/ext/bifrost
+ BUILD_IN_SOURCE 1
+ CONFIGURE_COMMAND mkdir -p build && cd build && cmake .. -DCMAKE_INSTALL_PREFIX=${PREFIX} -DCMAKE_CXX_FLAGS=${PROJECT_BIFROST_CMAKE_CXX_FLAGS}
+ BUILD_COMMAND cd build && make
+ INSTALL_COMMAND ""
+)
 
+if (ZLIBNG)
+ message("zlib-ng enabled.")
+ ExternalProject_Add(zlib-ng
+ PREFIX ${PROJECT_SOURCE_DIR}/ext/zlib-ng
+ SOURCE_DIR ${PROJECT_SOURCE_DIR}/ext/zlib-ng
+ BUILD_IN_SOURCE 1
+ CONFIGURE_COMMAND mkdir -p zlib-ng && cd zlib-ng && cmake .. -DZLIB_COMPAT=ON -DZLIB_ENABLE_TESTS=OFF -DCMAKE_INSTALL_PREFIX=${PREFIX}
+ BUILD_COMMAND cd zlib-ng && make
+ INSTALL_COMMAND ""
+ )
+endif(ZLIBNG)
+
+if (USE_BAM)
 include_directories(${htslib_PREFIX}/src/htslib)
+endif(USE_BAM)
+include_directories(${EXT_PROJECTS_DIR}/bifrost/build/src)
+
+ExternalProject_Get_Property(bifrost install_dir)
+include_directories(${install_dir}/src)
 
 
 

ext/bifrost/.gitignore

@@ -0,0 +1,42 @@
+# Compiled Object files
+*.slo
+*.lo
+*.o
+
+# Compiled Dynamic libraries
+*.so
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+
+# From SWIG
+*_wrap.cxx
+graph.py
+
+# Other
+*.pyc
+*.dot
+*.contigs
+*.graph
+*.txt
+*.out
+*.swp
+*.bf
+*~
+\#*
+.\#*
+build/
+CB_Project/
+old/
+dist/
+debug/
+*.dSYM/*
+prof
+BFGraph
+Naive
+data/
+core
+man/
+example/output/

ext/bifrost/CMakeLists.txt

@@ -0,0 +1,69 @@
+cmake_minimum_required(VERSION 3.0.0)
+
+project(Bifrost)
+
+find_package(Threads REQUIRED)
+
+# To enable a larger default k-mer size, replace MAX_KMER_SIZE with a larger multiple of 32: actual maximum k-mer size will be MAX_KMER_SIZE-1.
+SET(MAX_KMER_SIZE "32" CACHE STRING "MAX_KMER_SIZE")
+SET(MAX_GMER_SIZE "${MAX_KMER_SIZE}" CACHE STRING "MAX_GMER_SIZE")
+# Enable architecture optimizations
+SET(COMPILATION_ARCH "native" CACHE STRING "COMPILATION_ARCH")
+# Enable AVX2 instructions
+SET(ENABLE_AVX2 "ON" CACHE STRING "ENABLE_AVX2")
+
+# Set some default compile flags
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+
+set_property(SOURCE BlockedBloomFilter.cpp APPEND_STRING PROPERTY COMPILE_FLAGS " -funroll-loops")
+
+
+#check if we are on arm64 and apple, if so, disable AVX2
+if(APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm")
+ message("Disabling AVX2 instructions on arm64")
+ set(ENABLE_AVX2 "OFF")
+ set(COMPILATION_ARCH "OFF")
+endif(APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm")
+
+if(COMPILATION_ARCH MATCHES "OFF")
+ message("Disabling native architecture compilation (including AVX2)")
+else(COMPILATION_ARCH MATCHES "OFF")
+ message("Compilation architecture: ${COMPILATION_ARCH}")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=${COMPILATION_ARCH}")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${COMPILATION_ARCH}")
+endif(COMPILATION_ARCH MATCHES "OFF")
+
+if(ENABLE_AVX2 MATCHES "OFF")
+ message("Disabling AVX2 instructions")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mno-avx2")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mno-avx2")
+endif(ENABLE_AVX2 MATCHES "OFF")
+
+# Manages build types
+if(NOT CMAKE_BUILD_TYPE)
+ set(CMAKE_BUILD_TYPE "Release")
+endif(NOT CMAKE_BUILD_TYPE)
+
+if(CMAKE_BUILD_TYPE MATCHES Debug)
+ message("Build type: Debug")
+ add_compile_options(-g)
+else(CMAKE_BUILD_TYPE MATCHES Debug)
+ if(CMAKE_BUILD_TYPE MATCHES Profile)
+ message("Build type: Profiling")
+ add_compile_options(-pg)
+ set(CMAKE_SHARED_LINKER_FLAGS "-pg")
+ set(CMAKE_EXE_LINKER_FLAGS "-pg")
+ else(CMAKE_BUILD_TYPE MATCHES Profile)
+ message("Build type: Release")
+ add_compile_options(-O3)
+ endif(CMAKE_BUILD_TYPE MATCHES Profile) 
+endif(CMAKE_BUILD_TYPE MATCHES Debug)
+
+MATH(EXPR PRINT_MAX_KMER_SIZE "${MAX_KMER_SIZE}-1")
+message("Maximum k-mer size: " ${PRINT_MAX_KMER_SIZE})
+
+MATH(EXPR PRINT_MAX_GMER_SIZE "${MAX_GMER_SIZE}-1")
+message("Maximum g-mer size: " ${PRINT_MAX_GMER_SIZE})
+
+add_subdirectory(src)

ext/bifrost/Changelog.md

@@ -0,0 +1,30 @@
+### Changelog
+
+API only.
+
+* **15-06-2022**
+ * Function `CompactedDBG()::write()` takes additional arguments with default values:
+  * `compress_output` indicates whether the output should be compressed
+  * `write_index_file` indicates whether an index file should be generated to enable faster graph loading
+  * `GFA_output`, `FASTA_output` and `BFG_output` to select the output file format (previously, setting `outputGFA` set to false would automatically make `write()` output the graph in FASTA format).
+  
+  Beware that the new arguments come with default values which could override the default values of the previous versions of `write()`, e.g, the default value of parameter `FASTA_output` (`write()` with 8 parameters) could be used as the default value of parameter `verbose` if your code is not updated (`write()` with 5 parameters).
+ * There exists two versions of `CompactedDBG::read()` and `ColoredCDBG::read()`, the "usual" (slower) graph reading function and the same function with an additional index graph file as input. Using the index graph file as input considerably speeds-up the graph loading in memory. The "usual" graph reading function will automatically use the graph index file if available.
+* **04-28-2022**
+ * Color files generated prior to version 1.0.6.2 are **not** compatible with version 1.0.6.2 and onward.
+ * `CompactedDBG::simplify()` and `ColoredCDBG::simplify()` now return true even if no simplification was performed ("null-simplification" in case all input parameters are set to false). The goal is to only return false if the graph is invalid or in case of unexpected behavior. 
+* **08-29-2018**
+ * `UnitigColors::const_iterator` only considers now the k-mer positions of the unitig mapping provided in the `UnitigMap`/`UnitigColorMap` parameter of `UnitigColors::begin()`.
+* **08-28-2018**
+ * Functions `CDBG_Data_t::serialize()` and `CCDBG_Data_t::serialize()` have now one parameter which is a `const_UnitigMap`/`const_UnitigColorMap` reference representing the (reference) unitig to which the data are associated to.
+* **08-23-2018**
+ * Function `UnitigMap::toString()` was ambiguous as if it would generate the string of the mapped sequence or the string of the reference unitig used in the mapping. It has been replaced by two functions: `UnitigMap::mappedSequenceToString()` and `UnitigMap::referenceUnitigToString()`.
+* **08-07-2018**
+ * Add de Bruijn graphs merging functions (`CompactedDBG::merge()` and `ColoredCDBG::merge()`) and addition assignment operators (`CompactedDBG::operator+=()` and `ColoredCDBG::operator+=()`, same as `merge()` but uses only one thread).
+ * Add de Bruijn graphs comparison functions `CompactedDBG::operator==()`, `CompactedDBG::operator!=()`, `ColoredCDBG::operator==()` and `ColoredCDBG::operator!=()`.
+ * Delete `CompactedDBG::empty()` and `ColoredCDBG::empty()` to be consistent with STD containers (those functions were emptying the graph of its content while `empty()` of STD containers returns whether the container is empty). Now, to empty the graph, use `CompactedDBG::clear()` and `ColoredCDBG::clear()`.
+ * Major changes in the abstract class `CDBG_Data_t` and `CCDBG_Data_t`:
+  * All the functions are now **non**-static.
+  * Function `join()` is renamed `concat()` and works a bit differently (have a look at the doc). Quickly, `join()` was concatenating two unitigs A and B such that the result was A=AB and B was deleted from the graph. Now, `concat()` deletes A and B from the graph and adds a new unitig C=AB.
+  * Function `sub()` is renamed `extract()`.
+  * Add the functions `merge()` and `clear()` which **must** be overloaded too in the derived class of `CDBG_Data_t` and `CCDBG_Data_t`.