Sparsity format Reference and OpenMP kernels

core/CMakeLists.txt

@@ -22,7 +22,7 @@ target_sources(ginkgo
  matrix/hybrid.cpp
  matrix/identity.cpp
  matrix/sellp.cpp
- matrix/sparsity.cpp
+ matrix/sparsity_csr.cpp
  preconditioner/jacobi.cpp
  solver/bicgstab.cpp
  solver/cg.cpp

core/device_hooks/common_kernels.inc.cpp

@@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/ell_kernels.hpp"
 #include "core/matrix/hybrid_kernels.hpp"
 #include "core/matrix/sellp_kernels.hpp"
-#include "core/matrix/sparsity_kernels.hpp"
+#include "core/matrix/sparsity_csr_kernels.hpp"
 #include "core/preconditioner/jacobi_kernels.hpp"
 #include "core/solver/bicgstab_kernels.hpp"
 #include "core/solver/cg_kernels.hpp"
@@ -125,6 +125,12 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
  GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL);
 
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+ GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL);
+
 template <typename ValueType>
 GKO_DECLARE_DENSE_COUNT_NONZEROS_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
@@ -352,52 +358,54 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 } // namespace ir
 
 
-namespace sparsity {
+namespace sparsity_csr {
 
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_SPARSITY_SPMV_KERNEL(ValueType, IndexType)
+GKO_DECLARE_SPARSITY_CSR_SPMV_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_SPARSITY_SPMV_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+ GKO_DECLARE_SPARSITY_CSR_SPMV_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_SPARSITY_ADVANCED_SPMV_KERNEL(ValueType, IndexType)
+GKO_DECLARE_SPARSITY_CSR_ADVANCED_SPMV_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
- GKO_DECLARE_SPARSITY_ADVANCED_SPMV_KERNEL);
+ GKO_DECLARE_SPARSITY_CSR_ADVANCED_SPMV_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_SPARSITY_COUNT_NUM_DIAGONAL_ELEMENTS_KERNEL(ValueType, IndexType)
+GKO_DECLARE_SPARSITY_CSR_COUNT_NUM_DIAGONAL_ELEMENTS_KERNEL(ValueType,
+ IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
- GKO_DECLARE_SPARSITY_COUNT_NUM_DIAGONAL_ELEMENTS_KERNEL);
+ GKO_DECLARE_SPARSITY_CSR_COUNT_NUM_DIAGONAL_ELEMENTS_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_SPARSITY_REMOVE_DIAGONAL_ELEMENTS_KERNEL(ValueType, IndexType)
+GKO_DECLARE_SPARSITY_CSR_REMOVE_DIAGONAL_ELEMENTS_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
- GKO_DECLARE_SPARSITY_REMOVE_DIAGONAL_ELEMENTS_KERNEL);
+ GKO_DECLARE_SPARSITY_CSR_REMOVE_DIAGONAL_ELEMENTS_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_SPARSITY_TRANSPOSE_KERNEL(ValueType, IndexType)
+GKO_DECLARE_SPARSITY_CSR_TRANSPOSE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
- GKO_DECLARE_SPARSITY_TRANSPOSE_KERNEL);
+ GKO_DECLARE_SPARSITY_CSR_TRANSPOSE_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_SPARSITY_SORT_BY_COLUMN_INDEX(ValueType, IndexType)
+GKO_DECLARE_SPARSITY_CSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
- GKO_DECLARE_SPARSITY_SORT_BY_COLUMN_INDEX);
+ GKO_DECLARE_SPARSITY_CSR_SORT_BY_COLUMN_INDEX);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_SPARSITY_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType)
+GKO_DECLARE_SPARSITY_CSR_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
- GKO_DECLARE_SPARSITY_IS_SORTED_BY_COLUMN_INDEX);
+ GKO_DECLARE_SPARSITY_CSR_IS_SORTED_BY_COLUMN_INDEX);
 
 
-} // namespace sparsity
+} // namespace sparsity_csr
 
 
 namespace csr {

core/matrix/csr.cpp

@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/ell.hpp>
 #include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
 #include "core/matrix/csr_kernels.hpp"
@@ -187,6 +188,32 @@ void Csr<ValueType, IndexType>::move_to(Sellp<ValueType, IndexType> *result)
 }
 
 
+template <typename ValueType, typename IndexType>
+void Csr<ValueType, IndexType>::convert_to(
+ SparsityCsr<ValueType, IndexType> *result) const
+{
+ auto exec = this->get_executor();
+ auto tmp = SparsityCsr<ValueType, IndexType>::create(
+ exec, this->get_size(), this->get_num_stored_elements());
+ tmp->col_idxs_ = this->col_idxs_;
+ tmp->row_ptrs_ = this->row_ptrs_;
+ if (result->value_.get_data()) {
+ tmp->value_ = result->value_;
+ } else {
+ tmp->value_ = gko::Array<ValueType>(exec, {one<ValueType>()});
+ }
+ tmp->move_to(result);
+}
+
+
+template <typename ValueType, typename IndexType>
+void Csr<ValueType, IndexType>::move_to(
+ SparsityCsr<ValueType, IndexType> *result)
+{
+ this->convert_to(result);
+}
+
+
 template <typename ValueType, typename IndexType>
 void Csr<ValueType, IndexType>::convert_to(
  Ell<ValueType, IndexType> *result) const

core/matrix/csr_kernels.hpp

@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/ell.hpp>
 #include <ginkgo/core/matrix/hybrid.hpp>
 #include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
 namespace gko {

core/matrix/dense.cpp

@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/ell.hpp>
 #include <ginkgo/core/matrix/hybrid.hpp>
 #include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
 #include "core/matrix/dense_kernels.hpp"
@@ -75,6 +76,7 @@ GKO_REGISTER_OPERATION(convert_to_csr, dense::convert_to_csr);
 GKO_REGISTER_OPERATION(convert_to_ell, dense::convert_to_ell);
 GKO_REGISTER_OPERATION(convert_to_hybrid, dense::convert_to_hybrid);
 GKO_REGISTER_OPERATION(convert_to_sellp, dense::convert_to_sellp);
+GKO_REGISTER_OPERATION(convert_to_sparsity_csr, dense::convert_to_sparsity_csr);
 
 
 } // namespace dense
@@ -181,6 +183,22 @@ inline void conversion_helper(Sellp<ValueType, IndexType> *result,
 }
 
 
+template <typename ValueType, typename IndexType, typename MatrixType,
+ typename OperationType>
+inline void conversion_helper(SparsityCsr<ValueType, IndexType> *result,
+ MatrixType *source, const OperationType &op)
+{
+ auto exec = source->get_executor();
+
+ size_type num_stored_nonzeros = 0;
+ exec->run(dense::make_count_nonzeros(source, &num_stored_nonzeros));
+ auto tmp = SparsityCsr<ValueType, IndexType>::create(
+ exec, source->get_size(), num_stored_nonzeros);
+ exec->run(op(tmp.get(), source));
+ tmp->move_to(result);
+}
+
+
 } // namespace
 
 
@@ -423,6 +441,38 @@ void Dense<ValueType>::move_to(Sellp<ValueType, int64> *result)
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::convert_to(SparsityCsr<ValueType, int32> *result) const
+{
+ conversion_helper(result, this,
+ dense::template make_convert_to_sparsity_csr<
+ decltype(result), const Dense<ValueType> *&>);
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::move_to(SparsityCsr<ValueType, int32> *result)
+{
+ this->convert_to(result);
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::convert_to(SparsityCsr<ValueType, int64> *result) const
+{
+ conversion_helper(result, this,
+ dense::template make_convert_to_sparsity_csr<
+ decltype(result), const Dense<ValueType> *&>);
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::move_to(SparsityCsr<ValueType, int64> *result)
+{
+ this->convert_to(result);
+}
+
+
 namespace {
 
 

core/matrix/dense_kernels.hpp

@@ -98,6 +98,11 @@ namespace kernels {
  matrix::Sellp<_type, _prec> *other, \
  const matrix::Dense<_type> *source)
 
+#define GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL(_type, _prec) \
+ void convert_to_sparsity_csr(std::shared_ptr<const DefaultExecutor> exec, \
+ matrix::SparsityCsr<_type, _prec> *other, \
+ const matrix::Dense<_type> *source)
+
 #define GKO_DECLARE_DENSE_COUNT_NONZEROS_KERNEL(_type) \
  void count_nonzeros(std::shared_ptr<const DefaultExecutor> exec, \
  const matrix::Dense<_type> *source, size_type *result)
@@ -128,40 +133,42 @@ namespace kernels {
  matrix::Dense<_type> *trans, \
  const matrix::Dense<_type> *orig)
 
-#define GKO_DECLARE_ALL_AS_TEMPLATES \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL(ValueType); \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType); \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_SCALE_KERNEL(ValueType); \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_ADD_SCALED_KERNEL(ValueType); \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL(ValueType); \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_COMPUTE_NORM2_KERNEL(ValueType); \
- template <typename ValueType, typename IndexType> \
- GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL(ValueType, IndexType); \
- template <typename ValueType, typename IndexType> \
- GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL(ValueType, IndexType); \
- template <typename ValueType, typename IndexType> \
- GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL(ValueType, IndexType); \
- template <typename ValueType, typename IndexType> \
- GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType); \
- template <typename ValueType, typename IndexType> \
- GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType); \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_COUNT_NONZEROS_KERNEL(ValueType); \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType); \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_CALCULATE_NONZEROS_PER_ROW_KERNEL(ValueType); \
- template <typename ValueType> \
- GKO_DECLARE_DENSE_CALCULATE_TOTAL_COLS_KERNEL(ValueType); \
- template <typename ValueType> \
- GKO_DECLARE_TRANSPOSE_KERNEL(ValueType); \
- template <typename ValueType> \
+#define GKO_DECLARE_ALL_AS_TEMPLATES \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL(ValueType); \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType); \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_SCALE_KERNEL(ValueType); \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_ADD_SCALED_KERNEL(ValueType); \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL(ValueType); \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_COMPUTE_NORM2_KERNEL(ValueType); \
+ template <typename ValueType, typename IndexType> \
+ GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL(ValueType, IndexType); \
+ template <typename ValueType, typename IndexType> \
+ GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL(ValueType, IndexType); \
+ template <typename ValueType, typename IndexType> \
+ GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL(ValueType, IndexType); \
+ template <typename ValueType, typename IndexType> \
+ GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType); \
+ template <typename ValueType, typename IndexType> \
+ GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType); \
+ template <typename ValueType, typename IndexType> \
+ GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL(ValueType, IndexType); \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_COUNT_NONZEROS_KERNEL(ValueType); \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType); \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_CALCULATE_NONZEROS_PER_ROW_KERNEL(ValueType); \
+ template <typename ValueType> \
+ GKO_DECLARE_DENSE_CALCULATE_TOTAL_COLS_KERNEL(ValueType); \
+ template <typename ValueType> \
+ GKO_DECLARE_TRANSPOSE_KERNEL(ValueType); \
+ template <typename ValueType> \
  GKO_DECLARE_CONJ_TRANSPOSE_KERNEL(ValueType)