Skip to content

Commit

Permalink
add new interface and kernels
Browse files Browse the repository at this point in the history
  • Loading branch information
@upsj
upsj committed Oct 6, 2023
1 parent c545cf4 commit 804aba3
Show file tree
Hide file tree
Showing 48 changed files with 6,581 additions and 4,292 deletions.
632 changes: 632 additions & 0 deletions common/cuda_hip/matrix/csr_kernels.hpp.inc
View file

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions common/unified/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ set(UNIFIED_SOURCES
matrix/csr_kernels.cpp
matrix/ell_kernels.cpp
matrix/hybrid_kernels.cpp
matrix/permutation_kernels.cpp
matrix/scaled_permutation_kernels.cpp
matrix/sellp_kernels.cpp
matrix/sparsity_csr_kernels.cpp
matrix/diagonal_kernels.cpp
Expand Down
56 changes: 37 additions & 19 deletions common/unified/matrix/csr_kernels.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,53 +54,71 @@ namespace GKO_DEVICE_NAMESPACE {
namespace csr {


template <typename IndexType>
void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
size_type size, const IndexType* permutation_indices,
IndexType* inv_permutation)
template <typename ValueType, typename IndexType>
void inv_col_permute(std::shared_ptr<const DefaultExecutor> exec,
const IndexType* perm,
const matrix::Csr<ValueType, IndexType>* orig,
matrix::Csr<ValueType, IndexType>* col_permuted)
{
auto num_rows = orig->get_size()[0];
auto nnz = orig->get_num_stored_elements();
auto size = std::max(num_rows, nnz);
run_kernel(
exec,
[] GKO_KERNEL(auto tid, auto permutation, auto inv_permutation) {
inv_permutation[permutation[tid]] = tid;
[] GKO_KERNEL(auto tid, auto num_rows, auto num_nonzeros,
auto permutation, auto in_row_ptrs, auto in_col_idxs,
auto in_vals, auto out_row_ptrs, auto out_col_idxs,
auto out_vals) {
if (tid < num_nonzeros) {
out_col_idxs[tid] = permutation[in_col_idxs[tid]];
out_vals[tid] = in_vals[tid];
}
if (tid <= num_rows) {
out_row_ptrs[tid] = in_row_ptrs[tid];
}
},
size, permutation_indices, inv_permutation);
size, num_rows, nnz, perm, orig->get_const_row_ptrs(),
orig->get_const_col_idxs(), orig->get_const_values(),
col_permuted->get_row_ptrs(), col_permuted->get_col_idxs(),
col_permuted->get_values());
}

GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_CSR_INV_COL_PERMUTE_KERNEL);


template <typename ValueType, typename IndexType>
void inverse_column_permute(std::shared_ptr<const DefaultExecutor> exec,
const IndexType* perm,
const matrix::Csr<ValueType, IndexType>* orig,
matrix::Csr<ValueType, IndexType>* column_permuted)
void inv_col_scale_permute(std::shared_ptr<const DefaultExecutor> exec,
const ValueType* scale, const IndexType* perm,
const matrix::Csr<ValueType, IndexType>* orig,
matrix::Csr<ValueType, IndexType>* col_permuted)
{
auto num_rows = orig->get_size()[0];
auto nnz = orig->get_num_stored_elements();
auto size = std::max(num_rows, nnz);
run_kernel(
exec,
[] GKO_KERNEL(auto tid, auto num_rows, auto num_nonzeros,
[] GKO_KERNEL(auto tid, auto num_rows, auto num_nonzeros, auto scale,
auto permutation, auto in_row_ptrs, auto in_col_idxs,
auto in_vals, auto out_row_ptrs, auto out_col_idxs,
auto out_vals) {
if (tid < num_nonzeros) {
out_col_idxs[tid] = permutation[in_col_idxs[tid]];
out_vals[tid] = in_vals[tid];
const auto in_col = in_col_idxs[tid];
out_col_idxs[tid] = permutation[in_col];
out_vals[tid] = in_vals[tid] / scale[in_col];
}
if (tid <= num_rows) {
out_row_ptrs[tid] = in_row_ptrs[tid];
}
},
size, num_rows, nnz, perm, orig->get_const_row_ptrs(),
size, num_rows, nnz, scale, perm, orig->get_const_row_ptrs(),
orig->get_const_col_idxs(), orig->get_const_values(),
column_permuted->get_row_ptrs(), column_permuted->get_col_idxs(),
column_permuted->get_values());
col_permuted->get_row_ptrs(), col_permuted->get_col_idxs(),
col_permuted->get_values());
}

GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
GKO_DECLARE_CSR_INV_COL_SCALE_PERMUTE_KERNEL);


template <typename ValueType, typename IndexType>
Expand Down
24 changes: 22 additions & 2 deletions common/unified/matrix/dense_kernels.instantiate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,36 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_NONSYMM_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_INV_NONSYMM_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE_2(
GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE_2(
GKO_DECLARE_DENSE_ADVANCED_ROW_GATHER_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
GKO_DECLARE_DENSE_COL_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
GKO_DECLARE_DENSE_INV_COL_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_SYMM_SCALE_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_INV_SYMM_SCALE_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_NONSYMM_SCALE_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_INV_NONSYMM_SCALE_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_ROW_SCALE_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_INV_ROW_SCALE_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_COL_SCALE_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_INV_COL_SCALE_PERMUTE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
Expand Down
Loading

0 comments on commit 804aba3

Please sign in to comment.