Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

【complex op】No.贰〇 add complex support for tanhshrink #56277

Closed
wants to merge 17 commits into from
Closed

【complex op】No.贰〇 add complex support for tanhshrink #56277

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
81ce4cc
add tanhshrink in complex.h
Liyulingyue Aug 14, 2023
cbd6714
add tanhshrink in cpu/gpu forward files
Liyulingyue Aug 14, 2023
1ea10f8
add tanhshrink in cpu/gpu background files
Liyulingyue Aug 14, 2023
635de4a
add tanhshrink in activation_functor.h
Liyulingyue Aug 14, 2023
ec823b3
Apply suggestions from code review
Liyulingyue Aug 18, 2023
dc7f746
Update activation_functor.h
Liyulingyue Aug 19, 2023
6ce8370
Update activation_grad_kernel.cu
Liyulingyue Aug 19, 2023
9e35b7c
Update activation_grad_kernel.cu
Liyulingyue Aug 19, 2023
6e52158
Update activation_kernel.cu
Liyulingyue Aug 19, 2023
0b04f3c
Apply suggestions from code review
Liyulingyue Aug 19, 2023
d2af304
Update activation_grad_kernel.cu
Liyulingyue Aug 20, 2023
8bf987a
codestyle
Liyulingyue Aug 20, 2023
82d5a12
add docs of tanhshrink
Liyulingyue Aug 20, 2023
b3d456a
add test of tanhshrink
Liyulingyue Aug 20, 2023
915f6ca
add cuda functor
Liyulingyue Aug 23, 2023
02a5add
add unittest
Liyulingyue Aug 23, 2023
42b21f3
Merge branch 'PaddlePaddle:develop' into tanhshrink
Liyulingyue Aug 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions paddle/phi/common/complex.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,16 @@ HOSTDEVICE inline complex<T> sin(const complex<T>& a) {
#endif
}

template <typename T>
HOSTDEVICE inline complex<T> tanhshrink(const complex<T>& a) {
#if defined(PADDLE_WITH_CUDA_OR_HIP_COMPLEX) && \
(defined(__CUDA_ARCH__) || defined(__HIPCC__))
return complex<T>(thrust::tanhshrink(thrust::complex<T>(a)));
#else
return complex<T>(std::tanhshrink(std::complex<T>(a)));
Liyulingyue marked this conversation as resolved.
Show resolved Hide resolved
#endif
}

Liyulingyue marked this conversation as resolved.
Show resolved Hide resolved
template <typename T>
HOSTDEVICE inline complex<T> cos(const complex<T>& a) {
#if defined(PADDLE_WITH_CUDA_OR_HIP_COMPLEX) && \
Expand Down
2 changes: 2 additions & 0 deletions paddle/phi/kernels/cpu/activation_grad_kernel.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,8 @@ PD_REGISTER_ACTIVATION_GRAD_KERNEL(relu6_grad, Relu6GradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(softshrink_grad, SoftShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(hard_shrink_grad, HardShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(tanh_shrink_grad, TanhShrinkGradKernel)
Liyulingyue marked this conversation as resolved.
Show resolved Hide resolved
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX(tanh_shrink_grad,
TanhShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(elu_grad, EluGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(silu_grad, SiluGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(mish_grad, MishGradKernel)
Expand Down
1 change: 1 addition & 0 deletions paddle/phi/kernels/cpu/activation_kernel.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ PD_REGISTER_ACTIVATION_KERNEL(thresholded_relu, ThresholdedReluKernel)
PD_REGISTER_ACTIVATION_KERNEL(hard_shrink, HardShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL(softshrink, SoftShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL(tanh_shrink, TanhShrinkKernel)
Liyulingyue marked this conversation as resolved.
Show resolved Hide resolved
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(tanh_shrink, TanhShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL(elu, EluKernel)
PD_REGISTER_ACTIVATION_KERNEL(silu, SiluKernel)
PD_REGISTER_ACTIVATION_KERNEL(mish, MishKernel)
Expand Down
18 changes: 18 additions & 0 deletions paddle/phi/kernels/funcs/activation_functor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1628,6 +1628,24 @@ struct TanhShrinkFunctor : public BaseActivationFunctor<T> {
}
};

template <typename T>
struct TanhShrinkGradFunctor<ComplexType<T>>
: public BaseActivationFunctor<ComplexType<T>> {
template <typename Device,
typename X,
typename Out,
typename dOut,
typename dX>
void operator()(Device d, X x UNUSED, Out out, dOut dout, dX dx) const {
dx.device(d) =
static_cast<ComplexType<T>>(1) -
dout *
(static_cast<ComplexType<T>>(1) - out * out).unaryExpr(Conj<T>());
}

static constexpr ActBwdOpFwdDeps FwdDeps() { return ActBwdOpFwdDeps::kDepX; }
};

template <typename T>
struct TanhShrinkGradFunctor : public BaseActivationFunctor<T> {
template <typename Device,
Expand Down
2 changes: 2 additions & 0 deletions paddle/phi/kernels/gpu/activation_grad_kernel.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,8 @@ PD_REGISTER_KERNEL(exp_grad,
PD_REGISTER_ACTIVATION_GRAD_KERNEL(softshrink_grad, SoftShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(hard_shrink_grad, HardShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(tanh_shrink_grad, TanhShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(
PD_REGISTER_ACTIVATION_GRAD_KERNEL_WITH_COMPLEX, TanhShrinkGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(silu_grad, SiluGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(elu_grad, EluGradKernel)
PD_REGISTER_ACTIVATION_GRAD_KERNEL(elu_double_grad, EluDoubleGradKernel)
Expand Down
1 change: 1 addition & 0 deletions paddle/phi/kernels/gpu/activation_kernel.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,7 @@ PD_REGISTER_KERNEL(square,
phi::dtype::bfloat16) {}

PD_REGISTER_ACTIVATION_KERNEL(hard_shrink, HardShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL_WITH_COMPLEX(hard_shrink, HardShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL(softshrink, SoftShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL(tanh_shrink, TanhShrinkKernel)
PD_REGISTER_ACTIVATION_KERNEL(elu, EluKernel)
Expand Down