New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
【PaddlePaddle Hackathon 2】16 新增 API RRelu #41823
Changes from 13 commits
b4022f3
b03c8d1
a83e2bb
b34da04
e5f3910
a5388cb
1ee342e
f74eb8a
bb3b47e
71fdbab
c51cc32
f7cf53b
a0cd822
e5cf547
a6f7c04
fc85f79
b50216e
db09397
90102c0
28cd511
a5fed7f
f2a9834
916b5b8
51493a3
726a5d5
4502a01
2132bf5
116873b
8f626c2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,153 @@ | ||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. */ | ||
|
||
#include <memory> | ||
#include <string> | ||
#include "paddle/fluid/framework/infershape_utils.h" | ||
#include "paddle/fluid/framework/op_registry.h" | ||
#include "paddle/phi/infermeta/unary.h" | ||
|
||
namespace paddle { | ||
namespace operators { | ||
|
||
using framework::Tensor; | ||
|
||
class RReluOp : public framework::OperatorWithKernel { | ||
public: | ||
using framework::OperatorWithKernel::OperatorWithKernel; | ||
|
||
protected: | ||
framework::OpKernelType GetExpectedKernelType( | ||
const framework::ExecutionContext& ctx) const override { | ||
return framework::OpKernelType( | ||
OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace()); | ||
} | ||
}; | ||
|
||
class RReluOpMaker : public framework::OpProtoAndCheckerMaker { | ||
public: | ||
void Make() override { | ||
AddInput("X", "The input of RReLU op."); | ||
AddOutput("Out", "The output of RReLU op."); | ||
AddOutput("Noise", "The random sampled RReLU noise.") | ||
.AsIntermediate() | ||
.AsExtra(); | ||
AddAttr<bool>("is_test", | ||
"(bool, default false) Set to true for inference only, false " | ||
"for training. Some layers may run faster when this is true.") | ||
.SetDefault(false); | ||
AddAttr<bool>("fix_seed", | ||
"A flag indicating whether to use a fixed seed to generate " | ||
"random mask. NOTE: DO NOT set this flag to true in " | ||
"training. Setting this flag to true is only useful in " | ||
"unittest or for debug that always the same output units " | ||
"will be dropped.") | ||
.SetDefault(false) | ||
.AsExtra(); | ||
AddAttr<int>("seed", "Rrelu random seed.").SetDefault(0).AsExtra(); | ||
|
||
float default_lower = 1. / 8.; | ||
AddAttr<float>("lower", "Lower bound of the uniform distribution.") | ||
.SetDefault(default_lower) | ||
.AddCustomChecker([](const float& lower) { | ||
PADDLE_ENFORCE_EQ(lower >= 0.0f && lower < 1.0f, true, | ||
platform::errors::InvalidArgument( | ||
"'RRelu_lower' must be between 0.0 and 1.0.")); | ||
}); | ||
float defalut_upper = 1. / 3.; | ||
AddAttr<float>("upper", "Upper bound of the uniform distribution.") | ||
.SetDefault(defalut_upper) | ||
.AddCustomChecker([](const float& upper) { | ||
PADDLE_ENFORCE_EQ(upper > 0.0f && upper <= 1.0f, true, | ||
platform::errors::InvalidArgument( | ||
"'RRelu_upper' must be between 0.0 and 1.0.")); | ||
}); | ||
AddComment(R"DOC( | ||
RReLU Operator. | ||
|
||
Applies the randomized leaky rectified liner unit function, element-wise, | ||
as described in the paper: | ||
|
||
`Empirical Evaluation of Rectified Activations in Convolutional Network`_. | ||
|
||
The function is defined as: | ||
|
||
.. math:: | ||
\text{RReLU}(x) = | ||
\begin{cases} | ||
x & \text{if } x \geq 0 \\ | ||
ax & \text{ otherwise } | ||
\end{cases} | ||
|
||
where :math:`a` is randomly sampled from uniform distribution | ||
:math:`\mathcal{U}(\text{lower}, \text{upper})`. | ||
|
||
See: https://arxiv.org/pdf/1505.00853.pdf | ||
|
||
)DOC"); | ||
} | ||
}; | ||
|
||
class RReluGradOp : public framework::OperatorWithKernel { | ||
public: | ||
using framework::OperatorWithKernel::OperatorWithKernel; | ||
|
||
void InferShape(framework::InferShapeContext* ctx) const override { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 正向的infershape已经放到phi里了,反向的infershape也一起放过去吧 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 完成 |
||
OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "rrelu"); | ||
OP_INOUT_CHECK(ctx->HasInput("Noise"), "Input", "Noise", "rrelu"); | ||
OP_INOUT_CHECK(ctx->HasInput(framework::GradVarName("Out")), "Input", | ||
framework::GradVarName("Out"), "rrelu"); | ||
|
||
auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); | ||
ctx->SetOutputDim(framework::GradVarName("X"), out_dims); | ||
ctx->ShareLoD(framework::GradVarName("Out"), | ||
/*->*/ framework::GradVarName("X")); | ||
} | ||
|
||
protected: | ||
framework::OpKernelType GetExpectedKernelType( | ||
const framework::ExecutionContext& ctx) const override { | ||
return framework::OpKernelType(OperatorWithKernel::IndicateVarDataType( | ||
ctx, framework::GradVarName("Out")), | ||
ctx.GetPlace()); | ||
} | ||
}; | ||
|
||
template <typename T> | ||
class RReluGradOpMaker : public framework::SingleGradOpMaker<T> { | ||
public: | ||
using framework::SingleGradOpMaker<T>::SingleGradOpMaker; | ||
|
||
protected: | ||
void Apply(GradOpPtr<T> op) const override { | ||
op->SetType("rrelu_grad"); | ||
op->SetInput("X", this->Input("X")); | ||
op->SetInput("Noise", this->Output("Noise")); | ||
op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); | ||
op->SetOutput(framework::GradVarName("X"), this->InputGrad("X")); | ||
} | ||
}; | ||
|
||
} // namespace operators | ||
} // namespace paddle | ||
|
||
namespace ops = paddle::operators; | ||
DECLARE_INFER_SHAPE_FUNCTOR(rrelu, RReluInferShapeFunctor, | ||
PD_INFER_META(phi::RReluInferMeta)); | ||
|
||
REGISTER_OPERATOR(rrelu, ops::RReluOp, ops::RReluOpMaker, | ||
ops::RReluGradOpMaker<paddle::framework::OpDesc>, | ||
ops::RReluGradOpMaker<paddle::imperative::OpBase>, | ||
RReluInferShapeFunctor); | ||
REGISTER_OPERATOR(rrelu_grad, ops::RReluGradOp); |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
#include "paddle/phi/kernels/rrelu_grad_kernel.h" | ||
|
||
#include "paddle/phi/backends/cpu/cpu_context.h" | ||
#include "paddle/phi/core/kernel_registry.h" | ||
|
||
namespace phi { | ||
|
||
template <typename T, typename Context> | ||
void RReluGradKernel(const Context& dev_ctx, | ||
const DenseTensor& x, | ||
const DenseTensor& noise, | ||
const DenseTensor& out_grad, | ||
DenseTensor* x_grad) { | ||
const T* n_ptr = noise.data<T>(); | ||
const T* x_ptr = x.data<T>(); | ||
const T* out_grad_ptr = out_grad.data<T>(); | ||
int numel = x.numel(); | ||
if (!x_grad) return; | ||
|
||
int i = 0; | ||
T* x_grad_ptr = dev_ctx.template Alloc<T>(x_grad); | ||
for (i = 0; i < numel; i++) { | ||
x_grad_ptr[i] = x_ptr[i] > 0 ? out_grad_ptr[i] : n_ptr[i] * out_grad_ptr[i]; | ||
} | ||
} | ||
|
||
} // namespace phi | ||
|
||
PD_REGISTER_KERNEL( | ||
rrelu_grad, CPU, ALL_LAYOUT, phi::RReluGradKernel, float, double) {} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
#include "paddle/phi/kernels/rrelu_kernel.h" | ||
|
||
#include "paddle/fluid/framework/generator.h" | ||
#include "paddle/phi/backends/cpu/cpu_context.h" | ||
#include "paddle/phi/core/kernel_registry.h" | ||
|
||
namespace phi { | ||
|
||
template <typename T, typename Context> | ||
void RReluKernel(const Context& dev_ctx, | ||
const DenseTensor& x, | ||
const float lower, | ||
const float upper, | ||
bool is_test, | ||
bool fix_seed, | ||
int seed, | ||
DenseTensor* out, | ||
DenseTensor* noise) { | ||
const T* x_ptr = x.data<T>(); | ||
T* o_ptr = dev_ctx.template Alloc<T>(out); | ||
T* n_ptr = dev_ctx.template Alloc<T>(noise); | ||
T zero = static_cast<T>(0); | ||
int numel = x.numel(); | ||
int i = 0; | ||
|
||
if (is_test) { | ||
for (i = 0; i < numel; i++) { | ||
T mid_val = static_cast<T>((lower + upper) / 2.0); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这个mid_val计算可以放到for循环外边吧 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 完成 |
||
if (x_ptr[i] < zero) { | ||
o_ptr[i] = mid_val * x_ptr[i]; | ||
n_ptr[i] = mid_val; | ||
} else { | ||
o_ptr[i] = x_ptr[i]; | ||
n_ptr[i] = 1.0; | ||
} | ||
} | ||
|
||
return; | ||
} | ||
|
||
int seed_data = fix_seed ? seed : 0; | ||
auto engine = paddle::framework::GetCPURandomEngine(seed_data); | ||
|
||
std::uniform_real_distribution<float> dist(lower, upper); | ||
|
||
for (i = 0; i < numel; i++) { | ||
if (x_ptr[i] < zero) { | ||
T scale = static_cast<T>(dist(*engine)); | ||
o_ptr[i] = scale * x_ptr[i]; | ||
n_ptr[i] = scale; | ||
} else { | ||
o_ptr[i] = x_ptr[i]; | ||
n_ptr[i] = 1.0; | ||
} | ||
} | ||
} | ||
|
||
} // namespace phi | ||
|
||
PD_REGISTER_KERNEL(rrelu, CPU, ALL_LAYOUT, phi::RReluKernel, float, double) {} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
2016->2022,其他文件自查一下吧
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
完成